From 95b781ca66e9bdd030c7e99bf974279f4deb540e Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 1 Nov 2023 16:46:20 +0100
Subject: [PATCH 001/205] chore: refactor core import_elements

---
 rdmo/management/imports.py | 48 +++++++++++++-------------------------
 1 file changed, 16 insertions(+), 32 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index e47a3c7001..a9cce13967 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -7,6 +7,20 @@
 from rdmo.tasks.imports import import_task
 from rdmo.views.imports import import_view
 
+ELEMENT_IMPORT_METHODS = {
+    "conditions.condition": import_condition,
+    "domain.attribute": import_attribute,
+    "options.optionset": import_optionset,
+    "options.option": import_option,
+    "questions.catalog": import_catalog,
+    "questions.section": import_section,
+    "questions.page": import_page,
+    "questions.questionset": import_questionset,
+    "questions.question": import_question,
+    "tasks.task": import_task,
+    "views.view": import_view,
+}
+
 
 def import_elements(elements, save=True, user=None):
     for element in elements:
@@ -19,38 +33,8 @@ def import_elements(elements, save=True, user=None):
             'updated': False
         })
 
-        if model == 'conditions.condition':
-            import_condition(element, save, user)
-
-        elif model == 'domain.attribute':
-            import_attribute(element, save, user)
-
-        elif model == 'options.optionset':
-            import_optionset(element, save, user)
-
-        elif model == 'options.option':
-            import_option(element, save, user)
-
-        elif model == 'questions.catalog':
-            import_catalog(element, save, user)
-
-        elif model == 'questions.section':
-            import_section(element, save, user)
-
-        elif model == 'questions.page':
-            import_page(element, save, user)
-
-        elif model == 'questions.questionset':
-            import_questionset(element, save, user)
-
-        elif model == 'questions.question':
-            import_question(element, save, user)
-
-        elif model == 'tasks.task':
-            import_task(element, save, user)
-
-        elif model == 'views.view':
-            import_view(element, save, user)
+        import_method = ELEMENT_IMPORT_METHODS[model]
+        import_method(element, save, user)
 
         element = filter_warnings(element, elements)
 

From 9dcb0438b3bedd68f5d71f5c1e3b697ed59ddb66 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 1 Nov 2023 16:47:22 +0100
Subject: [PATCH 002/205] feat: add check_diff_instance

---
 rdmo/core/imports.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 587afd54e1..b71a675e6e 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -1,3 +1,4 @@
+import difflib
 import logging
 import tempfile
 import time
@@ -261,3 +262,34 @@ def check_permissions(instance, element, user):
         )
         logger.info(message)
         element['errors'].append(message)
+
+
+def check_diff_instance(instance: dict, element: dict, check=False) -> dict[str, list[str]]:
+    if not check:
+        return {}
+    overlapping_keys = set(element.keys()) & set(instance.keys())
+
+    diffs = {}
+    for key in overlapping_keys:
+        element_val = element[key]
+        instance_val = instance[key]
+        if element_val is instance_val:
+            continue
+        if element_val == instance_val:
+            continue
+
+        if isinstance(element_val, str) and isinstance(instance_val, str):
+            diff = diff_str_str(element_val, instance_val)
+            if not diff:
+                continue
+
+            diffs[key] = diff
+    return diffs
+
+
+def diff_str_str(value: str, other: str) -> list[str]:
+    """ checks the diff between two strings """
+    d = difflib.Differ()
+    diff = d.compare(value.splitlines(keepends=True), other.splitlines(keepends=True))
+    ret = list(diff)
+    return ret

From c5d1cf191e012088b8f87dc4797eb1eb1ca497ae Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 1 Nov 2023 16:47:54 +0100
Subject: [PATCH 003/205] feat: add diff to import_attribute method

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/domain/imports.py | 33 +++++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 5a97f6bcfa..fda00dc08a 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -1,8 +1,15 @@
+import copy
 import logging
 
 from django.contrib.sites.models import Site
 
-from rdmo.core.imports import check_permissions, set_common_fields, set_foreign_field, validate_instance
+from rdmo.core.imports import (
+    check_diff_instance,
+    check_permissions,
+    set_common_fields,
+    set_foreign_field,
+    validate_instance,
+)
 
 from .models import Attribute
 from .validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
@@ -13,8 +20,13 @@
 def import_attribute(element, save=False, user=None):
     try:
         attribute = Attribute.objects.get(uri=element.get('uri'))
+        original_attribute = copy.deepcopy(vars(attribute))
+        element['updated'] = True
+        _msg = 'Attribute %s updated.', element.get('uri')
     except Attribute.DoesNotExist:
         attribute = Attribute()
+        element['created'] = True
+        _msg = 'Attribute created with uri %s.', element.get('uri')
 
     set_common_fields(attribute, element)
 
@@ -27,15 +39,20 @@ def import_attribute(element, save=False, user=None):
 
     check_permissions(attribute, element, user)
 
-    if save and not element.get('errors'):
-        if attribute.id:
-            element['updated'] = True
-            logger.debug('Attribute %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.debug('Attribute created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return attribute
 
+    if element['updated']:
+
+        diffs = check_diff_instance(original_attribute, element, check=True)
+        if diffs:
+            # breakpoint()
+            diff_warning = "\n".join([f"{k}:{' '.join(val)}" for k, val in diffs.items()])
+            element['diffs'] = diff_warning
+
+    if save:
         attribute.save()
         attribute.editors.add(Site.objects.get_current())
+        logger.debug(_msg)
 
     return attribute

From da0716440ce05f0dbc36f10b5ad9d7ae89d5b6ff Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 3 Nov 2023 11:59:42 +0100
Subject: [PATCH 004/205] chore: add import component diffs

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/ImportAttribute.js   |  2 ++
 .../js/components/import/common/Diffs.js      | 23 +++++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 rdmo/management/assets/js/components/import/common/Diffs.js

diff --git a/rdmo/management/assets/js/components/import/ImportAttribute.js b/rdmo/management/assets/js/components/import/ImportAttribute.js
index a360f9c14f..3db4396ad0 100644
--- a/rdmo/management/assets/js/components/import/ImportAttribute.js
+++ b/rdmo/management/assets/js/components/import/ImportAttribute.js
@@ -7,6 +7,7 @@ import Errors from './common/Errors'
 import Fields from './common/Fields'
 import Form from './common/Form'
 import Warnings from './common/Warnings'
+import Diffs from './common/Diffs'
 
 import { codeClass } from '../../constants/elements'
 
@@ -35,6 +36,7 @@ const ImportAttribute = ({ config, attribute, importActions }) => {
           <Fields element={attribute} />
           <Warnings element={attribute} />
           <Errors element={attribute} />
+          <Diffs element={attribute} />
         </>
       }
     </li>
diff --git a/rdmo/management/assets/js/components/import/common/Diffs.js b/rdmo/management/assets/js/components/import/common/Diffs.js
new file mode 100644
index 0000000000..07fa75f52b
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/common/Diffs.js
@@ -0,0 +1,23 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+import isEmpty from 'lodash/isEmpty'
+
+
+const Diffs = ({ element }) => {
+  return !isEmpty(element.difftables) && <div className="row text-danger mt-10">
+    <div className="col-sm-3 text-right">
+      {gettext('Diffs')}
+    </div>
+    <div className="col-sm-9">
+    {
+      element.difftables
+    }
+    </div>
+  </div>
+}
+
+Diffs.propTypes = {
+  element: PropTypes.object.isRequired
+}
+
+export default Diffs

From 1bf1c16045566e59a4be936ed0182cda31c36782 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 8 Nov 2023 16:06:20 +0100
Subject: [PATCH 005/205] build: add react-diff-viewer-continued to
 package.json

---
 package-lock.json | 296 ++++++++++++++++++++++++++++++----------------
 package.json      |   3 +-
 2 files changed, 194 insertions(+), 105 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 510d52f058..861f0ce3a9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -21,6 +21,7 @@
         "prop-types": "^15.7.2",
         "react": "^18.2.0",
         "react-bootstrap": "0.33.1",
+        "react-diff-viewer-continued": "^3.3.1",
         "react-dnd": "^16.0.1",
         "react-dnd-html5-backend": "^16.0.1",
         "react-dom": "^18.2.0",
@@ -62,6 +63,7 @@
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz",
       "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==",
+      "devOptional": true,
       "dependencies": {
         "@jridgewell/gen-mapping": "^0.1.0",
         "@jridgewell/trace-mapping": "^0.3.9"
@@ -118,9 +120,10 @@
       }
     },
     "node_modules/@babel/compat-data": {
-      "version": "7.23.3",
-      "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.23.3.tgz",
-      "integrity": "sha512-BmR4bWbDIoFJmJ9z2cZ8Gmm2MXgEDgjdWgpKmKWUt54UGFJdlj31ECtbaDvCG/qVdG3AQ1SfpZEs01lUFbzLOQ==",
+      "version": "7.17.10",
+      "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.17.10.tgz",
+      "integrity": "sha512-GZt/TCsG70Ms19gfZO1tM4CVnXsPgEPBCpJu+Qz3L0LUDsY5nZqFZglIoPC1kIYOtNBZlrnFT+klg12vFGZXrw==",
+      "devOptional": true,
       "engines": {
         "node": ">=6.9.0"
       }
@@ -2016,25 +2019,21 @@
       }
     },
     "node_modules/@emotion/babel-plugin": {
-      "version": "11.10.5",
-      "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.10.5.tgz",
-      "integrity": "sha512-xE7/hyLHJac7D2Ve9dKroBBZqBT7WuPQmWcq7HSGb84sUuP4mlOWoB8dvVfD9yk5DHkU1m6RW7xSoDtnQHNQeA==",
+      "version": "11.11.0",
+      "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz",
+      "integrity": "sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ==",
       "dependencies": {
         "@babel/helper-module-imports": "^7.16.7",
-        "@babel/plugin-syntax-jsx": "^7.17.12",
         "@babel/runtime": "^7.18.3",
-        "@emotion/hash": "^0.9.0",
-        "@emotion/memoize": "^0.8.0",
-        "@emotion/serialize": "^1.1.1",
+        "@emotion/hash": "^0.9.1",
+        "@emotion/memoize": "^0.8.1",
+        "@emotion/serialize": "^1.1.2",
         "babel-plugin-macros": "^3.1.0",
         "convert-source-map": "^1.5.0",
         "escape-string-regexp": "^4.0.0",
         "find-root": "^1.1.0",
         "source-map": "^0.5.7",
-        "stylis": "4.1.3"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0"
+        "stylis": "4.2.0"
       }
     },
     "node_modules/@emotion/babel-plugin/node_modules/escape-string-regexp": {
@@ -2057,26 +2056,38 @@
       }
     },
     "node_modules/@emotion/cache": {
-      "version": "11.10.5",
-      "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.10.5.tgz",
-      "integrity": "sha512-dGYHWyzTdmK+f2+EnIGBpkz1lKc4Zbj2KHd4cX3Wi8/OWr5pKslNjc3yABKH4adRGCvSX4VDC0i04mrrq0aiRA==",
+      "version": "11.11.0",
+      "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.11.0.tgz",
+      "integrity": "sha512-P34z9ssTCBi3e9EI1ZsWpNHcfY1r09ZO0rZbRO2ob3ZQMnFI35jB536qoXbkdesr5EUhYi22anuEJuyxifaqAQ==",
       "dependencies": {
-        "@emotion/memoize": "^0.8.0",
-        "@emotion/sheet": "^1.2.1",
-        "@emotion/utils": "^1.2.0",
-        "@emotion/weak-memoize": "^0.3.0",
-        "stylis": "4.1.3"
+        "@emotion/memoize": "^0.8.1",
+        "@emotion/sheet": "^1.2.2",
+        "@emotion/utils": "^1.2.1",
+        "@emotion/weak-memoize": "^0.3.1",
+        "stylis": "4.2.0"
+      }
+    },
+    "node_modules/@emotion/css": {
+      "version": "11.11.2",
+      "resolved": "https://registry.npmjs.org/@emotion/css/-/css-11.11.2.tgz",
+      "integrity": "sha512-VJxe1ucoMYMS7DkiMdC2T7PWNbrEI0a39YRiyDvK2qq4lXwjRbVP/z4lpG+odCsRzadlR+1ywwrTzhdm5HNdew==",
+      "dependencies": {
+        "@emotion/babel-plugin": "^11.11.0",
+        "@emotion/cache": "^11.11.0",
+        "@emotion/serialize": "^1.1.2",
+        "@emotion/sheet": "^1.2.2",
+        "@emotion/utils": "^1.2.1"
       }
     },
     "node_modules/@emotion/hash": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.0.tgz",
-      "integrity": "sha512-14FtKiHhy2QoPIzdTcvh//8OyBlknNs2nXRwIhG904opCby3l+9Xaf/wuPvICBF0rc1ZCNBd3nKe9cd2mecVkQ=="
+      "version": "0.9.1",
+      "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.1.tgz",
+      "integrity": "sha512-gJB6HLm5rYwSLI6PQa+X1t5CFGrv1J1TWG+sOyMCeKz2ojaj6Fnl/rZEspogG+cvqbt4AE/2eIyD2QfLKTBNlQ=="
     },
     "node_modules/@emotion/memoize": {
-      "version": "0.8.0",
-      "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.8.0.tgz",
-      "integrity": "sha512-G/YwXTkv7Den9mXDO7AhLWkE3q+I92B+VqAE+dYG4NGPaHZGvt3G8Q0p9vmE+sq7rTGphUbAvmQ9YpbfMQGGlA=="
+      "version": "0.8.1",
+      "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.8.1.tgz",
+      "integrity": "sha512-W2P2c/VRW1/1tLox0mVUalvnWXxavmv/Oum2aPsRcoDJuob75FC3Y8FbpfLwUegRcxINtGUMPq0tFCvYNTBXNA=="
     },
     "node_modules/@emotion/react": {
       "version": "11.10.5",
@@ -2106,26 +2117,26 @@
       }
     },
     "node_modules/@emotion/serialize": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.1.1.tgz",
-      "integrity": "sha512-Zl/0LFggN7+L1liljxXdsVSVlg6E/Z/olVWpfxUTxOAmi8NU7YoeWeLfi1RmnB2TATHoaWwIBRoL+FvAJiTUQA==",
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.1.2.tgz",
+      "integrity": "sha512-zR6a/fkFP4EAcCMQtLOhIgpprZOwNmCldtpaISpvz348+DP4Mz8ZoKaGGCQpbzepNIUWbq4w6hNZkwDyKoS+HA==",
       "dependencies": {
-        "@emotion/hash": "^0.9.0",
-        "@emotion/memoize": "^0.8.0",
-        "@emotion/unitless": "^0.8.0",
-        "@emotion/utils": "^1.2.0",
+        "@emotion/hash": "^0.9.1",
+        "@emotion/memoize": "^0.8.1",
+        "@emotion/unitless": "^0.8.1",
+        "@emotion/utils": "^1.2.1",
         "csstype": "^3.0.2"
       }
     },
     "node_modules/@emotion/sheet": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.2.1.tgz",
-      "integrity": "sha512-zxRBwl93sHMsOj4zs+OslQKg/uhF38MB+OMKoCrVuS0nyTkqnau+BM3WGEoOptg9Oz45T/aIGs1qbVAsEFo3nA=="
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.2.2.tgz",
+      "integrity": "sha512-0QBtGvaqtWi+nx6doRwDdBIzhNdZrXUppvTM4dtZZWEGTXL/XE/yJxLMGlDT1Gt+UHH5IX1n+jkXyytE/av7OA=="
     },
     "node_modules/@emotion/unitless": {
-      "version": "0.8.0",
-      "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.8.0.tgz",
-      "integrity": "sha512-VINS5vEYAscRl2ZUDiT3uMPlrFQupiKgHz5AA4bCH1miKBg4qtwkim1qPmJj/4WG6TreYMY111rEFsjupcOKHw=="
+      "version": "0.8.1",
+      "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.8.1.tgz",
+      "integrity": "sha512-KOEGMu6dmJZtpadb476IsZBclKvILjopjUii3V+7MnXIQCYh8W3NgNcgwo21n9LXZX6EDIKvqfjYxXebDwxKmQ=="
     },
     "node_modules/@emotion/use-insertion-effect-with-fallbacks": {
       "version": "1.0.0",
@@ -2136,14 +2147,14 @@
       }
     },
     "node_modules/@emotion/utils": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.2.0.tgz",
-      "integrity": "sha512-sn3WH53Kzpw8oQ5mgMmIzzyAaH2ZqFEbozVVBSYp538E06OSE6ytOp7pRAjNQR+Q/orwqdQYJSe2m3hCOeznkw=="
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.2.1.tgz",
+      "integrity": "sha512-Y2tGf3I+XVnajdItskUCn6LX+VUDmP6lTL4fcqsXAv43dnlbZiuW4MWQW38rW/BVWSE7Q/7+XQocmpnRYILUmg=="
     },
     "node_modules/@emotion/weak-memoize": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.3.0.tgz",
-      "integrity": "sha512-AHPmaAx+RYfZz0eYu6Gviiagpmiyw98ySSlQvCUhVGDRtDFe4DBS0x1bSjdF3gqUDYOczB+yYvBTtEylYSdRhg=="
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz",
+      "integrity": "sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww=="
     },
     "node_modules/@eslint-community/eslint-utils": {
       "version": "4.4.0",
@@ -2266,6 +2277,7 @@
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz",
       "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==",
+      "devOptional": true,
       "dependencies": {
         "@jridgewell/set-array": "^1.0.0",
         "@jridgewell/sourcemap-codec": "^1.4.10"
@@ -2286,6 +2298,7 @@
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.0.tgz",
       "integrity": "sha512-SfJxIxNVYLTsKwzB3MoOQ1yxf4w/E6MdkvTgrgAt1bfxjSrLUoHMKrDOykwN14q65waezZIdqDneUIPh4/sKxg==",
+      "devOptional": true,
       "engines": {
         "node": ">=6.0.0"
       }
@@ -3517,6 +3530,7 @@
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
       "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "devOptional": true,
       "dependencies": {
         "ms": "2.1.2"
       },
@@ -3571,6 +3585,14 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/diff": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz",
+      "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==",
+      "engines": {
+        "node": ">=0.3.1"
+      }
+    },
     "node_modules/dir-glob": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
@@ -3785,6 +3807,7 @@
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
       "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
+      "devOptional": true,
       "engines": {
         "node": ">=6"
       }
@@ -4577,6 +4600,7 @@
       "version": "1.0.0-beta.2",
       "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
       "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==",
+      "devOptional": true,
       "engines": {
         "node": ">=6.9.0"
       }
@@ -4654,6 +4678,7 @@
       "version": "11.12.0",
       "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
       "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
+      "devOptional": true,
       "engines": {
         "node": ">=4"
       }
@@ -5397,6 +5422,7 @@
       "version": "2.5.2",
       "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",
       "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==",
+      "devOptional": true,
       "bin": {
         "jsesc": "bin/jsesc"
       },
@@ -5425,6 +5451,7 @@
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
       "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
+      "devOptional": true,
       "bin": {
         "json5": "lib/cli.js"
       },
@@ -5701,7 +5728,8 @@
     "node_modules/ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+      "devOptional": true
     },
     "node_modules/nanoid": {
       "version": "3.3.4",
@@ -5979,7 +6007,8 @@
     "node_modules/picocolors": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz",
-      "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ=="
+      "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==",
+      "devOptional": true
     },
     "node_modules/picomatch": {
       "version": "2.3.1",
@@ -6237,6 +6266,25 @@
         "loose-envify": "^1.0.0"
       }
     },
+    "node_modules/react-diff-viewer-continued": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/react-diff-viewer-continued/-/react-diff-viewer-continued-3.3.1.tgz",
+      "integrity": "sha512-YhjWjCUq6cs8k9iErpWh/xB2jFCndigGAz2TKubdqrSTtDH5Ib+tdQgzBWVXMMqgtEwoPLi+WFmSsdSoYbDVpw==",
+      "dependencies": {
+        "@emotion/css": "^11.11.2",
+        "classnames": "^2.3.2",
+        "diff": "^5.1.0",
+        "memoize-one": "^6.0.0",
+        "prop-types": "^15.8.1"
+      },
+      "engines": {
+        "node": ">= 8"
+      },
+      "peerDependencies": {
+        "react": "^15.3.0 || ^16.0.0 || ^17.0.0 || ^18.0.0",
+        "react-dom": "^15.3.0 || ^16.0.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
     "node_modules/react-dnd": {
       "version": "16.0.1",
       "resolved": "https://registry.npmjs.org/react-dnd/-/react-dnd-16.0.1.tgz",
@@ -6830,6 +6878,7 @@
       "version": "6.3.1",
       "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
       "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+      "devOptional": true,
       "bin": {
         "semver": "bin/semver.js"
       }
@@ -7056,9 +7105,9 @@
       "integrity": "sha512-OPhtyEjyyN9x3nhPsu76f52yUGXiZcgvsrFVtvTkyGRQJ0XK+GPc6ov1z+lRpbeabka+MYEQxOYRnt5nF30aMw=="
     },
     "node_modules/stylis": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.1.3.tgz",
-      "integrity": "sha512-GP6WDNWf+o403jrEp9c5jibKavrtLW+/qYGhFxFrG8maXhwTBI7gLLhiBb0o7uFccWN+EOS9aMO6cGHWAO07OA=="
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.2.0.tgz",
+      "integrity": "sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw=="
     },
     "node_modules/supports-color": {
       "version": "5.5.0",
@@ -7717,6 +7766,7 @@
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz",
       "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==",
+      "devOptional": true,
       "requires": {
         "@jridgewell/gen-mapping": "^0.1.0",
         "@jridgewell/trace-mapping": "^0.3.9"
@@ -9097,22 +9147,21 @@
       "dev": true
     },
     "@emotion/babel-plugin": {
-      "version": "11.10.5",
-      "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.10.5.tgz",
-      "integrity": "sha512-xE7/hyLHJac7D2Ve9dKroBBZqBT7WuPQmWcq7HSGb84sUuP4mlOWoB8dvVfD9yk5DHkU1m6RW7xSoDtnQHNQeA==",
+      "version": "11.11.0",
+      "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz",
+      "integrity": "sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ==",
       "requires": {
         "@babel/helper-module-imports": "^7.16.7",
-        "@babel/plugin-syntax-jsx": "^7.17.12",
         "@babel/runtime": "^7.18.3",
-        "@emotion/hash": "^0.9.0",
-        "@emotion/memoize": "^0.8.0",
-        "@emotion/serialize": "^1.1.1",
+        "@emotion/hash": "^0.9.1",
+        "@emotion/memoize": "^0.8.1",
+        "@emotion/serialize": "^1.1.2",
         "babel-plugin-macros": "^3.1.0",
         "convert-source-map": "^1.5.0",
         "escape-string-regexp": "^4.0.0",
         "find-root": "^1.1.0",
         "source-map": "^0.5.7",
-        "stylis": "4.1.3"
+        "stylis": "4.2.0"
       },
       "dependencies": {
         "escape-string-regexp": {
@@ -9128,26 +9177,38 @@
       }
     },
     "@emotion/cache": {
-      "version": "11.10.5",
-      "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.10.5.tgz",
-      "integrity": "sha512-dGYHWyzTdmK+f2+EnIGBpkz1lKc4Zbj2KHd4cX3Wi8/OWr5pKslNjc3yABKH4adRGCvSX4VDC0i04mrrq0aiRA==",
+      "version": "11.11.0",
+      "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.11.0.tgz",
+      "integrity": "sha512-P34z9ssTCBi3e9EI1ZsWpNHcfY1r09ZO0rZbRO2ob3ZQMnFI35jB536qoXbkdesr5EUhYi22anuEJuyxifaqAQ==",
       "requires": {
-        "@emotion/memoize": "^0.8.0",
-        "@emotion/sheet": "^1.2.1",
-        "@emotion/utils": "^1.2.0",
-        "@emotion/weak-memoize": "^0.3.0",
-        "stylis": "4.1.3"
+        "@emotion/memoize": "^0.8.1",
+        "@emotion/sheet": "^1.2.2",
+        "@emotion/utils": "^1.2.1",
+        "@emotion/weak-memoize": "^0.3.1",
+        "stylis": "4.2.0"
+      }
+    },
+    "@emotion/css": {
+      "version": "11.11.2",
+      "resolved": "https://registry.npmjs.org/@emotion/css/-/css-11.11.2.tgz",
+      "integrity": "sha512-VJxe1ucoMYMS7DkiMdC2T7PWNbrEI0a39YRiyDvK2qq4lXwjRbVP/z4lpG+odCsRzadlR+1ywwrTzhdm5HNdew==",
+      "requires": {
+        "@emotion/babel-plugin": "^11.11.0",
+        "@emotion/cache": "^11.11.0",
+        "@emotion/serialize": "^1.1.2",
+        "@emotion/sheet": "^1.2.2",
+        "@emotion/utils": "^1.2.1"
       }
     },
     "@emotion/hash": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.0.tgz",
-      "integrity": "sha512-14FtKiHhy2QoPIzdTcvh//8OyBlknNs2nXRwIhG904opCby3l+9Xaf/wuPvICBF0rc1ZCNBd3nKe9cd2mecVkQ=="
+      "version": "0.9.1",
+      "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.1.tgz",
+      "integrity": "sha512-gJB6HLm5rYwSLI6PQa+X1t5CFGrv1J1TWG+sOyMCeKz2ojaj6Fnl/rZEspogG+cvqbt4AE/2eIyD2QfLKTBNlQ=="
     },
     "@emotion/memoize": {
-      "version": "0.8.0",
-      "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.8.0.tgz",
-      "integrity": "sha512-G/YwXTkv7Den9mXDO7AhLWkE3q+I92B+VqAE+dYG4NGPaHZGvt3G8Q0p9vmE+sq7rTGphUbAvmQ9YpbfMQGGlA=="
+      "version": "0.8.1",
+      "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.8.1.tgz",
+      "integrity": "sha512-W2P2c/VRW1/1tLox0mVUalvnWXxavmv/Oum2aPsRcoDJuob75FC3Y8FbpfLwUegRcxINtGUMPq0tFCvYNTBXNA=="
     },
     "@emotion/react": {
       "version": "11.10.5",
@@ -9165,26 +9226,26 @@
       }
     },
     "@emotion/serialize": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.1.1.tgz",
-      "integrity": "sha512-Zl/0LFggN7+L1liljxXdsVSVlg6E/Z/olVWpfxUTxOAmi8NU7YoeWeLfi1RmnB2TATHoaWwIBRoL+FvAJiTUQA==",
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.1.2.tgz",
+      "integrity": "sha512-zR6a/fkFP4EAcCMQtLOhIgpprZOwNmCldtpaISpvz348+DP4Mz8ZoKaGGCQpbzepNIUWbq4w6hNZkwDyKoS+HA==",
       "requires": {
-        "@emotion/hash": "^0.9.0",
-        "@emotion/memoize": "^0.8.0",
-        "@emotion/unitless": "^0.8.0",
-        "@emotion/utils": "^1.2.0",
+        "@emotion/hash": "^0.9.1",
+        "@emotion/memoize": "^0.8.1",
+        "@emotion/unitless": "^0.8.1",
+        "@emotion/utils": "^1.2.1",
         "csstype": "^3.0.2"
       }
     },
     "@emotion/sheet": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.2.1.tgz",
-      "integrity": "sha512-zxRBwl93sHMsOj4zs+OslQKg/uhF38MB+OMKoCrVuS0nyTkqnau+BM3WGEoOptg9Oz45T/aIGs1qbVAsEFo3nA=="
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.2.2.tgz",
+      "integrity": "sha512-0QBtGvaqtWi+nx6doRwDdBIzhNdZrXUppvTM4dtZZWEGTXL/XE/yJxLMGlDT1Gt+UHH5IX1n+jkXyytE/av7OA=="
     },
     "@emotion/unitless": {
-      "version": "0.8.0",
-      "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.8.0.tgz",
-      "integrity": "sha512-VINS5vEYAscRl2ZUDiT3uMPlrFQupiKgHz5AA4bCH1miKBg4qtwkim1qPmJj/4WG6TreYMY111rEFsjupcOKHw=="
+      "version": "0.8.1",
+      "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.8.1.tgz",
+      "integrity": "sha512-KOEGMu6dmJZtpadb476IsZBclKvILjopjUii3V+7MnXIQCYh8W3NgNcgwo21n9LXZX6EDIKvqfjYxXebDwxKmQ=="
     },
     "@emotion/use-insertion-effect-with-fallbacks": {
       "version": "1.0.0",
@@ -9193,14 +9254,14 @@
       "requires": {}
     },
     "@emotion/utils": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.2.0.tgz",
-      "integrity": "sha512-sn3WH53Kzpw8oQ5mgMmIzzyAaH2ZqFEbozVVBSYp538E06OSE6ytOp7pRAjNQR+Q/orwqdQYJSe2m3hCOeznkw=="
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.2.1.tgz",
+      "integrity": "sha512-Y2tGf3I+XVnajdItskUCn6LX+VUDmP6lTL4fcqsXAv43dnlbZiuW4MWQW38rW/BVWSE7Q/7+XQocmpnRYILUmg=="
     },
     "@emotion/weak-memoize": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.3.0.tgz",
-      "integrity": "sha512-AHPmaAx+RYfZz0eYu6Gviiagpmiyw98ySSlQvCUhVGDRtDFe4DBS0x1bSjdF3gqUDYOczB+yYvBTtEylYSdRhg=="
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz",
+      "integrity": "sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww=="
     },
     "@eslint-community/eslint-utils": {
       "version": "4.4.0",
@@ -9291,6 +9352,7 @@
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz",
       "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==",
+      "devOptional": true,
       "requires": {
         "@jridgewell/set-array": "^1.0.0",
         "@jridgewell/sourcemap-codec": "^1.4.10"
@@ -9304,7 +9366,8 @@
     "@jridgewell/set-array": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.0.tgz",
-      "integrity": "sha512-SfJxIxNVYLTsKwzB3MoOQ1yxf4w/E6MdkvTgrgAt1bfxjSrLUoHMKrDOykwN14q65waezZIdqDneUIPh4/sKxg=="
+      "integrity": "sha512-SfJxIxNVYLTsKwzB3MoOQ1yxf4w/E6MdkvTgrgAt1bfxjSrLUoHMKrDOykwN14q65waezZIdqDneUIPh4/sKxg==",
+      "devOptional": true
     },
     "@jridgewell/source-map": {
       "version": "0.3.5",
@@ -10278,6 +10341,7 @@
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
       "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "devOptional": true,
       "requires": {
         "ms": "2.1.2"
       }
@@ -10315,6 +10379,11 @@
         "object-keys": "^1.1.1"
       }
     },
+    "diff": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz",
+      "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw=="
+    },
     "dir-glob": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
@@ -10495,7 +10564,8 @@
     "escalade": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
-      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
+      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
+      "devOptional": true
     },
     "escape-string-regexp": {
       "version": "1.0.5",
@@ -11059,7 +11129,8 @@
     "gensync": {
       "version": "1.0.0-beta.2",
       "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
-      "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="
+      "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==",
+      "devOptional": true
     },
     "get-intrinsic": {
       "version": "1.2.2",
@@ -11115,7 +11186,8 @@
     "globals": {
       "version": "11.12.0",
       "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
-      "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA=="
+      "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
+      "devOptional": true
     },
     "globalthis": {
       "version": "1.0.3",
@@ -11662,7 +11734,8 @@
     "json5": {
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
-      "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="
+      "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
+      "devOptional": true
     },
     "jsx-ast-utils": {
       "version": "3.3.3",
@@ -11872,7 +11945,8 @@
     "ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+      "devOptional": true
     },
     "nanoid": {
       "version": "3.3.4",
@@ -12072,7 +12146,8 @@
     "picocolors": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz",
-      "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ=="
+      "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==",
+      "devOptional": true
     },
     "picomatch": {
       "version": "2.3.1",
@@ -12246,6 +12321,18 @@
         }
       }
     },
+    "react-diff-viewer-continued": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/react-diff-viewer-continued/-/react-diff-viewer-continued-3.3.1.tgz",
+      "integrity": "sha512-YhjWjCUq6cs8k9iErpWh/xB2jFCndigGAz2TKubdqrSTtDH5Ib+tdQgzBWVXMMqgtEwoPLi+WFmSsdSoYbDVpw==",
+      "requires": {
+        "@emotion/css": "^11.11.2",
+        "classnames": "^2.3.2",
+        "diff": "^5.1.0",
+        "memoize-one": "^6.0.0",
+        "prop-types": "^15.8.1"
+      }
+    },
     "react-dnd": {
       "version": "16.0.1",
       "resolved": "https://registry.npmjs.org/react-dnd/-/react-dnd-16.0.1.tgz",
@@ -12676,7 +12763,8 @@
     "semver": {
       "version": "6.3.1",
       "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="
+      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+      "devOptional": true
     },
     "serialize-javascript": {
       "version": "6.0.1",
@@ -12849,9 +12937,9 @@
       "integrity": "sha512-OPhtyEjyyN9x3nhPsu76f52yUGXiZcgvsrFVtvTkyGRQJ0XK+GPc6ov1z+lRpbeabka+MYEQxOYRnt5nF30aMw=="
     },
     "stylis": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.1.3.tgz",
-      "integrity": "sha512-GP6WDNWf+o403jrEp9c5jibKavrtLW+/qYGhFxFrG8maXhwTBI7gLLhiBb0o7uFccWN+EOS9aMO6cGHWAO07OA=="
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.2.0.tgz",
+      "integrity": "sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw=="
     },
     "supports-color": {
       "version": "5.5.0",
diff --git a/package.json b/package.json
index ad6ee8cdf9..aef7550df0 100644
--- a/package.json
+++ b/package.json
@@ -35,7 +35,8 @@
     "react-select": "^5.7.0",
     "redux": "^4.1.1",
     "redux-logger": "^3.0.6",
-    "redux-thunk": "^2.3.0"
+    "redux-thunk": "^2.3.0",
+    "react-diff-viewer-continued": "^3.3.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.23.4",

From cc47a04c1f2cfc6f80c6caa25fd4c6d0f9c04954 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 8 Nov 2023 16:13:27 +0100
Subject: [PATCH 006/205] feat: refactor diff import to attribute

---
 rdmo/core/imports.py   | 20 ++++----------------
 rdmo/domain/imports.py |  7 ++-----
 2 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index b71a675e6e..aaa640692f 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -1,4 +1,3 @@
-import difflib
 import logging
 import tempfile
 import time
@@ -264,32 +263,21 @@ def check_permissions(instance, element, user):
         element['errors'].append(message)
 
 
-def check_diff_instance(instance: dict, element: dict, check=False) -> dict[str, list[str]]:
+def check_diff_instance(element: dict, original_instance: dict, check=False) -> dict[str, list[str]]:
     if not check:
         return {}
-    overlapping_keys = set(element.keys()) & set(instance.keys())
+    overlapping_keys = set(element.keys()) & set(original_instance.keys())
 
     diffs = {}
     for key in overlapping_keys:
         element_val = element[key]
-        instance_val = instance[key]
+        instance_val = original_instance[key]
         if element_val is instance_val:
             continue
         if element_val == instance_val:
             continue
 
         if isinstance(element_val, str) and isinstance(instance_val, str):
-            diff = diff_str_str(element_val, instance_val)
-            if not diff:
-                continue
+            diffs[key] = {"old_value": element_val, "new_value": instance_val}
 
-            diffs[key] = diff
     return diffs
-
-
-def diff_str_str(value: str, other: str) -> list[str]:
-    """ checks the diff between two strings """
-    d = difflib.Differ()
-    diff = d.compare(value.splitlines(keepends=True), other.splitlines(keepends=True))
-    ret = list(diff)
-    return ret
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index fda00dc08a..e090c54f6f 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -44,11 +44,8 @@ def import_attribute(element, save=False, user=None):
 
     if element['updated']:
 
-        diffs = check_diff_instance(original_attribute, element, check=True)
-        if diffs:
-            # breakpoint()
-            diff_warning = "\n".join([f"{k}:{' '.join(val)}" for k, val in diffs.items()])
-            element['diffs'] = diff_warning
+        diffs = check_diff_instance(element, original_attribute, check=True)
+        element['diffs'] = diffs
 
     if save:
         attribute.save()

From 5fb58a86ba768ec961620d2436a655f1c6745fe2 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 8 Nov 2023 16:14:06 +0100
Subject: [PATCH 007/205] chore: revert ImportAttribute

---
 rdmo/management/assets/js/components/import/ImportAttribute.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportAttribute.js b/rdmo/management/assets/js/components/import/ImportAttribute.js
index 3db4396ad0..a360f9c14f 100644
--- a/rdmo/management/assets/js/components/import/ImportAttribute.js
+++ b/rdmo/management/assets/js/components/import/ImportAttribute.js
@@ -7,7 +7,6 @@ import Errors from './common/Errors'
 import Fields from './common/Fields'
 import Form from './common/Form'
 import Warnings from './common/Warnings'
-import Diffs from './common/Diffs'
 
 import { codeClass } from '../../constants/elements'
 
@@ -36,7 +35,6 @@ const ImportAttribute = ({ config, attribute, importActions }) => {
           <Fields element={attribute} />
           <Warnings element={attribute} />
           <Errors element={attribute} />
-          <Diffs element={attribute} />
         </>
       }
     </li>

From 8b8135e24a47f148c322f06360e43f7ed79d4cef Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 8 Nov 2023 16:14:45 +0100
Subject: [PATCH 008/205] feat: add diffs to Fields component

---
 .../js/components/import/common/Diffs.js      | 25 +++++++++++--------
 .../js/components/import/common/Fields.js     |  7 +++++-
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/Diffs.js b/rdmo/management/assets/js/components/import/common/Diffs.js
index 07fa75f52b..fb121c23c0 100644
--- a/rdmo/management/assets/js/components/import/common/Diffs.js
+++ b/rdmo/management/assets/js/components/import/common/Diffs.js
@@ -1,23 +1,26 @@
 import React from 'react'
 import PropTypes from 'prop-types'
 import isEmpty from 'lodash/isEmpty'
+import ReactDiffViewer from 'react-diff-viewer-continued'
 
 
-const Diffs = ({ element }) => {
-  return !isEmpty(element.difftables) && <div className="row text-danger mt-10">
-    <div className="col-sm-3 text-right">
-      {gettext('Diffs')}
+const Diffs = ({ element, field }) => {
+  return !isEmpty(element.diffs[field]) && <div className="col-sm-12">
+    <ReactDiffViewer
+        oldValue={element.diffs[field].old_value}
+        newValue={element.diffs[field].new_value}
+        splitView={true}
+        hideLineNumbers={true}
+        leftTitle={gettext('Current')}
+        rightTitle={gettext('Imported')}
+        >
+        </ReactDiffViewer>
     </div>
-    <div className="col-sm-9">
-    {
-      element.difftables
-    }
-    </div>
-  </div>
 }
 
 Diffs.propTypes = {
-  element: PropTypes.object.isRequired
+  element: PropTypes.object.isRequired,
+  field: PropTypes.string.isRequired,
 }
 
 export default Diffs
diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index da483eb9cb..f47c4d9b05 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -5,6 +5,7 @@ import isString from 'lodash/isString'
 import isUndefined from 'lodash/isUndefined'
 import truncate from 'lodash/truncate'
 import uniqueId from 'lodash/uniqueId'
+import Diffs from './Diffs'
 
 import { codeClass } from '../../../constants/elements'
 
@@ -21,7 +22,8 @@ const excludeKeys = [
   'uri_path',
   'uri_prefix',
   'valid',
-  'warnings'
+  'warnings',
+  'diffs'
 ]
 
 const Fields = ({ element }) => {
@@ -50,6 +52,9 @@ const Fields = ({ element }) => {
                     isString(value) && <span>{truncate(value, {length: 512})}</span>
                   }
                 </div>
+                {
+                    <Diffs element={element} field={key} />
+                  }
               </div>
             )
           }

From daa3fc331fe8025d8900a37d546a7766cbf20fba Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 9 Nov 2023 10:20:59 +0100
Subject: [PATCH 009/205] rename Diffs to FieldsDiffs

---
 .../js/components/import/common/{Diffs.js => FieldsDiffs.js}  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename rdmo/management/assets/js/components/import/common/{Diffs.js => FieldsDiffs.js} (85%)

diff --git a/rdmo/management/assets/js/components/import/common/Diffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
similarity index 85%
rename from rdmo/management/assets/js/components/import/common/Diffs.js
rename to rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index fb121c23c0..07ea596a29 100644
--- a/rdmo/management/assets/js/components/import/common/Diffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -7,8 +7,8 @@ import ReactDiffViewer from 'react-diff-viewer-continued'
 const Diffs = ({ element, field }) => {
   return !isEmpty(element.diffs[field]) && <div className="col-sm-12">
     <ReactDiffViewer
-        oldValue={element.diffs[field].old_value}
-        newValue={element.diffs[field].new_value}
+        oldValue={element.diffs[field].new_value}
+        newValue={element.diffs[field].old_value}
         splitView={true}
         hideLineNumbers={true}
         leftTitle={gettext('Current')}

From f1f3209351b70810d72d3ec2cfe0600390f55a12 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 9 Nov 2023 10:22:30 +0100
Subject: [PATCH 010/205] add info and filter to import.js

---
 .../assets/js/components/main/Import.js       | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 6f93e80fdc..20451fb697 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -2,6 +2,11 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import uniqueId from 'lodash/uniqueId'
 import isEmpty from 'lodash/isEmpty'
+import get from 'lodash/get'
+
+import { getUriPrefixes } from '../../utils/filter'
+import { FilterString, FilterUriPrefix } from '../common/Filter'
+import { Checkbox } from '../common/Checkboxes'
 
 import ImportAttribute from '../import/ImportAttribute'
 import ImportCatalog from '../import/ImportCatalog'
@@ -19,13 +24,40 @@ import { codeClass, verboseNames } from '../../constants/elements'
 
 const Import = ({ config, imports, importActions }) => {
   const { elements, success } = imports
+  const updateFilterString = (value) => importActions.updateConfig('filter.import.elements.search', value)
+  const updateFilterUriPrefix = (value) => importActions.updateConfig('filter.import.elements.uri_prefix', value)
+  const updateDisplayCatalogURI = (value) => importActions.updateConfig('display.uri.catalogs', value)
+  const updatedElements = elements.filter(element => element.updated)
 
   return (
     <div className="panel panel-default panel-import">
       <div className="panel-heading">
         <strong>{gettext('Import')}</strong>
+        <div className="pull-right">
+          {
+            updatedElements.length > 0 && <p>Updated {updatedElements.length} </p>
+          }
+        </div>
+      </div>
+    <div className="panel-body">
+    <div className="row">
+      <div className={'col-sm-8'}>
+        <FilterString value={get(config, 'filter.catalogs.search', '')} onChange={updateFilterString}
+                      placeholder={gettext('Filter uri')} />
+      </div>
+      <div className="col-sm-4">
+        <FilterUriPrefix value={get(config, 'filter.catalogs.uri_prefix', '')} onChange={updateFilterUriPrefix}
+                        options={getUriPrefixes(elements)} />
       </div>
 
+    </div>
+    <div className="checkboxes">
+      <span className="mr-10">{gettext('Show URIs:')}</span>
+      <Checkbox label={<code className="code-questions">{gettext('Catalogs')}</code>}
+                value={get(config, 'display.uri.catalogs', true)} onChange={updateDisplayCatalogURI} />
+    </div>
+    </div>
+
       <ul className="list-group">
       {
         elements.map((element, index) => {

From ac0f34a33fadedd035bb8dd674d5dd521f3287db Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 9 Nov 2023 10:23:54 +0100
Subject: [PATCH 011/205] add condition for Diffs div

---
 rdmo/management/assets/js/components/import/common/Fields.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index f47c4d9b05..6ebdc134db 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -5,10 +5,11 @@ import isString from 'lodash/isString'
 import isUndefined from 'lodash/isUndefined'
 import truncate from 'lodash/truncate'
 import uniqueId from 'lodash/uniqueId'
-import Diffs from './Diffs'
+import Diffs from './FieldsDiffs'
 
 import { codeClass } from '../../../constants/elements'
 
+
 const excludeKeys = [
   'created',
   'errors',
@@ -53,7 +54,7 @@ const Fields = ({ element }) => {
                   }
                 </div>
                 {
-                    <Diffs element={element} field={key} />
+                  element.updated && <Diffs element={element} field={key}/>
                   }
               </div>
             )

From e76570ab1b93f800c3c8948f49b784f77b4b26c8 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 9 Nov 2023 17:43:32 +0100
Subject: [PATCH 012/205] chore: refactor import methods, add
 get_or_return_instance function

---
 rdmo/conditions/imports.py |  30 ++++++-----
 rdmo/core/imports.py       |  23 ++++++++
 rdmo/domain/imports.py     |  26 ++++-----
 rdmo/options/imports.py    |  45 ++++++++--------
 rdmo/questions/imports.py  | 107 ++++++++++++++++++-------------------
 rdmo/tasks/imports.py      |  22 ++++----
 rdmo/views/imports.py      |  31 ++++++-----
 7 files changed, 155 insertions(+), 129 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index f9f5f513f1..31191be889 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -2,7 +2,14 @@
 
 from django.contrib.sites.models import Site
 
-from rdmo.core.imports import check_permissions, set_common_fields, set_foreign_field, validate_instance
+from rdmo.core.imports import (
+    check_permissions,
+    get_or_return_instance,
+    make_import_info_msg,
+    set_common_fields,
+    set_foreign_field,
+    validate_instance,
+)
 
 from .models import Condition
 from .validators import ConditionLockedValidator, ConditionUniqueURIValidator
@@ -11,10 +18,12 @@
 
 
 def import_condition(element, save=False, user=None):
-    try:
-        condition = Condition.objects.get(uri=element.get('uri'))
-    except Condition.DoesNotExist:
-        condition = Condition()
+
+    condition, _created = get_or_return_instance(Condition, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(condition._meta.verbose_name, _created, uri=element.get('uri'))
 
     set_common_fields(condition, element)
 
@@ -28,14 +37,11 @@ def import_condition(element, save=False, user=None):
 
     check_permissions(condition, element, user)
 
-    if save and not element.get('errors'):
-        if condition.id:
-            element['updated'] = True
-            logger.info('Condition %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.info('Condition created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return condition
 
+    if save:
+        logger.info(_msg)
         condition.save()
         condition.editors.add(Site.objects.get_current())
 
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index aaa640692f..6e88ffd143 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -4,8 +4,10 @@
 from os.path import join as pj
 from pathlib import Path
 from random import randint
+from typing import Optional, Tuple
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
+from django.db import models
 
 from rest_framework.utils import model_meta
 
@@ -14,6 +16,9 @@
 logger = logging.getLogger(__name__)
 
 
+IMPORT_INFO_MSG = 'Importing {model} {uri} from {filename}.'
+
+
 def handle_uploaded_file(filedata):
     tempfilename = generate_tempfile_name()
     with open(tempfilename, 'wb+') as destination:
@@ -40,6 +45,24 @@ def generate_tempfile_name():
     return fn
 
 
+def get_or_return_instance(model: models.Model, uri: Optional[str]=None) -> Tuple[models.Model, bool]:
+    if uri is None:
+        return model(), True
+    try:
+        return model.objects.get(uri=uri), False
+    except model.DoesNotExist:
+        return model(), True
+
+
+def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=None):
+    if uri is None:
+        return "%s, no uri", verbose_name
+    if created:
+        return "%s created with %s", verbose_name, uri
+
+    return "%s %s updated", verbose_name, uri
+
+
 def set_common_fields(instance, element):
     instance.uri_prefix = element.get('uri_prefix') or ''
     instance.uri_path = element.get('uri_path') or ''
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index e090c54f6f..5cc58ebefb 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -1,11 +1,11 @@
-import copy
 import logging
 
 from django.contrib.sites.models import Site
 
 from rdmo.core.imports import (
-    check_diff_instance,
     check_permissions,
+    get_or_return_instance,
+    make_import_info_msg,
     set_common_fields,
     set_foreign_field,
     validate_instance,
@@ -18,15 +18,12 @@
 
 
 def import_attribute(element, save=False, user=None):
-    try:
-        attribute = Attribute.objects.get(uri=element.get('uri'))
-        original_attribute = copy.deepcopy(vars(attribute))
-        element['updated'] = True
-        _msg = 'Attribute %s updated.', element.get('uri')
-    except Attribute.DoesNotExist:
-        attribute = Attribute()
-        element['created'] = True
-        _msg = 'Attribute created with uri %s.', element.get('uri')
+
+    attribute, _created = get_or_return_instance(Attribute, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(attribute._meta.verbose_name, _created, uri=element.get('uri'))
 
     set_common_fields(attribute, element)
 
@@ -42,14 +39,9 @@ def import_attribute(element, save=False, user=None):
     if element.get('errors'):
         return attribute
 
-    if element['updated']:
-
-        diffs = check_diff_instance(element, original_attribute, check=True)
-        element['diffs'] = diffs
-
     if save:
+        logger.debug(_msg)
         attribute.save()
         attribute.editors.add(Site.objects.get_current())
-        logger.debug(_msg)
 
     return attribute
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 9823b98944..515c37a294 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -4,6 +4,8 @@
 
 from rdmo.core.imports import (
     check_permissions,
+    get_or_return_instance,
+    make_import_info_msg,
     set_common_fields,
     set_lang_field,
     set_m2m_instances,
@@ -24,10 +26,12 @@
 
 
 def import_optionset(element, save=False, user=None):
-    try:
-        optionset = OptionSet.objects.get(uri=element.get('uri'))
-    except OptionSet.DoesNotExist:
-        optionset = OptionSet()
+
+    optionset, _created = get_or_return_instance(OptionSet, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(optionset._meta.verbose_name, _created, uri=element.get('uri'))
 
     set_common_fields(optionset, element)
 
@@ -38,14 +42,11 @@ def import_optionset(element, save=False, user=None):
 
     check_permissions(optionset, element, user)
 
-    if save and not element.get('errors'):
-        if optionset.id:
-            element['updated'] = True
-            logger.info('OptionSet %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.info('OptionSet created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return optionset
 
+    if save:
+        logger.info(_msg)
         optionset.save()
         set_m2m_instances(optionset, 'conditions', element)
         set_m2m_through_instances(optionset, 'options', element, 'optionset', 'option', 'optionset_options')
@@ -55,10 +56,13 @@ def import_optionset(element, save=False, user=None):
 
 
 def import_option(element, save=False, user=None):
-    try:
-        option = Option.objects.get(uri=element.get('uri'))
-    except Option.DoesNotExist:
-        option = Option()
+
+    option, _created = get_or_return_instance(Option, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(option._meta.verbose_name, _created, uri=element.get('uri'))
+
 
     set_common_fields(option, element)
 
@@ -72,14 +76,11 @@ def import_option(element, save=False, user=None):
 
     check_permissions(option, element, user)
 
-    if save and not element.get('errors'):
-        if option.id:
-            element['updated'] = True
-            logger.info('Option %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.info('Option created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return option
 
+    if save:
+        logger.info(_msg)
         option.save()
         set_reverse_m2m_through_instance(option, 'optionset', element, 'option', 'optionset', 'option_optionsets')
         option.editors.add(Site.objects.get_current())
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index b180584d27..8493f262e7 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -4,6 +4,8 @@
 
 from rdmo.core.imports import (
     check_permissions,
+    get_or_return_instance,
+    make_import_info_msg,
     set_common_fields,
     set_foreign_field,
     set_lang_field,
@@ -32,10 +34,12 @@
 
 
 def import_catalog(element, save=False, user=None):
-    try:
-        catalog = Catalog.objects.get(uri=element.get('uri'))
-    except Catalog.DoesNotExist:
-        catalog = Catalog()
+
+    catalog, _created = get_or_return_instance(Catalog, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(catalog._meta.verbose_name, _created, uri=element.get('uri'))
 
     set_common_fields(catalog, element)
 
@@ -50,14 +54,11 @@ def import_catalog(element, save=False, user=None):
 
     check_permissions(catalog, element, user)
 
-    if save and not element.get('errors'):
-        if catalog.id:
-            element['updated'] = True
-            logger.info('Catalog %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.info('Catalog created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return catalog
 
+    if save:
+        logger.debug(_msg)
         catalog.save()
         set_m2m_through_instances(catalog, 'sections', element, 'catalog', 'section', 'catalog_sections')
         catalog.sites.add(Site.objects.get_current())
@@ -67,10 +68,12 @@ def import_catalog(element, save=False, user=None):
 
 
 def import_section(element, save=False, user=None):
-    try:
-        section = Section.objects.get(uri=element.get('uri'))
-    except Section.DoesNotExist:
-        section = Section()
+
+    section, _created = get_or_return_instance(Section, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(section._meta.verbose_name, _created, uri=element.get('uri'))
 
     set_common_fields(section, element)
 
@@ -81,14 +84,11 @@ def import_section(element, save=False, user=None):
 
     check_permissions(section, element, user)
 
-    if save and not element.get('errors'):
-        if section.id:
-            element['updated'] = True
-            logger.info('Section %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.info('Section created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return section
 
+    if save:
+        logger.info(_msg)
         section.save()
         set_reverse_m2m_through_instance(section, 'catalog', element, 'section', 'catalog', 'section_catalogs')
         set_m2m_through_instances(section, 'pages', element, 'section', 'page', 'section_pages')
@@ -98,10 +98,12 @@ def import_section(element, save=False, user=None):
 
 
 def import_page(element, save=False, user=None):
-    try:
-        page = Page.objects.get(uri=element.get('uri'))
-    except Page.DoesNotExist:
-        page = Page()
+
+    page, _created = get_or_return_instance(Page, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(page._meta.verbose_name, _created, uri=element.get('uri'))
 
     set_common_fields(page, element)
     set_foreign_field(page, 'attribute', element)
@@ -117,14 +119,11 @@ def import_page(element, save=False, user=None):
 
     check_permissions(page, element, user)
 
-    if save and not element.get('errors'):
-        if page.id:
-            element['updated'] = True
-            logger.info('QuestionSet %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.info('QuestionSet created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return page
 
+    if save:
+        logger.info(_msg)
         page.save()
         set_m2m_instances(page, 'conditions', element)
         set_reverse_m2m_through_instance(page, 'section', element, 'page', 'section', 'page_sections')
@@ -136,10 +135,12 @@ def import_page(element, save=False, user=None):
 
 
 def import_questionset(element, save=False, user=None):
-    try:
-        questionset = QuestionSet.objects.get(uri=element.get('uri'))
-    except QuestionSet.DoesNotExist:
-        questionset = QuestionSet()
+
+    questionset, _created = get_or_return_instance(QuestionSet, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(questionset._meta.verbose_name, _created, uri=element.get('uri'))
 
     set_common_fields(questionset, element)
     set_foreign_field(questionset, 'attribute', element)
@@ -154,14 +155,11 @@ def import_questionset(element, save=False, user=None):
 
     check_permissions(questionset, element, user)
 
-    if save and not element.get('errors'):
-        if questionset.id:
-            element['updated'] = True
-            logger.info('QuestionSet %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.info('QuestionSet created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return questionset
 
+    if save:
+        logger.info(_msg)
         questionset.save()
         set_m2m_instances(questionset, 'conditions', element)
         set_reverse_m2m_through_instance(questionset, 'page', element, 'questionset', 'page', 'questionset_pages')
@@ -174,10 +172,12 @@ def import_questionset(element, save=False, user=None):
 
 
 def import_question(element, save=False, user=None):
-    try:
-        question = Question.objects.get(uri=element.get('uri'))
-    except Question.DoesNotExist:
-        question = Question()
+
+    question, _created = get_or_return_instance(Question, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(question._meta.verbose_name, _created, uri=element.get('uri'))
 
     set_common_fields(question, element)
     set_foreign_field(question, 'attribute', element)
@@ -210,14 +210,11 @@ def import_question(element, save=False, user=None):
 
     check_permissions(question, element, user)
 
-    if save and not element.get('errors'):
-        if question.id:
-            element['updated'] = True
-            logger.info('Question %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.info('Question created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return question
 
+    if save:
+        logger.info(_msg)
         question.save()
         set_reverse_m2m_through_instance(question, 'page', element, 'question', 'page', 'question_pages')
         set_reverse_m2m_through_instance(question, 'questionset', element, 'question', 'questionset', 'question_questionsets')  # noqa: E501
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 2f9ef7dca8..f292b859ac 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -4,6 +4,8 @@
 
 from rdmo.core.imports import (
     check_permissions,
+    get_or_return_instance,
+    make_import_info_msg,
     set_common_fields,
     set_foreign_field,
     set_lang_field,
@@ -18,10 +20,11 @@
 
 
 def import_task(element, save=False, user=None):
-    try:
-        task = Task.objects.get(uri=element.get('uri'))
-    except Task.DoesNotExist:
-        task = Task()
+    task, _created = get_or_return_instance(Task, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(task._meta.verbose_name, _created, uri=element.get('uri'))
 
     set_common_fields(task, element)
 
@@ -42,14 +45,11 @@ def import_task(element, save=False, user=None):
 
     check_permissions(task, element, user)
 
-    if save and not element.get('errors'):
-        if task.id:
-            element['updated'] = True
-            logger.info('Task %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.info('Task created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return task
 
+    if save:
+        logger.info(_msg)
         task.save()
         set_m2m_instances(task, 'catalogs', element)
         set_m2m_instances(task, 'conditions', element)
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index ce7a778cc2..07e16b2a44 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -2,7 +2,15 @@
 
 from django.contrib.sites.models import Site
 
-from rdmo.core.imports import check_permissions, set_common_fields, set_lang_field, set_m2m_instances, validate_instance
+from rdmo.core.imports import (
+    check_permissions,
+    get_or_return_instance,
+    make_import_info_msg,
+    set_common_fields,
+    set_lang_field,
+    set_m2m_instances,
+    validate_instance,
+)
 
 from .models import View
 from .validators import ViewLockedValidator, ViewUniqueURIValidator
@@ -11,10 +19,12 @@
 
 
 def import_view(element, save=False, user=None):
-    try:
-        view = View.objects.get(uri=element.get('uri'))
-    except View.DoesNotExist:
-        view = View()
+
+    view, _created = get_or_return_instance(View, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(view._meta.verbose_name, _created, uri=element.get('uri'))
 
     set_common_fields(view, element)
 
@@ -30,14 +40,11 @@ def import_view(element, save=False, user=None):
 
     check_permissions(view, element, user)
 
-    if save and not element.get('errors'):
-        if view.id:
-            element['updated'] = True
-            logger.info('View %s updated.', element.get('uri'))
-        else:
-            element['created'] = True
-            logger.info('View created with uri %s.', element.get('uri'))
+    if element.get('errors'):
+        return view
 
+    if save:
+        logger.info(_msg)
         view.save()
         set_m2m_instances(view, 'catalogs', element)
         view.sites.add(Site.objects.get_current())

From abad7b6a9fd3d31d850f932b2e15459dc677741a Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 10 Nov 2023 13:58:47 +0100
Subject: [PATCH 013/205] refactor imports: move methods partially to
 management

---
 rdmo/conditions/imports.py  |  45 +++----
 rdmo/core/imports.py        |  44 ++++---
 rdmo/domain/imports.py      |  35 ++---
 rdmo/management/imports.py  | 156 ++++++++++++++++++----
 rdmo/management/viewsets.py |   8 +-
 rdmo/options/imports.py     |  90 ++++++-------
 rdmo/questions/imports.py   | 253 ++++++++++++++++--------------------
 rdmo/tasks/imports.py       |  60 ++++-----
 rdmo/views/imports.py       |  51 ++++----
 9 files changed, 389 insertions(+), 353 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index 31191be889..74b3987b35 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -1,48 +1,41 @@
 import logging
+from typing import Callable, Tuple
 
 from django.contrib.sites.models import Site
+from django.db import models
 
 from rdmo.core.imports import (
     check_permissions,
-    get_or_return_instance,
-    make_import_info_msg,
-    set_common_fields,
     set_foreign_field,
     validate_instance,
 )
 
-from .models import Condition
-from .validators import ConditionLockedValidator, ConditionUniqueURIValidator
-
 logger = logging.getLogger(__name__)
 
 
-def import_condition(element, save=False, user=None):
-
-    condition, _created = get_or_return_instance(Condition, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
-
-    _msg = make_import_info_msg(condition._meta.verbose_name, _created, uri=element.get('uri'))
-
-    set_common_fields(condition, element)
+def import_condition(
+        instance: models.Model,
+        element: dict,
+        validators: Tuple[Callable],
+        save: bool = False,
+        user: models.Model = None
+    ):
 
-    set_foreign_field(condition, 'source', element)
-    set_foreign_field(condition, 'target_option', element)
+    set_foreign_field(instance, 'source', element)
+    set_foreign_field(instance, 'target_option', element)
 
-    condition.relation = element.get('relation')
-    condition.target_text = element.get('target_text') or ''
+    instance.relation = element.get('relation')
+    instance.target_text = element.get('target_text') or ''
 
-    validate_instance(condition, element, ConditionLockedValidator, ConditionUniqueURIValidator)
+    validate_instance(instance, element, *validators)
 
-    check_permissions(condition, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return condition
+        return instance
 
     if save:
-        logger.info(_msg)
-        condition.save()
-        condition.editors.add(Site.objects.get_current())
+        instance.save()
+        instance.editors.add(Site.objects.get_current())
 
-    return condition
+    return instance
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 6e88ffd143..06450552d5 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -8,6 +8,8 @@
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
+from django.forms.models import model_to_dict
+from django.utils.module_loading import import_string as django_import_string
 
 from rest_framework.utils import model_meta
 
@@ -70,6 +72,28 @@ def set_common_fields(instance, element):
     instance.comment = element.get('comment') or ''
 
 
+def common_import_methods(model_dotted_path: str,
+                              uri: Optional[str]=None,
+                              element: Optional[dict]=None):
+    model = django_import_string(model_dotted_path)
+    instance, _created = get_or_return_instance(model, uri=element.get('uri'))
+    element['created'] = _created
+    element['updated'] = not _created
+
+    _msg = make_import_info_msg(model._meta.verbose_name, _created, uri=element.get('uri'))
+
+     # TODO maybe move to another place
+    element['original'] = {}
+    if element.get("updated"):
+        original = model_to_dict(instance)
+        original.update(**{k : model_to_dict(val) for k, val in original.items() if isinstance(val, models.Model)})
+        element['original'] = original
+
+    set_common_fields(instance, element)
+    return instance, element, _msg
+
+
+
 def set_lang_field(instance, field_name, element):
     for lang_code, lang_string, lang_field in get_languages():
         field = element.get(f'{field_name}_{lang_code}')
@@ -284,23 +308,3 @@ def check_permissions(instance, element, user):
         )
         logger.info(message)
         element['errors'].append(message)
-
-
-def check_diff_instance(element: dict, original_instance: dict, check=False) -> dict[str, list[str]]:
-    if not check:
-        return {}
-    overlapping_keys = set(element.keys()) & set(original_instance.keys())
-
-    diffs = {}
-    for key in overlapping_keys:
-        element_val = element[key]
-        instance_val = original_instance[key]
-        if element_val is instance_val:
-            continue
-        if element_val == instance_val:
-            continue
-
-        if isinstance(element_val, str) and isinstance(instance_val, str):
-            diffs[key] = {"old_value": element_val, "new_value": instance_val}
-
-    return diffs
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 5cc58ebefb..be0ca7e829 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -1,47 +1,32 @@
 import logging
+from typing import Callable, Tuple
 
 from django.contrib.sites.models import Site
 
 from rdmo.core.imports import (
     check_permissions,
-    get_or_return_instance,
-    make_import_info_msg,
-    set_common_fields,
     set_foreign_field,
     validate_instance,
 )
 
-from .models import Attribute
-from .validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
-
 logger = logging.getLogger(__name__)
 
 
-def import_attribute(element, save=False, user=None):
-
-    attribute, _created = get_or_return_instance(Attribute, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
-
-    _msg = make_import_info_msg(attribute._meta.verbose_name, _created, uri=element.get('uri'))
-
-    set_common_fields(attribute, element)
+def import_attribute(instance, element, validators: Tuple[Callable], save=False, user=None):
 
-    set_foreign_field(attribute, 'parent', element)
+    set_foreign_field(instance, 'parent', element)
 
-    attribute.path = Attribute.build_path(attribute.key, attribute.parent)
+    instance.path = instance.build_path(instance.key, instance.parent)
 
-    validate_instance(attribute, element, AttributeLockedValidator,
-                      AttributeParentValidator, AttributeUniqueURIValidator)
+    validate_instance(instance, element, *validators)
 
-    check_permissions(attribute, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return attribute
+        return instance
 
     if save:
-        logger.debug(_msg)
-        attribute.save()
-        attribute.editors.add(Site.objects.get_current())
+        instance.save()
+        instance.editors.add(Site.objects.get_current())
 
-    return attribute
+    return instance
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index a9cce13967..1fa07d9790 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -1,42 +1,146 @@
+import logging
 from collections import defaultdict
+from typing import Optional
 
 from rdmo.conditions.imports import import_condition
+from rdmo.conditions.validators import ConditionLockedValidator, ConditionUniqueURIValidator
+from rdmo.core.imports import common_import_methods
 from rdmo.domain.imports import import_attribute
+from rdmo.domain.validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
 from rdmo.options.imports import import_option, import_optionset
+from rdmo.options.validators import (
+    OptionLockedValidator,
+    OptionSetLockedValidator,
+    OptionSetUniqueURIValidator,
+    OptionUniqueURIValidator,
+)
 from rdmo.questions.imports import import_catalog, import_page, import_question, import_questionset, import_section
+from rdmo.questions.validators import (
+    CatalogLockedValidator,
+    CatalogUniqueURIValidator,
+    PageLockedValidator,
+    PageUniqueURIValidator,
+    QuestionLockedValidator,
+    QuestionSetLockedValidator,
+    QuestionSetUniqueURIValidator,
+    QuestionUniqueURIValidator,
+    SectionLockedValidator,
+    SectionUniqueURIValidator,
+)
 from rdmo.tasks.imports import import_task
+from rdmo.tasks.validators import TaskLockedValidator, TaskUniqueURIValidator
 from rdmo.views.imports import import_view
+from rdmo.views.validators import ViewLockedValidator, ViewUniqueURIValidator
 
-ELEMENT_IMPORT_METHODS = {
-    "conditions.condition": import_condition,
-    "domain.attribute": import_attribute,
-    "options.optionset": import_optionset,
-    "options.option": import_option,
-    "questions.catalog": import_catalog,
-    "questions.section": import_section,
-    "questions.page": import_page,
-    "questions.questionset": import_questionset,
-    "questions.question": import_question,
-    "tasks.task": import_task,
-    "views.view": import_view,
+logger = logging.getLogger(__name__)
+
+ELEMENT_MODEL_IMPORT_MAPPER = {
+    "conditions.condition": {
+        "dotted_path": 'rdmo.conditions.models.Condition',
+        "import_method": import_condition,
+        "validators": (ConditionLockedValidator, ConditionUniqueURIValidator)
+    },
+    "domain.attribute": {
+        "dotted_path": 'rdmo.domain.models.Attribute',
+        "import_method": import_attribute,
+        "validators": (AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator)
+    },
+    "options.optionset": {
+        "dotted_path": "rdmo.options.models.OptionSet",
+        "import_method": import_optionset,
+        "validators": (OptionSetLockedValidator, OptionSetUniqueURIValidator),
+    },
+    "options.option": {
+        "dotted_path": "rdmo.options.models.Option",
+        "import_method": import_option,
+        "validators": (OptionLockedValidator, OptionUniqueURIValidator),
+    },
+    "questions.catalog": {
+        "dotted_path": "rdmo.questions.models.catalog.Catalog",
+        "import_method": import_catalog,
+        "validators": (CatalogLockedValidator, CatalogUniqueURIValidator),
+    },
+    "questions.section": {
+        "dotted_path": "rdmo.questions.models.section.Section",
+        "import_method": import_section,
+        "validators": (SectionLockedValidator, SectionUniqueURIValidator),
+    },
+    "questions.page": {
+        "dotted_path": "rdmo.questions.models.page.Page",
+        "import_method": import_page,
+        "validators":  (PageLockedValidator, PageUniqueURIValidator),
+    },
+    "questions.questionset": {
+        "dotted_path": "rdmo.questions.models.questionset.Questionset",
+        "import_method": import_questionset,
+        "validators": (QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
+    },
+    "questions.question": {
+        "dotted_path": "rdmo.questions.models.question.Question",
+        "import_method": import_question,
+        "validators": (QuestionLockedValidator, QuestionUniqueURIValidator),
+    },
+    "tasks.task": {
+        "dotted_path": "rdmo.tasks.models.task.Task",
+        "import_method": import_task,
+        "validators": (TaskLockedValidator, TaskUniqueURIValidator),
+    },
+    "views.view": {
+        "dotted_path": "rdmo.views.models.view.View",
+        "import_method": import_view,
+        "validators": (ViewLockedValidator, ViewUniqueURIValidator),
+    },
 }
 
 
-def import_elements(elements, save=True, user=None):
-    for element in elements:
+def import_elements(uploaded_elements, save=True, user=None):
+    imported_elements = []
+    for element in uploaded_elements:
         model = element.get('model')
+        if model is None:
+            continue
+        element = import_element(model_path=model, element=element, save=save, user=user)
+        element = filter_warnings(element, uploaded_elements)
+        imported_elements.append(element)
+    return imported_elements
+
+
+def import_element(
+        model_path: Optional[str] = None,
+        element: Optional[dict] = None,
+        save: bool = True,
+        user = None):
+
+    if element is None:
+        return element
+
+    element.update({
+        'warnings': defaultdict(list),
+        'errors': [],
+        'created': False,
+        'updated': False,
+        'original': defaultdict()
+    })
 
-        element.update({
-            'warnings': defaultdict(list),
-            'errors': [],
-            'created': False,
-            'updated': False
-        })
+    model_import = ELEMENT_MODEL_IMPORT_MAPPER[model_path]
+    import_method = model_import['import_method']
+    model_path = model_import['dotted_path']
+    validators = model_import['validators']
 
-        import_method = ELEMENT_IMPORT_METHODS[model]
-        import_method(element, save, user)
+    instance, element, _msg = common_import_methods(
+                            model_path,
+                            uri=element.get('uri'),
+                            element=element
+                    )
 
-        element = filter_warnings(element, elements)
+    instance = import_method(instance, element, validators, save, user)
+    if save and not element.get('errors'):
+        logger.info(_msg)
+
+    # TODO need to filter or serialize the same keys as in the element
+    element = filter_original(element)
+
+    return element
 
 
 def filter_warnings(element, elements):
@@ -48,3 +152,9 @@ def filter_warnings(element, elements):
 
     element['warnings'] = warnings
     return element
+
+def filter_original(element):
+    element['original'] = {k: val for k, val in
+                        element.get('original', {}).items()
+                        if k in element}
+    return element
diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index 08743a5de9..87eaef7233 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -86,13 +86,13 @@ def create(self, request, *args, **kwargs):
         elements = order_elements(elements)
 
         # step 6: convert elements to a list
-        elements = elements.values()
+        elements = list(elements.values())
 
         # step 8: import the elements if save=True is set
-        import_elements(elements, save=is_truthy(request.POST.get('import')), user=request.user)
+        imported_elements = import_elements(elements, save=is_truthy(request.POST.get('import')), user=request.user)
 
-        # step 9: return the list of elements
-        return Response(elements)
+        # step 9: return the list of, json-serializable, elements
+        return Response(imported_elements)
 
 
 class ImportViewSet(viewsets.ViewSet):
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 515c37a294..4437623bf4 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -1,12 +1,11 @@
 import logging
+from typing import Callable, Tuple
 
 from django.contrib.sites.models import Site
+from django.db import models
 
 from rdmo.core.imports import (
     check_permissions,
-    get_or_return_instance,
-    make_import_info_msg,
-    set_common_fields,
     set_lang_field,
     set_m2m_instances,
     set_m2m_through_instances,
@@ -14,75 +13,60 @@
     validate_instance,
 )
 
-from .models import Option, OptionSet
-from .validators import (
-    OptionLockedValidator,
-    OptionSetLockedValidator,
-    OptionSetUniqueURIValidator,
-    OptionUniqueURIValidator,
-)
-
 logger = logging.getLogger(__name__)
 
 
-def import_optionset(element, save=False, user=None):
-
-    optionset, _created = get_or_return_instance(OptionSet, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
-
-    _msg = make_import_info_msg(optionset._meta.verbose_name, _created, uri=element.get('uri'))
-
-    set_common_fields(optionset, element)
+def import_option(
+        instance: models.Model,
+        element: dict,
+        validators: Tuple[Callable],
+        save: bool = False,
+        user: models.Model = None
+    ):
 
-    optionset.order = element.get('order') or 0
-    optionset.provider_key = element.get('provider_key') or ''
+    instance.order = element.get('order') or 0
+    instance.provider_key = element.get('provider_key') or ''
 
-    validate_instance(optionset, element, OptionSetLockedValidator, OptionSetUniqueURIValidator)
+    validate_instance(instance, element, *validators)
 
-    check_permissions(optionset, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return optionset
+        return instance
 
     if save:
-        logger.info(_msg)
-        optionset.save()
-        set_m2m_instances(optionset, 'conditions', element)
-        set_m2m_through_instances(optionset, 'options', element, 'optionset', 'option', 'optionset_options')
-        optionset.editors.add(Site.objects.get_current())
-
-    return optionset
-
-
-def import_option(element, save=False, user=None):
-
-    option, _created = get_or_return_instance(Option, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
+        instance.save()
+        set_m2m_instances(instance, 'conditions', element)
+        set_m2m_through_instances(instance, 'options', element, 'optionset', 'option', 'optionset_options')
+        instance.editors.add(Site.objects.get_current())
 
-    _msg = make_import_info_msg(option._meta.verbose_name, _created, uri=element.get('uri'))
+    return instance
 
 
-    set_common_fields(option, element)
+def import_optionset(
+        instance: models.Model,
+        element: dict,
+        validators: Tuple[Callable],
+        save: bool = False,
+        user: models.Model = None
+    ):
 
-    option.additional_input = element.get('additional_input') or ''
+    instance.additional_input = element.get('additional_input') or ""
 
-    set_lang_field(option, 'text', element)
-    set_lang_field(option, 'help', element)
-    set_lang_field(option, 'view_text', element)
+    set_lang_field(instance, 'text', element)
+    set_lang_field(instance, 'help', element)
+    set_lang_field(instance, 'view_text', element)
 
-    validate_instance(option, element, OptionLockedValidator, OptionUniqueURIValidator)
+    validate_instance(instance, element, *validators)
 
-    check_permissions(option, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return option
+        return instance
 
     if save:
-        logger.info(_msg)
-        option.save()
-        set_reverse_m2m_through_instance(option, 'optionset', element, 'option', 'optionset', 'option_optionsets')
-        option.editors.add(Site.objects.get_current())
+        instance.save()
+        set_reverse_m2m_through_instance(instance, 'optionset', element, 'option', 'optionset', 'option_optionsets')
+        instance.editors.add(Site.objects.get_current())
 
-    return option
+    return instance
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 8493f262e7..dd84a4a86d 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -1,12 +1,11 @@
 import logging
+from typing import Callable, Tuple
 
 from django.contrib.sites.models import Site
+from django.db import models
 
 from rdmo.core.imports import (
     check_permissions,
-    get_or_return_instance,
-    make_import_info_msg,
-    set_common_fields,
     set_foreign_field,
     set_lang_field,
     set_m2m_instances,
@@ -15,211 +14,185 @@
     validate_instance,
 )
 
-from .models import Catalog, Page, Question, QuestionSet, Section
 from .utils import get_widget_types
-from .validators import (
-    CatalogLockedValidator,
-    CatalogUniqueURIValidator,
-    PageLockedValidator,
-    PageUniqueURIValidator,
-    QuestionLockedValidator,
-    QuestionSetLockedValidator,
-    QuestionSetUniqueURIValidator,
-    QuestionUniqueURIValidator,
-    SectionLockedValidator,
-    SectionUniqueURIValidator,
-)
 
 logger = logging.getLogger(__name__)
 
 
-def import_catalog(element, save=False, user=None):
-
-    catalog, _created = get_or_return_instance(Catalog, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
-
-    _msg = make_import_info_msg(catalog._meta.verbose_name, _created, uri=element.get('uri'))
-
-    set_common_fields(catalog, element)
+def import_catalog(
+        instance: models.Model,
+        element: dict,
+        validators: Tuple[Callable],
+        save: bool = False,
+        user: models.Model = None
+    ):
 
-    catalog.order = element.get('order') or 0
+    instance.order = element.get('order') or 0
 
-    set_lang_field(catalog, 'title', element)
-    set_lang_field(catalog, 'help', element)
+    set_lang_field(instance, 'title', element)
+    set_lang_field(instance, 'help', element)
 
-    catalog.available = element.get('available', True)
+    instance.available = element.get('available', True)
 
-    validate_instance(catalog, element, CatalogLockedValidator, CatalogUniqueURIValidator)
+    validate_instance(instance, element, *validators)
 
-    check_permissions(catalog, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return catalog
+        return instance
 
     if save:
-        logger.debug(_msg)
-        catalog.save()
-        set_m2m_through_instances(catalog, 'sections', element, 'catalog', 'section', 'catalog_sections')
-        catalog.sites.add(Site.objects.get_current())
-        catalog.editors.add(Site.objects.get_current())
-
-    return catalog
-
-
-def import_section(element, save=False, user=None):
+        set_m2m_through_instances(instance, 'sections', element, 'catalog', 'section', 'catalog_sections')
+        instance.sites.add(Site.objects.get_current())
+        instance.editors.add(Site.objects.get_current())
 
-    section, _created = get_or_return_instance(Section, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
+    return instance
 
-    _msg = make_import_info_msg(section._meta.verbose_name, _created, uri=element.get('uri'))
 
-    set_common_fields(section, element)
+def import_section(
+        instance: models.Model,
+        element: dict,
+        validators: Tuple[Callable],
+        save: bool = False,
+        user: models.Model = None
+    ):
 
-    set_lang_field(section, 'title', element)
+    set_lang_field(instance, 'title', element)
     set_lang_field(section, 'short_title', element)
 
-    validate_instance(section, element, SectionLockedValidator, SectionUniqueURIValidator)
+    validate_instance(instance, element, *validators)
 
-    check_permissions(section, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return section
+        return instance
 
     if save:
-        logger.info(_msg)
-        section.save()
-        set_reverse_m2m_through_instance(section, 'catalog', element, 'section', 'catalog', 'section_catalogs')
-        set_m2m_through_instances(section, 'pages', element, 'section', 'page', 'section_pages')
-        section.editors.add(Site.objects.get_current())
+        instance.save()
+        set_reverse_m2m_through_instance(instance, 'catalog', element, 'section', 'catalog', 'section_catalogs')
+        set_m2m_through_instances(instance, 'pages', element, 'section', 'page', 'section_pages')
+        instance.editors.add(Site.objects.get_current())
 
-    return section
+    return instance
 
 
-def import_page(element, save=False, user=None):
+def import_page(
+        instance: models.Model,
+        element: dict,
+        validators: Tuple[Callable],
+        save: bool = False,
+        user: models.Model = None
+    ):
 
-    page, _created = get_or_return_instance(Page, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
+    set_foreign_field(instance, 'attribute', element)
 
-    _msg = make_import_info_msg(page._meta.verbose_name, _created, uri=element.get('uri'))
+    instance.is_collection = element.get('is_collection') or False
 
-    set_common_fields(page, element)
-    set_foreign_field(page, 'attribute', element)
-
-    page.is_collection = element.get('is_collection') or False
-
-    set_lang_field(page, 'title', element)
+    set_lang_field(instance, 'title', element)
     set_lang_field(page, 'short_title', element)
-    set_lang_field(page, 'help', element)
-    set_lang_field(page, 'verbose_name', element)
+    set_lang_field(instance, 'help', element)
+    set_lang_field(instance, 'verbose_name', element)
 
-    validate_instance(page, element, PageLockedValidator, PageUniqueURIValidator)
+    validate_instance(instance, element,*validators)
 
-    check_permissions(page, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return page
+        return instance
 
     if save:
-        logger.info(_msg)
-        page.save()
-        set_m2m_instances(page, 'conditions', element)
-        set_reverse_m2m_through_instance(page, 'section', element, 'page', 'section', 'page_sections')
-        set_m2m_through_instances(page, 'questionsets', element, 'page', 'questionset', 'page_questionsets')
-        set_m2m_through_instances(page, 'questions', element, 'page', 'question', 'page_questions')
-        page.editors.add(Site.objects.get_current())
-
-    return page
+        instance.save()
+        set_m2m_instances(instance, 'conditions', element)
+        set_reverse_m2m_through_instance(instance, 'section', element, 'page', 'section', 'page_sections')
+        set_m2m_through_instances(instance, 'questionsets', element, 'page', 'questionset', 'page_questionsets')
+        set_m2m_through_instances(instance, 'questions', element, 'page', 'question', 'page_questions')
+        instance.editors.add(Site.objects.get_current())
 
+    return instance
 
-def import_questionset(element, save=False, user=None):
 
-    questionset, _created = get_or_return_instance(QuestionSet, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
+def import_questionset(
+        instance: models.Model,
+        element: dict,
+        validators: Tuple[Callable],
+        save: bool = False,
+        user: models.Model = None
+    ):
 
-    _msg = make_import_info_msg(questionset._meta.verbose_name, _created, uri=element.get('uri'))
+    set_foreign_field(instance, 'attribute', element)
 
-    set_common_fields(questionset, element)
-    set_foreign_field(questionset, 'attribute', element)
-
-    questionset.is_collection = element.get('is_collection') or False
+    instance.is_collection = element.get('is_collection') or False
 
     set_lang_field(questionset, 'title', element)
     set_lang_field(questionset, 'help', element)
     set_lang_field(questionset, 'verbose_name', element)
 
-    validate_instance(questionset, element, QuestionSetLockedValidator, QuestionSetUniqueURIValidator)
+    validate_instance(instance, element, *validators)
 
-    check_permissions(questionset, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return questionset
+        return instance
 
     if save:
-        logger.info(_msg)
-        questionset.save()
-        set_m2m_instances(questionset, 'conditions', element)
-        set_reverse_m2m_through_instance(questionset, 'page', element, 'questionset', 'page', 'questionset_pages')
-        set_reverse_m2m_through_instance(questionset, 'questionset', element, 'questionset', 'parent', 'questionset_parents')  # noqa: E501
-        set_m2m_through_instances(questionset, 'questionsets', element, 'parent', 'questionset', 'questionset_questionsets')  # noqa: E501
-        set_m2m_through_instances(questionset, 'questions', element, 'questionset', 'question', 'questionset_questions')
-        questionset.editors.add(Site.objects.get_current())
-
-    return questionset
-
+        instance.save()
+        set_m2m_instances(instance, 'conditions', element)
+        set_reverse_m2m_through_instance(instance, 'page', element, 'questionset', 'page', 'questionset_pages')
+        set_reverse_m2m_through_instance(instance, 'questionset', element, 'questionset', 'parent', 'questionset_parents')  # noqa: E501
+        set_m2m_through_instances(instance, 'questionsets', element, 'parent', 'questionset', 'questionset_questionsets')  # noqa: E501
+        set_m2m_through_instances(instance, 'questions', element, 'questionset', 'question', 'questionset_questions')
+        instance.editors.add(Site.objects.get_current())
 
-def import_question(element, save=False, user=None):
+    return instance
 
-    question, _created = get_or_return_instance(Question, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
 
-    _msg = make_import_info_msg(question._meta.verbose_name, _created, uri=element.get('uri'))
+def import_question(
+        instance: models.Model,
+        element: dict,
+        validators: Tuple[Callable],
+        save: bool = False,
+        user: models.Model = None
+    ):
 
-    set_common_fields(question, element)
-    set_foreign_field(question, 'attribute', element)
+    set_foreign_field(instance, 'attribute', element)
 
-    question.is_collection = element.get('is_collection') or False
-    question.is_optional = element.get('is_optional') or False
+    instance.is_collection = element.get('is_collection') or False
+    instance.is_optional = element.get('is_optional') or False
 
-    set_lang_field(question, 'text', element)
-    set_lang_field(question, 'help', element)
-    set_lang_field(question, 'default_text', element)
-    set_lang_field(question, 'verbose_name', element)
+    set_lang_field(instance, 'text', element)
+    set_lang_field(instance, 'help', element)
+    set_lang_field(instance, 'default_text', element)
+    set_lang_field(instance, 'verbose_name', element)
 
-    set_foreign_field(question, 'default_option', element)
+    set_foreign_field(instance, 'default_option', element)
 
-    question.default_external_id = element.get('default_external_id') or ''
+    instance.default_external_id = element.get('default_external_id') or ''
 
     if element.get('widget_type') in get_widget_types():
-        question.widget_type = element.get('widget_type')
+        instance.widget_type = element.get('widget_type')
     else:
-        question.widget_type = 'text'
+        instance.widget_type = 'text'
 
-    question.value_type = element.get('value_type') or ''
-    question.maximum = element.get('maximum')
-    question.minimum = element.get('minimum')
-    question.step = element.get('step')
-    question.unit = element.get('unit') or ''
-    question.width = element.get('width')
+    instance.value_type = element.get('value_type') or ''
+    instance.maximum = element.get('maximum')
+    instance.minimum = element.get('minimum')
+    instance.step = element.get('step')
+    instance.unit = element.get('unit') or ''
+    instance.width = element.get('width')
 
-    validate_instance(question, element, QuestionLockedValidator, QuestionUniqueURIValidator)
+    validate_instance(instance, element, *validators)
 
-    check_permissions(question, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return question
+        return instance
 
     if save:
-        logger.info(_msg)
-        question.save()
-        set_reverse_m2m_through_instance(question, 'page', element, 'question', 'page', 'question_pages')
-        set_reverse_m2m_through_instance(question, 'questionset', element, 'question', 'questionset', 'question_questionsets')  # noqa: E501
-        set_m2m_instances(question, 'conditions', element)
-        set_m2m_instances(question, 'optionsets', element)
-        question.editors.add(Site.objects.get_current())
-
-    return question
+        instance.save()
+        set_reverse_m2m_through_instance(instance, 'page', element, 'question', 'page', 'question_pages')
+        set_reverse_m2m_through_instance(instance, 'questionset', element, 'question', 'questionset', 'question_questionsets')  # noqa: E501
+        set_m2m_instances(instance, 'conditions', element)
+        set_m2m_instances(instance, 'optionsets', element)
+        instance.editors.add(Site.objects.get_current())
+
+    return instance
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index f292b859ac..19a6fe23e6 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -1,59 +1,53 @@
 import logging
+from typing import Callable, Tuple
 
 from django.contrib.sites.models import Site
+from django.db import models
 
 from rdmo.core.imports import (
     check_permissions,
-    get_or_return_instance,
-    make_import_info_msg,
-    set_common_fields,
     set_foreign_field,
     set_lang_field,
     set_m2m_instances,
     validate_instance,
 )
 
-from .models import Task
-from .validators import TaskLockedValidator, TaskUniqueURIValidator
-
 logger = logging.getLogger(__name__)
 
 
-def import_task(element, save=False, user=None):
-    task, _created = get_or_return_instance(Task, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
-
-    _msg = make_import_info_msg(task._meta.verbose_name, _created, uri=element.get('uri'))
+def import_task(
+        instance: models.Model,
+        element: dict,
+        validators: Tuple[Callable],
+        save: bool = False,
+        user: models.Model = None
+    ):
 
-    set_common_fields(task, element)
+    instance.order = element.get('order') or 0
 
-    task.order = element.get('order') or 0
+    set_lang_field(instance, 'title', element)
+    set_lang_field(instance, 'text', element)
 
-    set_lang_field(task, 'title', element)
-    set_lang_field(task, 'text', element)
+    set_foreign_field(instance, 'start_attribute', element)
+    set_foreign_field(instance, 'end_attribute', element)
 
-    set_foreign_field(task, 'start_attribute', element)
-    set_foreign_field(task, 'end_attribute', element)
+    instance.days_before = element.get('days_before')
+    instance.days_after = element.get('days_after')
 
-    task.days_before = element.get('days_before')
-    task.days_after = element.get('days_after')
+    instance.available = element.get('available', True)
 
-    task.available = element.get('available', True)
+    validate_instance(instance, element, *validators)
 
-    validate_instance(task, element, TaskUniqueURIValidator, TaskLockedValidator)
-
-    check_permissions(task, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return task
+        return instance
 
     if save:
-        logger.info(_msg)
-        task.save()
-        set_m2m_instances(task, 'catalogs', element)
-        set_m2m_instances(task, 'conditions', element)
-        task.sites.add(Site.objects.get_current())
-        task.editors.add(Site.objects.get_current())
-
-    return task
+        instance.save()
+        set_m2m_instances(instance, 'catalogs', element)
+        set_m2m_instances(instance, 'conditions', element)
+        instance.sites.add(Site.objects.get_current())
+        instance.editors.add(Site.objects.get_current())
+
+    return instance
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 07e16b2a44..9909926dc6 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -1,53 +1,46 @@
 import logging
+from typing import Callable, Tuple
 
 from django.contrib.sites.models import Site
+from django.db import models
 
 from rdmo.core.imports import (
     check_permissions,
-    get_or_return_instance,
-    make_import_info_msg,
-    set_common_fields,
     set_lang_field,
     set_m2m_instances,
     validate_instance,
 )
 
-from .models import View
-from .validators import ViewLockedValidator, ViewUniqueURIValidator
-
 logger = logging.getLogger(__name__)
 
 
-def import_view(element, save=False, user=None):
-
-    view, _created = get_or_return_instance(View, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
-
-    _msg = make_import_info_msg(view._meta.verbose_name, _created, uri=element.get('uri'))
-
-    set_common_fields(view, element)
+def import_view(
+        instance: models.Model,
+        element: dict,
+        validators: Tuple[Callable],
+        save: bool = False,
+        user: models.Model = None
+    ):
 
-    view.order = element.get('order') or 0
-    view.template = element.get('template')
+    instance.order = element.get('order') or 0
+    instance.template = element.get('template')
 
-    set_lang_field(view, 'title', element)
-    set_lang_field(view, 'help', element)
+    set_lang_field(instance, 'title', element)
+    set_lang_field(instance, 'help', element)
 
-    view.available = element.get('available', True)
+    instance.available = element.get('available', True)
 
-    validate_instance(view, element, ViewLockedValidator, ViewUniqueURIValidator)
+    validate_instance(instance, element, *validators)
 
-    check_permissions(view, element, user)
+    check_permissions(instance, element, user)
 
     if element.get('errors'):
-        return view
+        return instance
 
     if save:
-        logger.info(_msg)
-        view.save()
-        set_m2m_instances(view, 'catalogs', element)
-        view.sites.add(Site.objects.get_current())
-        view.editors.add(Site.objects.get_current())
+        instance.save()
+        set_m2m_instances(instance, 'catalogs', element)
+        instance.sites.add(Site.objects.get_current())
+        instance.editors.add(Site.objects.get_current())
 
-    return view
+    return instance

From 921d4959d5577df7fc3c4353b8e30e4adbd01b85 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 10 Nov 2023 14:04:01 +0100
Subject: [PATCH 014/205] feat: add FieldsDiffs import component

---
 .../js/components/import/ImportAttribute.js    |  4 ++++
 .../js/components/import/common/Fields.js      | 10 ++++++----
 .../js/components/import/common/FieldsDiffs.js | 18 ++++++++++++------
 .../assets/js/components/main/Import.js        | 11 ++++++++++-
 4 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportAttribute.js b/rdmo/management/assets/js/components/import/ImportAttribute.js
index a360f9c14f..c680d82a50 100644
--- a/rdmo/management/assets/js/components/import/ImportAttribute.js
+++ b/rdmo/management/assets/js/components/import/ImportAttribute.js
@@ -21,6 +21,10 @@ const ImportAttribute = ({ config, attribute, importActions }) => {
         <WarningLink element={attribute} onClick={showFields} />
         <ErrorLink element={attribute} onClick={showFields} />
         <ShowLink element={attribute} onClick={showFields} />
+        {
+          attribute.updated && !attribute.created &&
+          <p className="element-link fa fa-pencil"></p>
+          }
       </div>
       <div className="checkbox">
         <label className="mr-5">
diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index 6ebdc134db..ba8bf0f8b7 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -5,10 +5,11 @@ import isString from 'lodash/isString'
 import isUndefined from 'lodash/isUndefined'
 import truncate from 'lodash/truncate'
 import uniqueId from 'lodash/uniqueId'
-import Diffs from './FieldsDiffs'
+import isEmpty from 'lodash/isEmpty'
 
-import { codeClass } from '../../../constants/elements'
 
+import { codeClass } from '../../../constants/elements'
+import FieldsDiffs from './FieldsDiffs'
 
 const excludeKeys = [
   'created',
@@ -24,7 +25,7 @@ const excludeKeys = [
   'uri_prefix',
   'valid',
   'warnings',
-  'diffs'
+  'original'
 ]
 
 const Fields = ({ element }) => {
@@ -54,7 +55,8 @@ const Fields = ({ element }) => {
                   }
                 </div>
                 {
-                  element.updated && <Diffs element={element} field={key}/>
+                  isEmpty(element.errors) && !isEmpty(element.original) && element.updated &&
+                  element.original[key] != value && <FieldsDiffs element={element} field={key}/>
                   }
               </div>
             )
diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index 07ea596a29..a54f870611 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -2,13 +2,19 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import isEmpty from 'lodash/isEmpty'
 import ReactDiffViewer from 'react-diff-viewer-continued'
+import { isUndefined } from 'lodash'
 
 
-const Diffs = ({ element, field }) => {
-  return !isEmpty(element.diffs[field]) && <div className="col-sm-12">
+const FieldsDiffs = ({ element, field }) => {
+  return !isUndefined(element) &&
+          !isUndefined(element[field]) &&
+          !isEmpty(element.original) &&
+          !isEmpty(element.original[field]) &&
+          isEmpty(element.errors) &&
+   <div className="col-sm-12">
     <ReactDiffViewer
-        oldValue={element.diffs[field].new_value}
-        newValue={element.diffs[field].old_value}
+        oldValue={element.original[field]}
+        newValue={element[field]}
         splitView={true}
         hideLineNumbers={true}
         leftTitle={gettext('Current')}
@@ -18,9 +24,9 @@ const Diffs = ({ element, field }) => {
     </div>
 }
 
-Diffs.propTypes = {
+FieldsDiffs.propTypes = {
   element: PropTypes.object.isRequired,
   field: PropTypes.string.isRequired,
 }
 
-export default Diffs
+export default FieldsDiffs
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 20451fb697..ff6bdf971f 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -28,6 +28,8 @@ const Import = ({ config, imports, importActions }) => {
   const updateFilterUriPrefix = (value) => importActions.updateConfig('filter.import.elements.uri_prefix', value)
   const updateDisplayCatalogURI = (value) => importActions.updateConfig('display.uri.catalogs', value)
   const updatedElements = elements.filter(element => element.updated)
+  const createdElements = elements.filter(element => element.created)
+  const importErrors = elements.filter(element => !isEmpty(element.errors))
 
   return (
     <div className="panel panel-default panel-import">
@@ -35,11 +37,18 @@ const Import = ({ config, imports, importActions }) => {
         <strong>{gettext('Import')}</strong>
         <div className="pull-right">
           {
-            updatedElements.length > 0 && <p>Updated {updatedElements.length} </p>
+            updatedElements.length > -1 && <span>{gettext('Updated')}: {updatedElements.length} </span>
+          }
+          {
+            createdElements.length > -1 && <span>{gettext('Created')}: {createdElements.length} </span>
+          }
+          {
+            importErrors.length > -1 && <span>{gettext('Errors')}: {importErrors.length} </span>
           }
         </div>
       </div>
     <div className="panel-body">
+    {/* TODO: still to implement functions for filter, uri_prefix dropdown, disply checkox etc.. */}
     <div className="row">
       <div className={'col-sm-8'}>
         <FilterString value={get(config, 'filter.catalogs.search', '')} onChange={updateFilterString}

From 3313c2c1657a4225a87d80c06dd3e3a1e60af877 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 13 Nov 2023 09:31:49 +0100
Subject: [PATCH 015/205] fix import_catalog, add missing save and fix typo
 QuestionSet

---
 rdmo/management/imports.py | 36 +++++++++++++++++++-----------------
 rdmo/questions/imports.py  |  1 +
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 1fa07d9790..faa98e6e60 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -71,7 +71,7 @@
         "validators":  (PageLockedValidator, PageUniqueURIValidator),
     },
     "questions.questionset": {
-        "dotted_path": "rdmo.questions.models.questionset.Questionset",
+        "dotted_path": "rdmo.questions.models.questionset.QuestionSet",
         "import_method": import_questionset,
         "validators": (QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     },
@@ -81,12 +81,12 @@
         "validators": (QuestionLockedValidator, QuestionUniqueURIValidator),
     },
     "tasks.task": {
-        "dotted_path": "rdmo.tasks.models.task.Task",
+        "dotted_path": "rdmo.tasks.models.Task",
         "import_method": import_task,
         "validators": (TaskLockedValidator, TaskUniqueURIValidator),
     },
     "views.view": {
-        "dotted_path": "rdmo.views.models.view.View",
+        "dotted_path": "rdmo.views.models.View",
         "import_method": import_view,
         "validators": (ViewLockedValidator, ViewUniqueURIValidator),
     },
@@ -105,13 +105,25 @@ def import_elements(uploaded_elements, save=True, user=None):
     return imported_elements
 
 
+def filter_warnings(element, elements):
+    # remove warnings regarding elements which are in the elements list
+    warnings = []
+    for uri, messages in element['warnings'].items():
+        if not next(filter(lambda e: e['uri'] == uri, elements), None):
+            warnings += messages
+
+    element['warnings'] = warnings
+    return element
+
+
 def import_element(
         model_path: Optional[str] = None,
         element: Optional[dict] = None,
         save: bool = True,
-        user = None):
+        user = None
+    ):
 
-    if element is None:
+    if element is None or model_path is None:
         return element
 
     element.update({
@@ -137,24 +149,14 @@ def import_element(
     if save and not element.get('errors'):
         logger.info(_msg)
 
-    # TODO need to filter or serialize the same keys as in the element
     element = filter_original(element)
 
     return element
 
 
-def filter_warnings(element, elements):
-    # remove warnings regarding elements which are in the elements list
-    warnings = []
-    for uri, messages in element['warnings'].items():
-        if not next(filter(lambda e: e['uri'] == uri, elements), None):
-            warnings += messages
-
-    element['warnings'] = warnings
-    return element
-
 def filter_original(element):
+    '''select only keys for original that are in the element.'''
     element['original'] = {k: val for k, val in
                         element.get('original', {}).items()
-                        if k in element}
+                        if k in element and k != 'original'}
     return element
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index dd84a4a86d..9c6af16373 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -42,6 +42,7 @@ def import_catalog(
         return instance
 
     if save:
+        instance.save()
         set_m2m_through_instances(instance, 'sections', element, 'catalog', 'section', 'catalog_sections')
         instance.sites.add(Site.objects.get_current())
         instance.editors.add(Site.objects.get_current())

From 4cf1acfb0d9bac60267454e49311bc7b6ca9a33f Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 13 Nov 2023 14:29:37 +0100
Subject: [PATCH 016/205] chore: refactor original element when updated

---
 rdmo/core/imports.py       | 12 +--------
 rdmo/management/imports.py | 51 +++++++++++++++++++++++++++++++-------
 2 files changed, 43 insertions(+), 20 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 06450552d5..d5c4894827 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -8,7 +8,6 @@
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
-from django.forms.models import model_to_dict
 from django.utils.module_loading import import_string as django_import_string
 
 from rest_framework.utils import model_meta
@@ -77,20 +76,11 @@ def common_import_methods(model_dotted_path: str,
                               element: Optional[dict]=None):
     model = django_import_string(model_dotted_path)
     instance, _created = get_or_return_instance(model, uri=element.get('uri'))
-    element['created'] = _created
-    element['updated'] = not _created
 
     _msg = make_import_info_msg(model._meta.verbose_name, _created, uri=element.get('uri'))
 
-     # TODO maybe move to another place
-    element['original'] = {}
-    if element.get("updated"):
-        original = model_to_dict(instance)
-        original.update(**{k : model_to_dict(val) for k, val in original.items() if isinstance(val, models.Model)})
-        element['original'] = original
-
     set_common_fields(instance, element)
-    return instance, element, _msg
+    return instance, element, _msg, _created
 
 
 
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index faa98e6e60..96a23ec3f7 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -2,6 +2,9 @@
 from collections import defaultdict
 from typing import Optional
 
+from django.db import models
+from django.forms.models import model_to_dict
+
 from rdmo.conditions.imports import import_condition
 from rdmo.conditions.validators import ConditionLockedValidator, ConditionUniqueURIValidator
 from rdmo.core.imports import common_import_methods
@@ -123,7 +126,7 @@ def import_element(
         user = None
     ):
 
-    if element is None or model_path is None:
+    if element is None:
         return element
 
     element.update({
@@ -134,29 +137,59 @@ def import_element(
         'original': defaultdict()
     })
 
+    if model_path is None:
+        return element
+
     model_import = ELEMENT_MODEL_IMPORT_MAPPER[model_path]
     import_method = model_import['import_method']
     model_path = model_import['dotted_path']
     validators = model_import['validators']
 
-    instance, element, _msg = common_import_methods(
+    instance, element, _msg, _created = common_import_methods(
                             model_path,
                             uri=element.get('uri'),
                             element=element
                     )
+    _updated = not _created
+    # for when the element is updated
+    # keep a dict of the original
+    # needs to be created here, else the changes will be overwritten
+    original = model_to_dict(instance) if _updated else {}
 
     instance = import_method(instance, element, validators, save, user)
-    if save and not element.get('errors'):
+
+    if element.get('errors'):
+        return element
+
+    if save:
         logger.info(_msg)
+        element['created'] = _created
+        element['updated'] = _updated
 
-    element = filter_original(element)
+    if _updated:
+        original = filter_original(element, original)
+        element['updated'] = _updated
+        element['original'] = original
 
     return element
 
 
-def filter_original(element):
+def filter_original(element, original):
     '''select only keys for original that are in the element.'''
-    element['original'] = {k: val for k, val in
-                        element.get('original', {}).items()
-                        if k in element and k != 'original'}
-    return element
+    # filter for keys
+    original = {
+        k: val for k, val in
+        original.items()
+        if k in element
+    }
+    # replace the lists of nested model values
+    # in the "original" dict
+    original_replace = {
+        k: element[k] for k, val in
+        original.items() if (
+            isinstance(val, list) and
+            any(isinstance(i, models.Model) for i in val)
+        )
+    }
+    original.update(**original_replace)
+    return original

From f79187c9c0708a8783a0c794bf01555b01df1b88 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 13 Nov 2023 14:30:17 +0100
Subject: [PATCH 017/205] refactor imported_elements in ImportViewSet

---
 rdmo/management/viewsets.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index 87eaef7233..3ec14f3b25 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -109,10 +109,10 @@ def create(self, request, *args, **kwargs):
             raise ValidationError({'elements': [_('This is not a valid RDMO import JSON.')]}) from e
 
         # step 3: import the elements
-        import_elements(elements, user=request.user)
+        imported_elements = import_elements(elements, user=request.user)
 
         # step 4: return the list of elements
-        return Response(elements)
+        return Response(imported_elements)
 
 
 class ElementToggleCurrentSiteViewSetMixin:

From bd864eff0b529887b106e2e3d1275acc32162748 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 13 Nov 2023 14:31:36 +0100
Subject: [PATCH 018/205] tests: fix test_create in  test_viewset_upload

---
 rdmo/management/tests/test_viewset_upload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/rdmo/management/tests/test_viewset_upload.py b/rdmo/management/tests/test_viewset_upload.py
index 3715a44e0f..6132d67f16 100644
--- a/rdmo/management/tests/test_viewset_upload.py
+++ b/rdmo/management/tests/test_viewset_upload.py
@@ -54,7 +54,10 @@ def test_create(db, client, username, password):
     assert response.status_code == status_map['create'][username], response.json()
     if response.status_code == 200:
         for element in response.json():
-            assert element.get('updated') is False
+            if username in ['api', 'editor']:
+                assert element.get('updated') is True
+            else:
+                assert element.get('updated') is False
 
 
 @pytest.mark.parametrize('username,password', users)

From de911813bc2cb59323ddde54e62c813c803483b6 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 13 Nov 2023 21:24:47 +0100
Subject: [PATCH 019/205] chore: refactor lang field and common, mv
 ElementImportHelper

---
 rdmo/core/imports.py | 73 ++++++++++++++++++++++++++++++--------------
 1 file changed, 50 insertions(+), 23 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index d5c4894827..0c9632bc86 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -1,10 +1,12 @@
+import copy
 import logging
 import tempfile
 import time
+from dataclasses import dataclass, field
 from os.path import join as pj
 from pathlib import Path
 from random import randint
-from typing import Optional, Tuple
+from typing import Callable, Iterable, Optional, Sequence, Tuple
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
@@ -17,6 +19,12 @@
 logger = logging.getLogger(__name__)
 
 
+ELEMENT_COMMON_FIELDS = (
+    'uri_prefix',
+    'uri_path',
+    'key',
+    'comment',
+)
 IMPORT_INFO_MSG = 'Importing {model} {uri} from {filename}.'
 
 
@@ -63,34 +71,53 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
 
     return "%s %s updated", verbose_name, uri
 
+@dataclass
+class ElementImportHelper:
+    model: str
+    dotted_path: str
+    import_method: Callable
+    validators: Iterable[Callable]
+    lang_fields: Sequence[str] = field(default_factory=list)
+    common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
 
-def set_common_fields(instance, element):
-    instance.uri_prefix = element.get('uri_prefix') or ''
-    instance.uri_path = element.get('uri_path') or ''
-    instance.key = element.get('key') or ''
-    instance.comment = element.get('comment') or ''
 
+def common_import_methods(model_dotted_path: str, uri: Optional[str] = None):
 
-def common_import_methods(model_dotted_path: str,
-                              uri: Optional[str]=None,
-                              element: Optional[dict]=None):
     model = django_import_string(model_dotted_path)
-    instance, _created = get_or_return_instance(model, uri=element.get('uri'))
-
-    _msg = make_import_info_msg(model._meta.verbose_name, _created, uri=element.get('uri'))
-
-    set_common_fields(instance, element)
-    return instance, element, _msg, _created
-
-
+    instance, _created = get_or_return_instance(model, uri=uri)
+
+    # for when the element is updated
+    # keep a dict of the original
+    # needs to be created here, else the changes will be overwritten
+    original = copy.deepcopy(instance) if not _created else None
+
+    _msg = make_import_info_msg(model._meta.verbose_name, _created, uri=uri)
+    return instance, _msg, _created, original
+
+
+def get_lang_field_values(field_name: str,
+                            element: Optional[dict] = None,
+                            instance: models.Model = None,
+                            by_field: bool = True):
+    if (element and instance):
+        raise ValueError("Please choose one of each")
+
+    ret = {}
+    for lang_code, _, lang_field in get_languages():
+        name_code = f'{field_name}_{lang_code}'
+        name_field = f'{field_name}_{lang_field}'
+        get_key = name_field if by_field else name_code
+        set_key = name_code if by_field else name_field
+        if element:
+            ret[set_key] = element.get(get_key, '')
+        if instance:
+            ret[set_key] = getattr(instance, get_key, '')
+    return ret
 
 def set_lang_field(instance, field_name, element):
-    for lang_code, lang_string, lang_field in get_languages():
-        field = element.get(f'{field_name}_{lang_code}')
-        if field:
-            setattr(instance, f'{field_name}_{lang_field}', field)
-        else:
-            setattr(instance, f'{field_name}_{lang_field}', '')
+    lang_fields = get_lang_field_values(field_name, element=element)
+    for field_lang_name, field_value in lang_fields.items():
+        setattr(instance, field_lang_name, field_value)
 
 
 def set_foreign_field(instance, field_name, element) -> None:

From f10baef5a7198f95de03908a5faf5baa464a156d Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 13 Nov 2023 21:25:47 +0100
Subject: [PATCH 020/205] chore: refactor imports mv methods back to apps, fix
 original

---
 rdmo/conditions/imports.py |  10 ++
 rdmo/domain/imports.py     |  10 ++
 rdmo/management/imports.py | 198 +++++++++++++------------------------
 rdmo/options/imports.py    |  25 ++++-
 rdmo/questions/imports.py  |  59 +++++++++--
 rdmo/tasks/imports.py      |  12 ++-
 rdmo/views/imports.py      |  15 ++-
 7 files changed, 185 insertions(+), 144 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index 74b3987b35..2be876b7d2 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -4,7 +4,9 @@
 from django.contrib.sites.models import Site
 from django.db import models
 
+from rdmo.conditions.validators import ConditionLockedValidator, ConditionUniqueURIValidator
 from rdmo.core.imports import (
+    ElementImportHelper,
     check_permissions,
     set_foreign_field,
     validate_instance,
@@ -39,3 +41,11 @@ def import_condition(
         instance.editors.add(Site.objects.get_current())
 
     return instance
+
+
+import_helper_condition = ElementImportHelper(
+    model="conditions.condition",
+    dotted_path='rdmo.conditions.models.Condition',
+    import_method=import_condition,
+    validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
+)
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index be0ca7e829..0854b57fd0 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -4,10 +4,12 @@
 from django.contrib.sites.models import Site
 
 from rdmo.core.imports import (
+    ElementImportHelper,
     check_permissions,
     set_foreign_field,
     validate_instance,
 )
+from rdmo.domain.validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
 
 logger = logging.getLogger(__name__)
 
@@ -30,3 +32,11 @@ def import_attribute(instance, element, validators: Tuple[Callable], save=False,
         instance.editors.add(Site.objects.get_current())
 
     return instance
+
+
+import_helper_attribute = ElementImportHelper(
+    model="domain.attribute",
+    dotted_path='rdmo.domain.models.Attribute',
+    import_method=import_attribute,
+    validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
+)
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 96a23ec3f7..0d9e7ecd29 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -2,98 +2,46 @@
 from collections import defaultdict
 from typing import Optional
 
-from django.db import models
 from django.forms.models import model_to_dict
 
-from rdmo.conditions.imports import import_condition
-from rdmo.conditions.validators import ConditionLockedValidator, ConditionUniqueURIValidator
-from rdmo.core.imports import common_import_methods
-from rdmo.domain.imports import import_attribute
-from rdmo.domain.validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
-from rdmo.options.imports import import_option, import_optionset
-from rdmo.options.validators import (
-    OptionLockedValidator,
-    OptionSetLockedValidator,
-    OptionSetUniqueURIValidator,
-    OptionUniqueURIValidator,
+from rdmo.conditions.imports import import_helper_condition
+from rdmo.core.imports import common_import_methods, get_lang_field_values, set_lang_field
+from rdmo.domain.imports import import_helper_attribute
+from rdmo.options.imports import import_helper_option, import_helper_optionset
+from rdmo.questions.imports import (
+    import_helper_catalog,
+    import_helper_page,
+    import_helper_question,
+    import_helper_questionset,
+    import_helper_section,
 )
-from rdmo.questions.imports import import_catalog, import_page, import_question, import_questionset, import_section
-from rdmo.questions.validators import (
-    CatalogLockedValidator,
-    CatalogUniqueURIValidator,
-    PageLockedValidator,
-    PageUniqueURIValidator,
-    QuestionLockedValidator,
-    QuestionSetLockedValidator,
-    QuestionSetUniqueURIValidator,
-    QuestionUniqueURIValidator,
-    SectionLockedValidator,
-    SectionUniqueURIValidator,
-)
-from rdmo.tasks.imports import import_task
-from rdmo.tasks.validators import TaskLockedValidator, TaskUniqueURIValidator
-from rdmo.views.imports import import_view
-from rdmo.views.validators import ViewLockedValidator, ViewUniqueURIValidator
+from rdmo.tasks.imports import import_helper_task
+from rdmo.views.imports import import_helper_view
 
 logger = logging.getLogger(__name__)
 
-ELEMENT_MODEL_IMPORT_MAPPER = {
-    "conditions.condition": {
-        "dotted_path": 'rdmo.conditions.models.Condition',
-        "import_method": import_condition,
-        "validators": (ConditionLockedValidator, ConditionUniqueURIValidator)
-    },
-    "domain.attribute": {
-        "dotted_path": 'rdmo.domain.models.Attribute',
-        "import_method": import_attribute,
-        "validators": (AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator)
-    },
-    "options.optionset": {
-        "dotted_path": "rdmo.options.models.OptionSet",
-        "import_method": import_optionset,
-        "validators": (OptionSetLockedValidator, OptionSetUniqueURIValidator),
-    },
-    "options.option": {
-        "dotted_path": "rdmo.options.models.Option",
-        "import_method": import_option,
-        "validators": (OptionLockedValidator, OptionUniqueURIValidator),
-    },
-    "questions.catalog": {
-        "dotted_path": "rdmo.questions.models.catalog.Catalog",
-        "import_method": import_catalog,
-        "validators": (CatalogLockedValidator, CatalogUniqueURIValidator),
-    },
-    "questions.section": {
-        "dotted_path": "rdmo.questions.models.section.Section",
-        "import_method": import_section,
-        "validators": (SectionLockedValidator, SectionUniqueURIValidator),
-    },
-    "questions.page": {
-        "dotted_path": "rdmo.questions.models.page.Page",
-        "import_method": import_page,
-        "validators":  (PageLockedValidator, PageUniqueURIValidator),
-    },
-    "questions.questionset": {
-        "dotted_path": "rdmo.questions.models.questionset.QuestionSet",
-        "import_method": import_questionset,
-        "validators": (QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
-    },
-    "questions.question": {
-        "dotted_path": "rdmo.questions.models.question.Question",
-        "import_method": import_question,
-        "validators": (QuestionLockedValidator, QuestionUniqueURIValidator),
-    },
-    "tasks.task": {
-        "dotted_path": "rdmo.tasks.models.Task",
-        "import_method": import_task,
-        "validators": (TaskLockedValidator, TaskUniqueURIValidator),
-    },
-    "views.view": {
-        "dotted_path": "rdmo.views.models.View",
-        "import_method": import_view,
-        "validators": (ViewLockedValidator, ViewUniqueURIValidator),
-    },
-}
+
+ELEMENT_IMPORT_HELPERS = {
+    "conditions.condition": import_helper_condition,
+    "domain.attribute": import_helper_attribute,
+    "options.optionset": import_helper_optionset,
+    "options.option": import_helper_option,
+    "questions.catalog": import_helper_catalog,
+    "questions.section": import_helper_section,
+    "questions.page": import_helper_page,
+    "questions.questionset": import_helper_questionset,
+    "questions.question": import_helper_question,
+    "tasks.task": import_helper_task,
+    "views.view": import_helper_view
+    }
+
+IMPORT_ELEMENT_INIT_DICT = {
+        'warnings': defaultdict(list),
+        'errors': None,
+        'created': False,
+        'updated': False,
+        'original': defaultdict()
+    }
 
 
 def import_elements(uploaded_elements, save=True, user=None):
@@ -129,67 +77,55 @@ def import_element(
     if element is None:
         return element
 
-    element.update({
-        'warnings': defaultdict(list),
-        'errors': [],
-        'created': False,
-        'updated': False,
-        'original': defaultdict()
-    })
+    element.update(IMPORT_ELEMENT_INIT_DICT)
+    element['errors'] = []
 
     if model_path is None:
         return element
 
-    model_import = ELEMENT_MODEL_IMPORT_MAPPER[model_path]
-    import_method = model_import['import_method']
-    model_path = model_import['dotted_path']
-    validators = model_import['validators']
+    import_helper = ELEMENT_IMPORT_HELPERS[model_path]
+    import_method = import_helper.import_method
+    model_path = import_helper.dotted_path
+    validators = import_helper.validators
+    lang_field_names = import_helper.lang_fields
+    uri = element.get('uri')
 
-    instance, element, _msg, _created = common_import_methods(
+    instance, _msg, _created, original_instance = common_import_methods(
                             model_path,
-                            uri=element.get('uri'),
-                            element=element
+                            uri=uri
                     )
+    # prepare original element when updated (maybe rename into lookup)
     _updated = not _created
-    # for when the element is updated
-    # keep a dict of the original
-    # needs to be created here, else the changes will be overwritten
-    original = model_to_dict(instance) if _updated else {}
-
+    original_element = {}
+    if _updated:
+        original_element = model_to_dict(original_instance)
+        original_element = {k: original_element.get(k, element.get(k))
+                            for k in element.keys() if k not in IMPORT_ELEMENT_INIT_DICT}
+    # set common field values from element on instance
+    for common_field in import_helper.common_fields:
+        setattr(instance, common_field, element.get(common_field) or '')
+    # set language fields
+    for lang_field_name in lang_field_names:
+        set_lang_field(instance, lang_field_name, element)
+        if original_instance is not None:
+            # add the lang_code fields from the original instance
+            lang_field_values = get_lang_field_values(lang_field_name, instance=original_instance)
+            original_element.update(lang_field_values)
+    # call the element specific import method
     instance = import_method(instance, element, validators, save, user)
 
     if element.get('errors'):
         return element
 
+    if _updated:
+        element['updated'] = _updated
+        # make json serializable, keep only strings
+        original_element_json = {k: val for k, val in original_element.items() if isinstance(val, str)}
+        element['original'] = original_element_json
+
     if save:
         logger.info(_msg)
         element['created'] = _created
         element['updated'] = _updated
 
-    if _updated:
-        original = filter_original(element, original)
-        element['updated'] = _updated
-        element['original'] = original
-
     return element
-
-
-def filter_original(element, original):
-    '''select only keys for original that are in the element.'''
-    # filter for keys
-    original = {
-        k: val for k, val in
-        original.items()
-        if k in element
-    }
-    # replace the lists of nested model values
-    # in the "original" dict
-    original_replace = {
-        k: element[k] for k, val in
-        original.items() if (
-            isinstance(val, list) and
-            any(isinstance(i, models.Model) for i in val)
-        )
-    }
-    original.update(**original_replace)
-    return original
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 4437623bf4..ad8bc17554 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -5,13 +5,19 @@
 from django.db import models
 
 from rdmo.core.imports import (
+    ElementImportHelper,
     check_permissions,
-    set_lang_field,
     set_m2m_instances,
     set_m2m_through_instances,
     set_reverse_m2m_through_instance,
     validate_instance,
 )
+from rdmo.options.validators import (
+    OptionLockedValidator,
+    OptionSetLockedValidator,
+    OptionSetUniqueURIValidator,
+    OptionUniqueURIValidator,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +49,15 @@ def import_option(
     return instance
 
 
+import_helper_option = ElementImportHelper(
+    model="options.option",
+    dotted_path="rdmo.options.models.Option",
+    import_method=import_option,
+    validators=(OptionLockedValidator, OptionUniqueURIValidator),
+    lang_fields=('text',)
+)
+
+
 def import_optionset(
         instance: models.Model,
         element: dict,
@@ -70,3 +85,11 @@ def import_optionset(
         instance.editors.add(Site.objects.get_current())
 
     return instance
+
+
+import_helper_optionset = ElementImportHelper(
+    model="options.optionset",
+    dotted_path="rdmo.options.models.OptionSet",
+    import_method=import_optionset,
+    validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
+)
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 9c6af16373..92aed772f8 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -5,14 +5,26 @@
 from django.db import models
 
 from rdmo.core.imports import (
+    ElementImportHelper,
     check_permissions,
     set_foreign_field,
-    set_lang_field,
     set_m2m_instances,
     set_m2m_through_instances,
     set_reverse_m2m_through_instance,
     validate_instance,
 )
+from rdmo.questions.validators import (
+    CatalogLockedValidator,
+    CatalogUniqueURIValidator,
+    PageLockedValidator,
+    PageUniqueURIValidator,
+    QuestionLockedValidator,
+    QuestionSetLockedValidator,
+    QuestionSetUniqueURIValidator,
+    QuestionUniqueURIValidator,
+    SectionLockedValidator,
+    SectionUniqueURIValidator,
+)
 
 from .utils import get_widget_types
 
@@ -29,9 +41,6 @@ def import_catalog(
 
     instance.order = element.get('order') or 0
 
-    set_lang_field(instance, 'title', element)
-    set_lang_field(instance, 'help', element)
-
     instance.available = element.get('available', True)
 
     validate_instance(instance, element, *validators)
@@ -58,9 +67,6 @@ def import_section(
         user: models.Model = None
     ):
 
-    set_lang_field(instance, 'title', element)
-    set_lang_field(section, 'short_title', element)
-
     validate_instance(instance, element, *validators)
 
     check_permissions(instance, element, user)
@@ -197,3 +203,42 @@ def import_question(
         instance.editors.add(Site.objects.get_current())
 
     return instance
+
+
+import_helper_catalog = ElementImportHelper(
+    model="questions.catalog",
+    dotted_path="rdmo.questions.models.catalog.Catalog",
+    import_method=import_catalog,
+    validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
+    lang_fields=('title', 'help')
+)
+
+import_helper_section = ElementImportHelper(
+    model="questions.section",
+    dotted_path="rdmo.questions.models.section.Section",
+    import_method=import_section,
+    validators=(SectionLockedValidator, SectionUniqueURIValidator),
+    lang_fields=('title',)
+)
+import_helper_page = ElementImportHelper(
+    model="questions.page",
+    dotted_path="rdmo.questions.models.page.Page",
+    import_method=import_page,
+    validators= (PageLockedValidator, PageUniqueURIValidator),
+    lang_fields=('title', 'help', 'verbose_name', 'verbose_name_plural')
+)
+
+import_helper_questionset = ElementImportHelper(
+    model="questions.questionset",
+    dotted_path="rdmo.questions.models.questionset.QuestionSet",
+    import_method=import_questionset,
+    validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
+    lang_fields=('title', 'help', 'verbose_name', 'verbose_name_plural')
+)
+import_helper_question = ElementImportHelper(
+    model="questions.question",
+    dotted_path="rdmo.questions.models.question.Question",
+    import_method=import_question,
+    validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
+    lang_fields=('text', 'help', 'default_text', 'verbose_name', 'verbose_name_plural')
+)
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 19a6fe23e6..aef1132fa6 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -5,12 +5,13 @@
 from django.db import models
 
 from rdmo.core.imports import (
+    ElementImportHelper,
     check_permissions,
     set_foreign_field,
-    set_lang_field,
     set_m2m_instances,
     validate_instance,
 )
+from rdmo.tasks.validators import TaskLockedValidator, TaskUniqueURIValidator
 
 logger = logging.getLogger(__name__)
 
@@ -51,3 +52,12 @@ def import_task(
         instance.editors.add(Site.objects.get_current())
 
     return instance
+
+
+import_helper_task = ElementImportHelper(
+    model="tasks.task",
+    dotted_path="rdmo.tasks.models.Task",
+    import_method=import_task,
+    validators=(TaskLockedValidator, TaskUniqueURIValidator),
+    lang_fields=('title', 'text')
+)
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 9909926dc6..06bae45def 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -5,11 +5,12 @@
 from django.db import models
 
 from rdmo.core.imports import (
+    ElementImportHelper,
     check_permissions,
-    set_lang_field,
     set_m2m_instances,
     validate_instance,
 )
+from rdmo.views.validators import ViewLockedValidator, ViewUniqueURIValidator
 
 logger = logging.getLogger(__name__)
 
@@ -25,9 +26,6 @@ def import_view(
     instance.order = element.get('order') or 0
     instance.template = element.get('template')
 
-    set_lang_field(instance, 'title', element)
-    set_lang_field(instance, 'help', element)
-
     instance.available = element.get('available', True)
 
     validate_instance(instance, element, *validators)
@@ -44,3 +42,12 @@ def import_view(
         instance.editors.add(Site.objects.get_current())
 
     return instance
+
+
+import_helper_view = ElementImportHelper(
+    model="views.view",
+    dotted_path="rdmo.views.models.View",
+    import_method=import_view,
+    validators=(ViewLockedValidator, ViewUniqueURIValidator),
+    lang_fields=('title', 'help')
+)

From e6cb7b7ca808c18f6bddcbdf073a5d71aa1a0971 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 13 Nov 2023 21:26:45 +0100
Subject: [PATCH 021/205] chore: mv all import components to ImportElement

---
 .../js/components/import/ImportElement.js     | 54 +++++++++++++++++++
 .../assets/js/components/main/Import.js       | 39 +-------------
 2 files changed, 56 insertions(+), 37 deletions(-)
 create mode 100644 rdmo/management/assets/js/components/import/ImportElement.js

diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
new file mode 100644
index 0000000000..22c691b23d
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -0,0 +1,54 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+
+import { CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
+
+import Errors from './common/Errors'
+import Fields from './common/Fields'
+import Form from './common/Form'
+import Warnings from './common/Warnings'
+
+import { codeClass, verboseNames } from '../../constants/elements'
+
+const ImportElement = ({ config, instance, importActions }) => {
+  const showFields = () => importActions.updateElement(instance, {show: !instance.show})
+  const toggleImport = () => importActions.updateElement(instance, {import: !instance.import})
+  const updateInstance = (key, value) => importActions.updateElement(instance, {[key]: value})
+
+  return (
+    <li className="list-group-item">
+      <div className="pull-right">
+        <WarningLink element={instance} onClick={showFields} />
+        <ErrorLink element={instance} onClick={showFields} />
+        <ShowLink element={instance} onClick={showFields} />
+        {
+          instance.updated && !instance.created &&
+          <p className="element-link fa fa-pencil"></p>
+        }
+      </div>
+      <div className="checkbox">
+        <label className="mr-5">
+          <input type="checkbox" checked={instance.import} onChange={toggleImport} />
+          <strong>{verboseNames[instance.model]}{' '}</strong>
+        </label>
+        <CodeLink className={codeClass[instance.model]} uri={instance.uri} onClick={showFields} />
+      </div>
+      {
+        instance.show && <>
+          <Form config={config} element={instance} updateElement={updateInstance} />
+          <Fields element={instance} />
+          <Warnings element={instance} />
+          <Errors element={instance} />
+        </>
+      }
+    </li>
+  )
+}
+
+ImportElement.propTypes = {
+  config: PropTypes.object.isRequired,
+  instance: PropTypes.object.isRequired,
+  importActions: PropTypes.object.isRequired
+}
+
+export default ImportElement
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index ff6bdf971f..c451a21dc6 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -8,17 +8,7 @@ import { getUriPrefixes } from '../../utils/filter'
 import { FilterString, FilterUriPrefix } from '../common/Filter'
 import { Checkbox } from '../common/Checkboxes'
 
-import ImportAttribute from '../import/ImportAttribute'
-import ImportCatalog from '../import/ImportCatalog'
-import ImportCondition from '../import/ImportCondition'
-import ImportOption from '../import/ImportOption'
-import ImportOptionSet from '../import/ImportOptionSet'
-import ImportPage from '../import/ImportPage'
-import ImportQuestion from '../import/ImportQuestion'
-import ImportQuestionSet from '../import/ImportQuestionSet'
-import ImportSection from '../import/ImportSection'
-import ImportTask from '../import/ImportTask'
-import ImportView from '../import/ImportView'
+import ImportElement from '../import/ImportElement'
 
 import { codeClass, verboseNames } from '../../constants/elements'
 
@@ -93,32 +83,7 @@ const Import = ({ config, imports, importActions }) => {
               </li>
             )
           } else {
-            switch (element.model) {
-              case 'questions.catalog':
-                return <ImportCatalog key={index} config={config} catalog={element} importActions={importActions} />
-              case 'questions.section':
-                return <ImportSection key={index} config={config} section={element} importActions={importActions} />
-              case 'questions.page':
-                return <ImportPage key={index} config={config} page={element} importActions={importActions} />
-              case 'questions.questionset':
-                return <ImportQuestionSet key={index} config={config} questionset={element} importActions={importActions} />
-              case 'questions.question':
-                return <ImportQuestion key={index} config={config} question={element} importActions={importActions} />
-              case 'domain.attribute':
-                return <ImportAttribute key={index} config={config} attribute={element} importActions={importActions} />
-              case 'options.optionset':
-                return <ImportOptionSet key={index} config={config} optionset={element} importActions={importActions} />
-              case 'options.option':
-                return <ImportOption key={index} config={config} option={element} importActions={importActions} />
-              case 'conditions.condition':
-                return <ImportCondition key={index} config={config} condition={element} importActions={importActions} />
-              case 'tasks.task':
-                return <ImportTask key={index} config={config} task={element} importActions={importActions} />
-              case 'views.view':
-                return <ImportView key={index} config={config} view={element} importActions={importActions} />
-              default:
-                return null
-            }
+              return <ImportElement key={index} config={config} instance={element} importActions={importActions} />
           }
         })
       }

From e85df4d8d712c4c846fdbd8f499a0d38a072af68 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 13 Nov 2023 21:27:24 +0100
Subject: [PATCH 022/205] chore: rename element to instance, add type check for
 str

---
 .../components/import/common/FieldsDiffs.js   | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index a54f870611..d5ff25f9f7 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -5,16 +5,19 @@ import ReactDiffViewer from 'react-diff-viewer-continued'
 import { isUndefined } from 'lodash'
 
 
-const FieldsDiffs = ({ element, field }) => {
-  return !isUndefined(element) &&
-          !isUndefined(element[field]) &&
-          !isEmpty(element.original) &&
-          !isEmpty(element.original[field]) &&
-          isEmpty(element.errors) &&
+const FieldsDiffs = ({ instance, field }) => {
+
+  return !isUndefined(instance) &&
+          !isUndefined(instance[field]) &&
+          !isEmpty(instance.original) &&
+          !isEmpty(instance.original[field]) &&
+          isEmpty(instance.errors) &&
+          typeof(instance[field]) === 'string' &&
+          typeof(instance.original[field]) === 'string' &&
    <div className="col-sm-12">
     <ReactDiffViewer
-        oldValue={element.original[field]}
-        newValue={element[field]}
+        oldValue={instance.original[field]}
+        newValue={instance[field]}
         splitView={true}
         hideLineNumbers={true}
         leftTitle={gettext('Current')}
@@ -25,7 +28,7 @@ const FieldsDiffs = ({ element, field }) => {
 }
 
 FieldsDiffs.propTypes = {
-  element: PropTypes.object.isRequired,
+  instance: PropTypes.object.isRequired,
   field: PropTypes.string.isRequired,
 }
 

From d1557cd374e5bfa96e4ddf492fa45ffb56453b54 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 14 Nov 2023 01:11:24 +0100
Subject: [PATCH 023/205] chore: fix logging message string format

---
 rdmo/core/imports.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 0c9632bc86..50dd94ce1e 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -65,11 +65,10 @@ def get_or_return_instance(model: models.Model, uri: Optional[str]=None) -> Tupl
 
 def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=None):
     if uri is None:
-        return "%s, no uri", verbose_name
+        return "%s, no uri" % verbose_name
     if created:
-        return "%s created with %s", verbose_name, uri
-
-    return "%s %s updated", verbose_name, uri
+        return f"{verbose_name} created with {uri}"
+    return f"{verbose_name} {uri} updated"
 
 @dataclass
 class ElementImportHelper:

From 574b58a09090be7d4c3c5f2c605e060977e74629 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 21 Nov 2023 18:10:43 +0100
Subject: [PATCH 024/205] mv common import methods to management

---
 rdmo/core/imports.py | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 50dd94ce1e..4ec38e9e49 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -1,4 +1,3 @@
-import copy
 import logging
 import tempfile
 import time
@@ -10,7 +9,6 @@
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
-from django.utils.module_loading import import_string as django_import_string
 
 from rest_framework.utils import model_meta
 
@@ -73,27 +71,12 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
 @dataclass
 class ElementImportHelper:
     model: str
-    dotted_path: str
     import_method: Callable
     validators: Iterable[Callable]
     lang_fields: Sequence[str] = field(default_factory=list)
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
 
 
-def common_import_methods(model_dotted_path: str, uri: Optional[str] = None):
-
-    model = django_import_string(model_dotted_path)
-    instance, _created = get_or_return_instance(model, uri=uri)
-
-    # for when the element is updated
-    # keep a dict of the original
-    # needs to be created here, else the changes will be overwritten
-    original = copy.deepcopy(instance) if not _created else None
-
-    _msg = make_import_info_msg(model._meta.verbose_name, _created, uri=uri)
-    return instance, _msg, _created, original
-
-
 def get_lang_field_values(field_name: str,
                             element: Optional[dict] = None,
                             instance: models.Model = None,

From ac2f0197d47b3f07cfbca240bddc5ee5090ead95 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 21 Nov 2023 18:11:17 +0100
Subject: [PATCH 025/205] remove dotted_path from helpers

---
 rdmo/conditions/imports.py | 1 -
 rdmo/domain/imports.py     | 4 ++--
 rdmo/options/imports.py    | 2 --
 rdmo/questions/imports.py  | 9 +++------
 rdmo/tasks/imports.py      | 1 -
 rdmo/views/imports.py      | 1 -
 6 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index 2be876b7d2..1adacb7283 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -45,7 +45,6 @@ def import_condition(
 
 import_helper_condition = ElementImportHelper(
     model="conditions.condition",
-    dotted_path='rdmo.conditions.models.Condition',
     import_method=import_condition,
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
 )
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 0854b57fd0..94e3306336 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -2,6 +2,7 @@
 from typing import Callable, Tuple
 
 from django.contrib.sites.models import Site
+from django.db import models
 
 from rdmo.core.imports import (
     ElementImportHelper,
@@ -14,7 +15,7 @@
 logger = logging.getLogger(__name__)
 
 
-def import_attribute(instance, element, validators: Tuple[Callable], save=False, user=None):
+def import_attribute(instance: models.Model, element: dict, validators: Tuple[Callable], save=False, user=None):
 
     set_foreign_field(instance, 'parent', element)
 
@@ -36,7 +37,6 @@ def import_attribute(instance, element, validators: Tuple[Callable], save=False,
 
 import_helper_attribute = ElementImportHelper(
     model="domain.attribute",
-    dotted_path='rdmo.domain.models.Attribute',
     import_method=import_attribute,
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
 )
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index ad8bc17554..bcffa21fdc 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -51,7 +51,6 @@ def import_option(
 
 import_helper_option = ElementImportHelper(
     model="options.option",
-    dotted_path="rdmo.options.models.Option",
     import_method=import_option,
     validators=(OptionLockedValidator, OptionUniqueURIValidator),
     lang_fields=('text',)
@@ -89,7 +88,6 @@ def import_optionset(
 
 import_helper_optionset = ElementImportHelper(
     model="options.optionset",
-    dotted_path="rdmo.options.models.OptionSet",
     import_method=import_optionset,
     validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
 )
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 92aed772f8..13b0e6681e 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -100,7 +100,7 @@ def import_page(
     set_lang_field(instance, 'help', element)
     set_lang_field(instance, 'verbose_name', element)
 
-    validate_instance(instance, element,*validators)
+    validate_instance(instance, element, *validators)
 
     check_permissions(instance, element, user)
 
@@ -207,7 +207,6 @@ def import_question(
 
 import_helper_catalog = ElementImportHelper(
     model="questions.catalog",
-    dotted_path="rdmo.questions.models.catalog.Catalog",
     import_method=import_catalog,
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
     lang_fields=('title', 'help')
@@ -215,14 +214,13 @@ def import_question(
 
 import_helper_section = ElementImportHelper(
     model="questions.section",
-    dotted_path="rdmo.questions.models.section.Section",
     import_method=import_section,
     validators=(SectionLockedValidator, SectionUniqueURIValidator),
     lang_fields=('title',)
 )
+
 import_helper_page = ElementImportHelper(
     model="questions.page",
-    dotted_path="rdmo.questions.models.page.Page",
     import_method=import_page,
     validators= (PageLockedValidator, PageUniqueURIValidator),
     lang_fields=('title', 'help', 'verbose_name', 'verbose_name_plural')
@@ -230,14 +228,13 @@ def import_question(
 
 import_helper_questionset = ElementImportHelper(
     model="questions.questionset",
-    dotted_path="rdmo.questions.models.questionset.QuestionSet",
     import_method=import_questionset,
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     lang_fields=('title', 'help', 'verbose_name', 'verbose_name_plural')
 )
+
 import_helper_question = ElementImportHelper(
     model="questions.question",
-    dotted_path="rdmo.questions.models.question.Question",
     import_method=import_question,
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
     lang_fields=('text', 'help', 'default_text', 'verbose_name', 'verbose_name_plural')
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index aef1132fa6..7e926c0761 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -56,7 +56,6 @@ def import_task(
 
 import_helper_task = ElementImportHelper(
     model="tasks.task",
-    dotted_path="rdmo.tasks.models.Task",
     import_method=import_task,
     validators=(TaskLockedValidator, TaskUniqueURIValidator),
     lang_fields=('title', 'text')
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 06bae45def..6a5ef694b4 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -46,7 +46,6 @@ def import_view(
 
 import_helper_view = ElementImportHelper(
     model="views.view",
-    dotted_path="rdmo.views.models.View",
     import_method=import_view,
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
     lang_fields=('title', 'help')

From 937463d00fa63820fb1dc1208655b21c1ee85de1 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 21 Nov 2023 18:13:24 +0100
Subject: [PATCH 026/205] move rdmo model mapping to constans

---
 rdmo/management/constants.py | 21 +++++++++++++++++++++
 rdmo/management/viewsets.py  | 22 +++-------------------
 2 files changed, 24 insertions(+), 19 deletions(-)
 create mode 100644 rdmo/management/constants.py

diff --git a/rdmo/management/constants.py b/rdmo/management/constants.py
new file mode 100644
index 0000000000..4c1ddc78a9
--- /dev/null
+++ b/rdmo/management/constants.py
@@ -0,0 +1,21 @@
+
+from rdmo.conditions.models import Condition
+from rdmo.domain.models import Attribute
+from rdmo.options.models import Option, OptionSet
+from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
+from rdmo.tasks.models import Task
+from rdmo.views.models import View
+
+RDMO_MODEL_PATH_MAPPER = {
+    'conditions.condition': Condition,
+    'domain.attribute': Attribute,
+    'options.optionset': OptionSet,
+    'options.option': Option,
+    'questions.catalog': Catalog,
+    'questions.section': Section,
+    'questions.page': Page,
+    'questions.questionset': QuestionSet,
+    'questions.question': Question,
+    'tasks.task': Task,
+    'views.view': View
+    }
diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index 3ec14f3b25..1cbae314ec 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -9,40 +9,24 @@
 from rest_framework.response import Response
 from rest_framework.serializers import ValidationError
 
-from rdmo.conditions.models import Condition
 from rdmo.core.imports import handle_uploaded_file
 from rdmo.core.permissions import CanToggleElementCurrentSite
 from rdmo.core.utils import get_model_field_meta, is_truthy
 from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
-from rdmo.domain.models import Attribute
-from rdmo.options.models import Option, OptionSet
-from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
-from rdmo.tasks.models import Task
-from rdmo.views.models import View
 
+from .constants import RDMO_MODEL_PATH_MAPPER
 from .imports import import_elements
 
 logger = logging.getLogger(__name__)
 
 
+
 class MetaViewSet(viewsets.ViewSet):
 
     permission_classes = (IsAuthenticated, )
 
     def list(self, request, *args, **kwargs):
-        return Response({
-            'conditions.condition': get_model_field_meta(Condition),
-            'domain.attribute': get_model_field_meta(Attribute),
-            'options.optionset': get_model_field_meta(OptionSet),
-            'options.option': get_model_field_meta(Option),
-            'questions.catalog': get_model_field_meta(Catalog),
-            'questions.section': get_model_field_meta(Section),
-            'questions.page': get_model_field_meta(Page),
-            'questions.questionset': get_model_field_meta(QuestionSet),
-            'questions.question': get_model_field_meta(Question),
-            'tasks.task': get_model_field_meta(Task),
-            'views.view': get_model_field_meta(View)
-        })
+        return Response({k: get_model_field_meta(val) for k, val in RDMO_MODEL_PATH_MAPPER})
 
 
 class UploadViewSet(viewsets.ViewSet):

From 3e8f802f189e5258fdc495768b572cd0a682156e Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 21 Nov 2023 18:14:46 +0100
Subject: [PATCH 027/205] move common methods into import_element

---
 rdmo/management/imports.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 0d9e7ecd29..731cd9cd1d 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -1,11 +1,13 @@
+import copy
 import logging
 from collections import defaultdict
 from typing import Optional
 
+from django.db import models
 from django.forms.models import model_to_dict
 
 from rdmo.conditions.imports import import_helper_condition
-from rdmo.core.imports import common_import_methods, get_lang_field_values, set_lang_field
+from rdmo.core.imports import get_lang_field_values, get_or_return_instance, make_import_info_msg, set_lang_field
 from rdmo.domain.imports import import_helper_attribute
 from rdmo.options.imports import import_helper_option, import_helper_optionset
 from rdmo.questions.imports import (
@@ -18,6 +20,8 @@
 from rdmo.tasks.imports import import_helper_task
 from rdmo.views.imports import import_helper_view
 
+from .constants import RDMO_MODEL_PATH_MAPPER
+
 logger = logging.getLogger(__name__)
 
 
@@ -44,7 +48,7 @@
     }
 
 
-def import_elements(uploaded_elements, save=True, user=None):
+def import_elements(uploaded_elements: list[dict], save: bool = True, user: Optional[models.Model] = None):
     imported_elements = []
     for element in uploaded_elements:
         model = element.get('model')
@@ -56,7 +60,7 @@ def import_elements(uploaded_elements, save=True, user=None):
     return imported_elements
 
 
-def filter_warnings(element, elements):
+def filter_warnings(element: dict, elements: list[dict]):
     # remove warnings regarding elements which are in the elements list
     warnings = []
     for uri, messages in element['warnings'].items():
@@ -71,7 +75,7 @@ def import_element(
         model_path: Optional[str] = None,
         element: Optional[dict] = None,
         save: bool = True,
-        user = None
+        user: Optional[models.Model] = None
     ):
 
     if element is None:
@@ -83,17 +87,23 @@ def import_element(
     if model_path is None:
         return element
 
+    model = RDMO_MODEL_PATH_MAPPER[model_path]
     import_helper = ELEMENT_IMPORT_HELPERS[model_path]
     import_method = import_helper.import_method
-    model_path = import_helper.dotted_path
     validators = import_helper.validators
     lang_field_names = import_helper.lang_fields
     uri = element.get('uri')
 
-    instance, _msg, _created, original_instance = common_import_methods(
-                            model_path,
-                            uri=uri
-                    )
+    # get or create instance from uri and model_path
+    instance, _created = get_or_return_instance(model, uri=uri)
+
+    # keep a copy of the original
+    # when the element is updated
+    # needs to be created here, else the changes will be overwritten
+    original_instance = copy.deepcopy(instance) if not _created else None
+
+    _msg = make_import_info_msg(model._meta.verbose_name, _created, uri=uri)
+
     # prepare original element when updated (maybe rename into lookup)
     _updated = not _created
     original_element = {}
@@ -119,7 +129,7 @@ def import_element(
 
     if _updated:
         element['updated'] = _updated
-        # make json serializable, keep only strings
+        # keep only strings, make json serializable
         original_element_json = {k: val for k, val in original_element.items() if isinstance(val, str)}
         element['original'] = original_element_json
 

From 06d9d122bd32d7e4ed4a0864a640bde469e7a3a0 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 22 Nov 2023 10:40:25 +0100
Subject: [PATCH 028/205] chore: fix filter warnings and typing

---
 rdmo/management/imports.py | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 731cd9cd1d..ae813a0d6b 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -1,7 +1,7 @@
 import copy
 import logging
 from collections import defaultdict
-from typing import Optional
+from typing import Dict, List, Optional
 
 from django.db import models
 from django.forms.models import model_to_dict
@@ -48,32 +48,33 @@
     }
 
 
-def import_elements(uploaded_elements: list[dict], save: bool = True, user: Optional[models.Model] = None):
+def import_elements(uploaded_elements: List[Dict], save: bool = True, user: Optional[models.Model] = None):
     imported_elements = []
+    uploaded_uris = {i.get('uri') for i in uploaded_elements}
     for element in uploaded_elements:
-        model = element.get('model')
-        if model is None:
-            continue
-        element = import_element(model_path=model, element=element, save=save, user=user)
-        element = filter_warnings(element, uploaded_elements)
+        model_path = element.pop('model')
+        element = import_element(model_path=model_path, element=element, save=save, user=user)
+        # replace warnings with filtered list of warnings
+        warnings = element.pop('warnings')
+        element['warnings'] = filter_warnings(warnings, uploaded_uris)
         imported_elements.append(element)
     return imported_elements
 
 
-def filter_warnings(element: dict, elements: list[dict]):
+def filter_warnings(warnings: Dict, uploaded_uris: List[Dict]) -> List[str]:
     # remove warnings regarding elements which are in the elements list
-    warnings = []
-    for uri, messages in element['warnings'].items():
-        if not next(filter(lambda e: e['uri'] == uri, elements), None):
-            warnings += messages
-
-    element['warnings'] = warnings
-    return element
+    ret = []
+    if not warnings:
+        return ret
+    for uri, messages in warnings.items():
+        if uri not in uploaded_uris:
+            ret += messages
+    return ret
 
 
 def import_element(
         model_path: Optional[str] = None,
-        element: Optional[dict] = None,
+        element: Optional[Dict] = None,
         save: bool = True,
         user: Optional[models.Model] = None
     ):

From b5548ce1f5762557c9f601e4b4e2ca49987f0d2e Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 22 Nov 2023 10:41:08 +0100
Subject: [PATCH 029/205] chore: fix typo

---
 rdmo/management/viewsets.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index 1cbae314ec..4f5757b7b4 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -26,7 +26,7 @@ class MetaViewSet(viewsets.ViewSet):
     permission_classes = (IsAuthenticated, )
 
     def list(self, request, *args, **kwargs):
-        return Response({k: get_model_field_meta(val) for k, val in RDMO_MODEL_PATH_MAPPER})
+        return Response({k: get_model_field_meta(val) for k, val in RDMO_MODEL_PATH_MAPPER.items()})
 
 
 class UploadViewSet(viewsets.ViewSet):

From 1ee3d054c1cd606b2e25a567a96514b7443c4903 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 11 Jan 2024 09:39:21 +0100
Subject: [PATCH 030/205] chore: fix lang fields for imports

---
 rdmo/conditions/imports.py |  1 +
 rdmo/domain/imports.py     |  1 +
 rdmo/management/imports.py |  4 +++-
 rdmo/options/imports.py    |  7 +++----
 rdmo/questions/imports.py  | 26 +++++++-------------------
 rdmo/tasks/imports.py      |  5 +----
 rdmo/views/imports.py      |  2 +-
 7 files changed, 17 insertions(+), 29 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index 1adacb7283..b643b8fd75 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -47,4 +47,5 @@ def import_condition(
     model="conditions.condition",
     import_method=import_condition,
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
+    lang_fields=None
 )
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 94e3306336..91d00468d8 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -39,4 +39,5 @@ def import_attribute(instance: models.Model, element: dict, validators: Tuple[Ca
     model="domain.attribute",
     import_method=import_attribute,
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
+    lang_fields=None
 )
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index ae813a0d6b..b86245d089 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -92,7 +92,7 @@ def import_element(
     import_helper = ELEMENT_IMPORT_HELPERS[model_path]
     import_method = import_helper.import_method
     validators = import_helper.validators
-    lang_field_names = import_helper.lang_fields
+    lang_field_names = import_helper.lang_fields if import_helper.lang_fields is not None else []
     uri = element.get('uri')
 
     # get or create instance from uri and model_path
@@ -103,6 +103,7 @@ def import_element(
     # needs to be created here, else the changes will be overwritten
     original_instance = copy.deepcopy(instance) if not _created else None
 
+    # prepare a log message
     _msg = make_import_info_msg(model._meta.verbose_name, _created, uri=uri)
 
     # prepare original element when updated (maybe rename into lookup)
@@ -112,6 +113,7 @@ def import_element(
         original_element = model_to_dict(original_instance)
         original_element = {k: original_element.get(k, element.get(k))
                             for k in element.keys() if k not in IMPORT_ELEMENT_INIT_DICT}
+    # start to set values on the instance
     # set common field values from element on instance
     for common_field in import_helper.common_fields:
         setattr(instance, common_field, element.get(common_field) or '')
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index bcffa21fdc..7d2f3bd196 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -65,11 +65,9 @@ def import_optionset(
         user: models.Model = None
     ):
 
-    instance.additional_input = element.get('additional_input') or ""
+    # lang_fields are already set in management/import.py
 
-    set_lang_field(instance, 'text', element)
-    set_lang_field(instance, 'help', element)
-    set_lang_field(instance, 'view_text', element)
+    instance.additional_input = element.get('additional_input') or ""
 
     validate_instance(instance, element, *validators)
 
@@ -90,4 +88,5 @@ def import_optionset(
     model="options.optionset",
     import_method=import_optionset,
     validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
+    lang_fields=None
 )
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 13b0e6681e..21d3843bd1 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -90,16 +90,12 @@ def import_page(
         save: bool = False,
         user: models.Model = None
     ):
+    # lang_fields are already set in management/import.py
 
     set_foreign_field(instance, 'attribute', element)
 
     instance.is_collection = element.get('is_collection') or False
 
-    set_lang_field(instance, 'title', element)
-    set_lang_field(page, 'short_title', element)
-    set_lang_field(instance, 'help', element)
-    set_lang_field(instance, 'verbose_name', element)
-
     validate_instance(instance, element, *validators)
 
     check_permissions(instance, element, user)
@@ -125,15 +121,11 @@ def import_questionset(
         save: bool = False,
         user: models.Model = None
     ):
-
+    # lang_fields are already set in management/import.py
     set_foreign_field(instance, 'attribute', element)
 
     instance.is_collection = element.get('is_collection') or False
 
-    set_lang_field(questionset, 'title', element)
-    set_lang_field(questionset, 'help', element)
-    set_lang_field(questionset, 'verbose_name', element)
-
     validate_instance(instance, element, *validators)
 
     check_permissions(instance, element, user)
@@ -160,16 +152,12 @@ def import_question(
         save: bool = False,
         user: models.Model = None
     ):
-
+    # lang_fields are already set in management/import.py
     set_foreign_field(instance, 'attribute', element)
 
     instance.is_collection = element.get('is_collection') or False
     instance.is_optional = element.get('is_optional') or False
 
-    set_lang_field(instance, 'text', element)
-    set_lang_field(instance, 'help', element)
-    set_lang_field(instance, 'default_text', element)
-    set_lang_field(instance, 'verbose_name', element)
 
     set_foreign_field(instance, 'default_option', element)
 
@@ -209,7 +197,7 @@ def import_question(
     model="questions.catalog",
     import_method=import_catalog,
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
-    lang_fields=('title', 'help')
+    lang_fields=('help', 'title')
 )
 
 import_helper_section = ElementImportHelper(
@@ -223,19 +211,19 @@ def import_question(
     model="questions.page",
     import_method=import_page,
     validators= (PageLockedValidator, PageUniqueURIValidator),
-    lang_fields=('title', 'help', 'verbose_name', 'verbose_name_plural')
+    lang_fields=('help', 'title', 'verbose_name')
 )
 
 import_helper_questionset = ElementImportHelper(
     model="questions.questionset",
     import_method=import_questionset,
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
-    lang_fields=('title', 'help', 'verbose_name', 'verbose_name_plural')
+    lang_fields=('help', 'title', 'verbose_name')
 )
 
 import_helper_question = ElementImportHelper(
     model="questions.question",
     import_method=import_question,
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
-    lang_fields=('text', 'help', 'default_text', 'verbose_name', 'verbose_name_plural')
+    lang_fields=('text', 'help', 'default_text', 'verbose_name')
 )
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 7e926c0761..702250a696 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -23,12 +23,9 @@ def import_task(
         save: bool = False,
         user: models.Model = None
     ):
-
+    # lang_fields are already set in management/import.py
     instance.order = element.get('order') or 0
 
-    set_lang_field(instance, 'title', element)
-    set_lang_field(instance, 'text', element)
-
     set_foreign_field(instance, 'start_attribute', element)
     set_foreign_field(instance, 'end_attribute', element)
 
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 6a5ef694b4..43474e355f 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -48,5 +48,5 @@ def import_view(
     model="views.view",
     import_method=import_view,
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
-    lang_fields=('title', 'help')
+    lang_fields=( 'help', 'title')
 )

From 0ee3e7978b02b98db7a8c96b14b8b976fac667a5 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 11 Jan 2024 10:55:29 +0100
Subject: [PATCH 031/205] chore: refactor check permissions for import

---
 rdmo/core/imports.py       |  9 +++------
 rdmo/management/imports.py | 14 +++++++++++++-
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 4ec38e9e49..61978cc936 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -288,7 +288,7 @@ def validate_instance(instance, element, *validators):
             element['errors'].append(message)
 
 
-def check_permissions(instance, element, user):
+def check_permissions(instance: models.Model, element_uri: str, user: models.Model) -> Optional[str]:
     if user is None:
         return
 
@@ -301,9 +301,6 @@ def check_permissions(instance, element, user):
         perms = [f'{app_label}.add_{model_name}_object']
 
     if not user.has_perms(perms, instance):
-        message = 'You have no permissions to import {instance_model} {instance_uri}.'.format(
-            instance_model=instance._meta.object_name,
-            instance_uri=element.get('uri')
-        )
+        message = f'You have no permissions to import {instance._meta.object_name} {element_uri}.'
         logger.info(message)
-        element['errors'].append(message)
+        return message
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index b86245d089..bf2f39cec3 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -7,7 +7,13 @@
 from django.forms.models import model_to_dict
 
 from rdmo.conditions.imports import import_helper_condition
-from rdmo.core.imports import get_lang_field_values, get_or_return_instance, make_import_info_msg, set_lang_field
+from rdmo.core.imports import (
+    check_permissions,
+    get_lang_field_values,
+    get_or_return_instance,
+    make_import_info_msg,
+    set_lang_field,
+)
 from rdmo.domain.imports import import_helper_attribute
 from rdmo.options.imports import import_helper_option, import_helper_optionset
 from rdmo.questions.imports import (
@@ -106,6 +112,12 @@ def import_element(
     # prepare a log message
     _msg = make_import_info_msg(model._meta.verbose_name, _created, uri=uri)
 
+    # check the change or add permissions for the user on the instance
+    _perms_error_msg = check_permissions(instance, uri, user)
+    if _perms_error_msg:
+        # when there is an error msg, the import could be stopped and return
+        element["errors"].append(_perms_error_msg)
+
     # prepare original element when updated (maybe rename into lookup)
     _updated = not _created
     original_element = {}

From fc0765945de8314031f405f6d8041e7dcd4127ae Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 11 Jan 2024 10:59:11 +0100
Subject: [PATCH 032/205] refactor: add model type to import instance arg

---
 rdmo/conditions/imports.py |  6 ++++--
 rdmo/domain/imports.py     |  5 +++--
 rdmo/options/imports.py    |  6 ++++--
 rdmo/questions/imports.py  | 15 ++++++++++-----
 rdmo/tasks/imports.py      |  4 +++-
 rdmo/views/imports.py      |  4 +++-
 6 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index b643b8fd75..36e5d3d966 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -12,11 +12,13 @@
     validate_instance,
 )
 
+from .models import Condition
+
 logger = logging.getLogger(__name__)
 
 
 def import_condition(
-        instance: models.Model,
+        instance: Condition,
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
@@ -26,7 +28,7 @@ def import_condition(
     set_foreign_field(instance, 'source', element)
     set_foreign_field(instance, 'target_option', element)
 
-    instance.relation = element.get('relation')
+    instance.relation = element.get('relation') or ''
     instance.target_text = element.get('target_text') or ''
 
     validate_instance(instance, element, *validators)
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 91d00468d8..366d604af8 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -2,7 +2,6 @@
 from typing import Callable, Tuple
 
 from django.contrib.sites.models import Site
-from django.db import models
 
 from rdmo.core.imports import (
     ElementImportHelper,
@@ -12,10 +11,12 @@
 )
 from rdmo.domain.validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
 
+from .models import Attribute
+
 logger = logging.getLogger(__name__)
 
 
-def import_attribute(instance: models.Model, element: dict, validators: Tuple[Callable], save=False, user=None):
+def import_attribute(instance: Attribute, element: dict, validators: Tuple[Callable], save=False, user=None):
 
     set_foreign_field(instance, 'parent', element)
 
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 7d2f3bd196..053119c3d5 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -19,11 +19,13 @@
     OptionUniqueURIValidator,
 )
 
+from .models import Option, OptionSet
+
 logger = logging.getLogger(__name__)
 
 
 def import_option(
-        instance: models.Model,
+        instance: Option,
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
@@ -58,7 +60,7 @@ def import_option(
 
 
 def import_optionset(
-        instance: models.Model,
+        instance: OptionSet,
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 21d3843bd1..6473e64c6c 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -26,13 +26,18 @@
     SectionUniqueURIValidator,
 )
 
+from .models.catalog import Catalog
+from .models.page import Page
+from .models.question import Question
+from .models.questionset import QuestionSet
+from .models.section import Section
 from .utils import get_widget_types
 
 logger = logging.getLogger(__name__)
 
 
 def import_catalog(
-        instance: models.Model,
+        instance: Catalog,
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
@@ -60,7 +65,7 @@ def import_catalog(
 
 
 def import_section(
-        instance: models.Model,
+        instance: Section,
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
@@ -84,7 +89,7 @@ def import_section(
 
 
 def import_page(
-        instance: models.Model,
+        instance: Page,
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
@@ -115,7 +120,7 @@ def import_page(
 
 
 def import_questionset(
-        instance: models.Model,
+        instance: QuestionSet,
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
@@ -146,7 +151,7 @@ def import_questionset(
 
 
 def import_question(
-        instance: models.Model,
+        instance: Question,
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 702250a696..5d50ab26cd 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -13,11 +13,13 @@
 )
 from rdmo.tasks.validators import TaskLockedValidator, TaskUniqueURIValidator
 
+from .models import Task
+
 logger = logging.getLogger(__name__)
 
 
 def import_task(
-        instance: models.Model,
+        instance: Task,
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 43474e355f..01c0991198 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -12,11 +12,13 @@
 )
 from rdmo.views.validators import ViewLockedValidator, ViewUniqueURIValidator
 
+from .models import View
+
 logger = logging.getLogger(__name__)
 
 
 def import_view(
-        instance: models.Model,
+        instance: View,
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,

From cddd21084c509fd0b2835d3b12aa9a847e3fa928 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 11 Jan 2024 11:31:29 +0100
Subject: [PATCH 033/205] fix import option, add additional_input

---
 rdmo/options/imports.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 053119c3d5..9d81a6f704 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -34,6 +34,7 @@ def import_option(
 
     instance.order = element.get('order') or 0
     instance.provider_key = element.get('provider_key') or ''
+    instance.additional_input = element.get('additional_input') or ""
 
     validate_instance(instance, element, *validators)
 

From 06d9323a80ffea0da9fc9ed2c73bbddc1fcb228f Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 11 Jan 2024 11:37:34 +0100
Subject: [PATCH 034/205] refactor: import options tests

---
 rdmo/management/tests/test_import_options.py | 60 ++++++++++----------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 65dd16a82b..7139e92f85 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -15,14 +15,14 @@ def test_create_optionsets(db, settings):
     elements = flat_xml_to_elements(root)
     elements = convert_elements(elements, version)
     elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    parsed_elements = elements.values()
+    imported_elements = import_elements(parsed_elements)
 
-    assert len(root) == len(elements) == 13
+    assert len(root) == len(elements) == len(imported_elements) == 13
     assert OptionSet.objects.count() == 4
     assert Option.objects.count() == 9
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert all(element['created'] is True for element in parsed_elements)
+    assert all(element['updated'] is False for element in parsed_elements)
 
 
 def test_update_optionsets(db, settings):
@@ -32,12 +32,12 @@ def test_update_optionsets(db, settings):
     elements = flat_xml_to_elements(root)
     elements = convert_elements(elements, version)
     elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    parsed_elements = elements.values()
+    imported_elements = import_elements(parsed_elements)
 
-    assert len(root) == len(elements) == 13
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(elements) == len(imported_elements) == 13
+    assert all(element['created'] is False for element in parsed_elements)
+    assert all(element['updated'] is True for element in parsed_elements)
 
 
 def test_create_options(db, settings):
@@ -49,12 +49,12 @@ def test_create_options(db, settings):
     elements = flat_xml_to_elements(root)
     elements = convert_elements(elements, version)
     elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    parsed_elements = elements.values()
+    imported_elements = import_elements(parsed_elements)
 
-    assert len(root) == len(elements) == Option.objects.count() == 9
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert len(root) == len(elements) == len(imported_elements) == Option.objects.count() == 9
+    assert all(element['created'] is True for element in parsed_elements)
+    assert all(element['updated'] is False for element in parsed_elements)
 
 
 def test_update_options(db, settings):
@@ -64,12 +64,12 @@ def test_update_options(db, settings):
     elements = flat_xml_to_elements(root)
     elements = convert_elements(elements, version)
     elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    parsed_elements = elements.values()
+    imported_elements = import_elements(parsed_elements)
 
-    assert len(root) == len(elements) == 9
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(elements) == len(imported_elements) == 9
+    assert all(element['created'] is False for element in parsed_elements)
+    assert all(element['updated'] is True for element in parsed_elements)
 
 
 def test_create_legacy_options(db, settings):
@@ -83,14 +83,14 @@ def test_create_legacy_options(db, settings):
     elements = flat_xml_to_elements(root)
     elements = convert_elements(elements, version)
     elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    parsed_elements = elements.values()
+    imported_elements = import_elements(parsed_elements)
 
-    assert len(root) == len(elements) == 12
+    assert len(root) == len(elements) == len(imported_elements) == 12
     assert OptionSet.objects.count() == 4
     assert Option.objects.count() == 8
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert all(element['created'] is True for element in parsed_elements)
+    assert all(element['updated'] is False for element in parsed_elements)
 
 
 def test_update_legacy_options(db, settings):
@@ -101,9 +101,9 @@ def test_update_legacy_options(db, settings):
     elements = flat_xml_to_elements(root)
     elements = convert_elements(elements, version)
     elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    parsed_elements = elements.values()
+    imported_elements = import_elements(parsed_elements)
 
-    assert len(root) == len(elements) == 12
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(elements) == len(imported_elements) == 12
+    assert all(element['created'] is False for element in parsed_elements)
+    assert all(element['updated'] is True for element in parsed_elements)

From b4a1104b9df56dee8f386fa1b7885713c3ebbced Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 15 Jan 2024 18:27:07 +0100
Subject: [PATCH 035/205] chore: add isEmpty for updated_and_changed check

---
 rdmo/management/assets/js/components/import/ImportElement.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index 22c691b23d..c670272f44 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -9,6 +9,7 @@ import Form from './common/Form'
 import Warnings from './common/Warnings'
 
 import { codeClass, verboseNames } from '../../constants/elements'
+import { isEmpty } from 'lodash'
 
 const ImportElement = ({ config, instance, importActions }) => {
   const showFields = () => importActions.updateElement(instance, {show: !instance.show})
@@ -22,7 +23,7 @@ const ImportElement = ({ config, instance, importActions }) => {
         <ErrorLink element={instance} onClick={showFields} />
         <ShowLink element={instance} onClick={showFields} />
         {
-          instance.updated && !instance.created &&
+          instance.updated && !isEmpty(instance.updated_and_changed) && !instance.created &&
           <p className="element-link fa fa-pencil"></p>
         }
       </div>

From 32d6ce096a9ab313d3ef360a0ac699268d345b79 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 15 Jan 2024 18:30:22 +0100
Subject: [PATCH 036/205] chore: add component for ImportSuccesElement

---
 .../components/import/ImportSuccessElement.js | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 rdmo/management/assets/js/components/import/ImportSuccessElement.js

diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
new file mode 100644
index 0000000000..20440e7dc9
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -0,0 +1,36 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+import uniqueId from 'lodash/uniqueId'
+
+import { codeClass, verboseNames } from '../../constants/elements'
+import { isEmpty } from 'lodash'
+
+const ImportSuccessElement = ({ instance }) => {
+  return (
+    <li className="list-group-item">
+      <p>
+        <strong>{verboseNames[instance.model]}{' '}</strong>
+        <code className={codeClass[instance.model]}>{instance.uri}</code>
+        {instance.created && <span className="text-success">{' '}{gettext('created')}  </span> && <span className="muted element-link fa fa-plus"></span>}
+        {instance.updated && <span className="text-info">{' '}{gettext('updated')} </span> && <span className="muted element-link fa fa-pencil"></span>}
+        {
+          !isEmpty(instance.errors) && !(instance.created || instance.updated) &&
+          <span className="text-danger">{' '}{gettext('could not be imported')}</span>
+        }
+        {
+          !isEmpty(instance.errors) && (instance.created || instance.updated) &&
+          <>{', '}<span className="text-danger">{gettext('but could not be added to parent element')}</span></>
+        }
+        {'.'}
+      </p>
+      {instance.warnings.map(message => <p key={uniqueId()} className="text-warning">{message}</p>)}
+      {instance.errors.map(message => <p key={uniqueId()} className="text-danger">{message}</p>)}
+    </li>
+  )
+}
+
+ImportSuccessElement.propTypes = {
+  instance: PropTypes.object.isRequired,
+}
+
+export default ImportSuccessElement

From 7a3fa034e6ce84fff2775d010d0cdd9d6a7fd7d3 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 15 Jan 2024 18:48:31 +0100
Subject: [PATCH 037/205] feat: add import/showElements button

---
 rdmo/management/assets/js/actions/importActions.js    |  4 ++++
 .../assets/js/components/sidebar/ImportSidebar.js     | 11 +++++++++++
 rdmo/management/assets/js/reducers/importsReducer.js  |  4 ++++
 3 files changed, 19 insertions(+)

diff --git a/rdmo/management/assets/js/actions/importActions.js b/rdmo/management/assets/js/actions/importActions.js
index fa88c51870..7abb1d1bdb 100644
--- a/rdmo/management/assets/js/actions/importActions.js
+++ b/rdmo/management/assets/js/actions/importActions.js
@@ -66,6 +66,10 @@ export function selectElements(value) {
   return {type: 'import/selectElements', value}
 }
 
+export function showElements(value) {
+  return {type: 'import/showElements', value}
+}
+
 export function updateUriPrefix(uriPrefix) {
   return {type: 'import/updateUriPrefix', uriPrefix}
 }
diff --git a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
index 9d27019d28..c1cecb8b40 100644
--- a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
+++ b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
@@ -56,6 +56,17 @@ const ImportSidebar = ({ config, imports, importActions }) => {
               {gettext('Unselect all')}
             </Link>
           </li>
+          <br></br>
+          <li>
+            <Link onClick={() => importActions.showElements(true)}>
+              {gettext('Show all')}
+            </Link>
+          </li>
+          <li>
+            <Link onClick={() => importActions.showElements(false)}>
+              {gettext('Hide all')}
+            </Link>
+          </li>
         </ul>
 
         <h2>{gettext('URI prefix')}</h2>
diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index 5b6d115fa6..909af7a103 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -53,6 +53,10 @@ export default function importsReducer(state = initialState, action) {
       return {...state, elements: state.elements.map(element => {
         return {...element, import: action.value}
       })}
+    case 'import/showElements':
+      return {...state, elements: state.elements.map(element => {
+        return {...element, show: action.value}
+      })}
     case 'import/updateUriPrefix':
       elements = state.elements.map(element => {
         element.uri_prefix = action.uriPrefix

From 1346840d672492f35572418288270dce2a6dc842 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 15 Jan 2024 18:50:40 +0100
Subject: [PATCH 038/205] chore: rename fields to element

---
 .../components/import/common/FieldsDiffs.js   | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index d5ff25f9f7..882a03c74e 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -5,30 +5,30 @@ import ReactDiffViewer from 'react-diff-viewer-continued'
 import { isUndefined } from 'lodash'
 
 
-const FieldsDiffs = ({ instance, field }) => {
+const FieldsDiffs = ({ element, field }) => {
 
-  return !isUndefined(instance) &&
-          !isUndefined(instance[field]) &&
-          !isEmpty(instance.original) &&
-          !isEmpty(instance.original[field]) &&
-          isEmpty(instance.errors) &&
-          typeof(instance[field]) === 'string' &&
-          typeof(instance.original[field]) === 'string' &&
+  return !isUndefined(element) &&
+          !isUndefined(element[field]) &&
+          !isEmpty(element.original) &&
+          !isEmpty(element.original[field]) &&
+          isEmpty(element.errors) &&
+          typeof(element[field]) === 'string' &&
+          typeof(element.original[field]) === 'string' &&
    <div className="col-sm-12">
     <ReactDiffViewer
-        oldValue={instance.original[field]}
-        newValue={instance[field]}
+        oldValue={element.original[field]}
+        newValue={element[field]}
         splitView={true}
         hideLineNumbers={true}
         leftTitle={gettext('Current')}
-        rightTitle={gettext('Imported')}
+        rightTitle={gettext('Uploaded')}
         >
         </ReactDiffViewer>
     </div>
 }
 
 FieldsDiffs.propTypes = {
-  instance: PropTypes.object.isRequired,
+  element: PropTypes.object.isRequired,
   field: PropTypes.string.isRequired,
 }
 

From 0b07aead7d4def8c4edbb8b06c14e5bc358d32be Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 15 Jan 2024 18:50:58 +0100
Subject: [PATCH 039/205] chore: add updated_and_changed to excludeKeys

---
 rdmo/management/assets/js/components/import/common/Fields.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index ba8bf0f8b7..f0147d4910 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -25,7 +25,8 @@ const excludeKeys = [
   'uri_prefix',
   'valid',
   'warnings',
-  'original'
+  'original',
+  'updated_and_changed',
 ]
 
 const Fields = ({ element }) => {

From a296380e67e3caa1b3b6d2a8b153bb1c3625707c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 15 Jan 2024 18:51:23 +0100
Subject: [PATCH 040/205] chore: update Import component

---
 .../assets/js/components/main/Import.js       | 56 ++++++-------------
 1 file changed, 18 insertions(+), 38 deletions(-)

diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index c451a21dc6..5b53ae903c 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -1,34 +1,39 @@
 import React from 'react'
 import PropTypes from 'prop-types'
-import uniqueId from 'lodash/uniqueId'
 import isEmpty from 'lodash/isEmpty'
 import get from 'lodash/get'
 
 import { getUriPrefixes } from '../../utils/filter'
 import { FilterString, FilterUriPrefix } from '../common/Filter'
-import { Checkbox } from '../common/Checkboxes'
 
 import ImportElement from '../import/ImportElement'
-
-import { codeClass, verboseNames } from '../../constants/elements'
+import ImportSuccesElement from '../import/ImportSuccessElement'
 
 const Import = ({ config, imports, importActions }) => {
-  const { elements, success } = imports
+  const { filename, elements, success } = imports
   const updateFilterString = (value) => importActions.updateConfig('filter.import.elements.search', value)
   const updateFilterUriPrefix = (value) => importActions.updateConfig('filter.import.elements.uri_prefix', value)
-  const updateDisplayCatalogURI = (value) => importActions.updateConfig('display.uri.catalogs', value)
   const updatedElements = elements.filter(element => element.updated)
   const createdElements = elements.filter(element => element.created)
+  const updatedAndChangedElements = elements.filter(element => element.updated && !isEmpty(element.updated_and_changed))
+  const updatedAndSameElements = elements.filter(element => element.updated && isEmpty(element.updated_and_changed))
   const importErrors = elements.filter(element => !isEmpty(element.errors))
 
+
   return (
     <div className="panel panel-default panel-import">
       <div className="panel-heading">
-        <strong>{gettext('Import')}</strong>
+        <strong>{gettext('Import')} from file {filename}</strong>
         <div className="pull-right">
           {
-            updatedElements.length > -1 && <span>{gettext('Updated')}: {updatedElements.length} </span>
+            elements.length > -1 && <span>{gettext('Total')}: {elements.length} </span>
           }
+          {
+            updatedElements.length > -1 && <span> {gettext('Updated')}: {updatedElements.length}
+            {' ('}{gettext('Changed')}: {updatedAndChangedElements.length}
+            {' '}{gettext('Same')}: {updatedAndSameElements.length}{') '}
+              </span>
+            }
           {
             createdElements.length > -1 && <span>{gettext('Created')}: {createdElements.length} </span>
           }
@@ -38,50 +43,25 @@ const Import = ({ config, imports, importActions }) => {
         </div>
       </div>
     <div className="panel-body">
-    {/* TODO: still to implement functions for filter, uri_prefix dropdown, disply checkox etc.. */}
+    {/* TODO: still to implement functions for filter, uri_prefix dropdown. */}
     <div className="row">
       <div className={'col-sm-8'}>
-        <FilterString value={get(config, 'filter.catalogs.search', '')} onChange={updateFilterString}
+        <FilterString value={get(config, 'filter.import.elements.search', '')} onChange={updateFilterString}
                       placeholder={gettext('Filter uri')} />
       </div>
       <div className="col-sm-4">
-        <FilterUriPrefix value={get(config, 'filter.catalogs.uri_prefix', '')} onChange={updateFilterUriPrefix}
+      {/* TODO: add update of the filter to elements */}
+        <FilterUriPrefix value={get(config, 'filter.import.elements.uri_prefix', '')} onChange={updateFilterUriPrefix}
                         options={getUriPrefixes(elements)} />
       </div>
 
     </div>
-    <div className="checkboxes">
-      <span className="mr-10">{gettext('Show URIs:')}</span>
-      <Checkbox label={<code className="code-questions">{gettext('Catalogs')}</code>}
-                value={get(config, 'display.uri.catalogs', true)} onChange={updateDisplayCatalogURI} />
     </div>
-    </div>
-
       <ul className="list-group">
       {
         elements.map((element, index) => {
           if (success) {
-            return (
-              <li key={index} className="list-group-item">
-                <p>
-                  <strong>{verboseNames[element.model]}{' '}</strong>
-                  <code className={codeClass[element.model]}>{element.uri}</code>
-                  {element.created && <span className="text-success">{' '}{gettext('created')}</span>}
-                  {element.updated && <span className="text-info">{' '}{gettext('updated')}</span>}
-                  {
-                    !isEmpty(element.errors) && !(element.created || element.updated) &&
-                    <span className="text-danger">{' '}{gettext('could not be imported')}</span>
-                  }
-                  {
-                    !isEmpty(element.errors) && (element.created || element.updated) &&
-                    <>{', '}<span className="text-danger">{gettext('but could not be added to parent element')}</span></>
-                  }
-                  {'.'}
-                </p>
-                {element.warnings.map(message => <p key={uniqueId()} className="text-warning">{message}</p>)}
-                {element.errors.map(message => <p key={uniqueId()} className="text-danger">{message}</p>)}
-              </li>
-            )
+            return <ImportSuccesElement key={index} instance={element}/>
           } else {
               return <ImportElement key={index} config={config} instance={element} importActions={importActions} />
           }

From 2a1c27a414d9c75aec840f1a91925c1dc050d30b Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 15 Jan 2024 18:58:10 +0100
Subject: [PATCH 041/205] chore: refactor import permissions check and set
 foreign fields

---
 rdmo/conditions/imports.py | 14 ++++---------
 rdmo/core/imports.py       | 35 ++++++++++++++++++++-------------
 rdmo/domain/imports.py     | 16 +++++++--------
 rdmo/options/imports.py    | 10 +++-------
 rdmo/questions/imports.py  | 40 ++++++++++++++------------------------
 rdmo/tasks/imports.py      | 13 +++++--------
 rdmo/views/imports.py      |  6 +-----
 7 files changed, 58 insertions(+), 76 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index 36e5d3d966..981e038f98 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -2,13 +2,10 @@
 from typing import Callable, Tuple
 
 from django.contrib.sites.models import Site
-from django.db import models
 
 from rdmo.conditions.validators import ConditionLockedValidator, ConditionUniqueURIValidator
 from rdmo.core.imports import (
     ElementImportHelper,
-    check_permissions,
-    set_foreign_field,
     validate_instance,
 )
 
@@ -22,19 +19,15 @@ def import_condition(
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
-        user: models.Model = None
     ):
 
-    set_foreign_field(instance, 'source', element)
-    set_foreign_field(instance, 'target_option', element)
-
+    # set_foreign_field are already set in management/import.py
+    # check_permissions already done in management/import.py
     instance.relation = element.get('relation') or ''
     instance.target_text = element.get('target_text') or ''
 
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 
@@ -49,5 +42,6 @@ def import_condition(
     model="conditions.condition",
     import_method=import_condition,
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
-    lang_fields=None
+    lang_fields=[],
+    foreign_fields=('source', 'target_option')
 )
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 61978cc936..006a12a5fa 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -73,14 +73,16 @@ class ElementImportHelper:
     model: str
     import_method: Callable
     validators: Iterable[Callable]
-    lang_fields: Sequence[str] = field(default_factory=list)
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
+    lang_fields: Sequence[str] = field(default_factory=list)
+    foreign_fields: Sequence[str] = field(default_factory=list)
+
 
 
 def get_lang_field_values(field_name: str,
-                            element: Optional[dict] = None,
-                            instance: models.Model = None,
-                            by_field: bool = True):
+                        element: Optional[dict] = None,
+                        instance: Optional[models.Model] = None,
+                        by_field: bool = True):
     if (element and instance):
         raise ValueError("Please choose one of each")
 
@@ -102,7 +104,7 @@ def set_lang_field(instance, field_name, element):
         setattr(instance, field_lang_name, field_value)
 
 
-def set_foreign_field(instance, field_name, element) -> None:
+def set_foreign_field(instance, field_name, element, uploaded_uris=None) -> None:
     if field_name not in element:
         return
 
@@ -120,15 +122,22 @@ def set_foreign_field(instance, field_name, element) -> None:
     try:
         foreign_instance = foreign_model.objects.get(uri=foreign_uri)
         setattr(instance, field_name, foreign_instance)
+        return
     except foreign_model.DoesNotExist:
-        message = '{foreign_model} {foreign_uri} for {instance_model} {instance_uri} does not exist.'.format(
-            foreign_model=foreign_model._meta.object_name,
-            foreign_uri=foreign_uri,
-            instance_model=instance._meta.object_name,
-            instance_uri=element.get('uri')
-        )
-        logger.info(message)
-        element['warnings'][foreign_uri].append(message)
+        # check for existence of foreign_uri in currently uploaded uris
+        uploaded_uris = uploaded_uris if uploaded_uris is not None else []
+        if foreign_uri in uploaded_uris and foreign_uri is not None:
+            setattr(instance, field_name, foreign_uri)
+            return
+
+    message = '{foreign_model} {foreign_uri} for {instance_model} {instance_uri} does not exist.'.format(
+        foreign_model=foreign_model._meta.object_name,
+        foreign_uri=foreign_uri,
+        instance_model=instance._meta.object_name,
+        instance_uri=element.get('uri')
+    )
+    logger.info(message)
+    element['warnings'][foreign_uri].append(message)
 
 
 def set_m2m_instances(instance, field_name, element):
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 366d604af8..0c47a85876 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -5,8 +5,6 @@
 
 from rdmo.core.imports import (
     ElementImportHelper,
-    check_permissions,
-    set_foreign_field,
     validate_instance,
 )
 from rdmo.domain.validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
@@ -16,16 +14,17 @@
 logger = logging.getLogger(__name__)
 
 
-def import_attribute(instance: Attribute, element: dict, validators: Tuple[Callable], save=False, user=None):
-
-    set_foreign_field(instance, 'parent', element)
+def import_attribute(
+    instance: Attribute, element: dict,
+    validators: Tuple[Callable],
+    save: bool = False) -> Attribute:
 
+    # set_foreign_field are already set in management/import.py
+    # check_permissions already done in management/import.py
     instance.path = instance.build_path(instance.key, instance.parent)
 
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 
@@ -40,5 +39,6 @@ def import_attribute(instance: Attribute, element: dict, validators: Tuple[Calla
     model="domain.attribute",
     import_method=import_attribute,
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
-    lang_fields=None
+    lang_fields=[],
+    foreign_fields=('parent',)
 )
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 9d81a6f704..5f08ded56e 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -6,7 +6,6 @@
 
 from rdmo.core.imports import (
     ElementImportHelper,
-    check_permissions,
     set_m2m_instances,
     set_m2m_through_instances,
     set_reverse_m2m_through_instance,
@@ -31,15 +30,13 @@ def import_option(
         save: bool = False,
         user: models.Model = None
     ):
-
+    # check_permissions already done in management/import.py
     instance.order = element.get('order') or 0
     instance.provider_key = element.get('provider_key') or ''
     instance.additional_input = element.get('additional_input') or ""
 
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 
@@ -69,13 +66,12 @@ def import_optionset(
     ):
 
     # lang_fields are already set in management/import.py
+    # check_permissions already done in management/import.py
 
     instance.additional_input = element.get('additional_input') or ""
 
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 
@@ -91,5 +87,5 @@ def import_optionset(
     model="options.optionset",
     import_method=import_optionset,
     validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
-    lang_fields=None
+    lang_fields=[]
 )
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 6473e64c6c..2c235f794f 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -6,8 +6,6 @@
 
 from rdmo.core.imports import (
     ElementImportHelper,
-    check_permissions,
-    set_foreign_field,
     set_m2m_instances,
     set_m2m_through_instances,
     set_reverse_m2m_through_instance,
@@ -43,15 +41,13 @@ def import_catalog(
         save: bool = False,
         user: models.Model = None
     ):
-
+    # check_permissions already done in management/import.py
     instance.order = element.get('order') or 0
 
     instance.available = element.get('available', True)
 
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 
@@ -71,11 +67,9 @@ def import_section(
         save: bool = False,
         user: models.Model = None
     ):
-
+    # check_permissions already done in management/import.py
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 
@@ -96,15 +90,13 @@ def import_page(
         user: models.Model = None
     ):
     # lang_fields are already set in management/import.py
-
-    set_foreign_field(instance, 'attribute', element)
+    # set_foreign_field are already set in management/import.py
+    # check_permissions already done in management/import.py
 
     instance.is_collection = element.get('is_collection') or False
 
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 
@@ -127,14 +119,13 @@ def import_questionset(
         user: models.Model = None
     ):
     # lang_fields are already set in management/import.py
-    set_foreign_field(instance, 'attribute', element)
+    # set_foreign_field are already set in management/import.py
+    # check_permissions already done in management/import.py
 
     instance.is_collection = element.get('is_collection') or False
 
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 
@@ -158,14 +149,12 @@ def import_question(
         user: models.Model = None
     ):
     # lang_fields are already set in management/import.py
-    set_foreign_field(instance, 'attribute', element)
+    # set_foreign_fields are already set in management/import.py
+    # check_permissions already done in management/import.py
 
     instance.is_collection = element.get('is_collection') or False
     instance.is_optional = element.get('is_optional') or False
 
-
-    set_foreign_field(instance, 'default_option', element)
-
     instance.default_external_id = element.get('default_external_id') or ''
 
     if element.get('widget_type') in get_widget_types():
@@ -182,8 +171,6 @@ def import_question(
 
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 
@@ -215,20 +202,23 @@ def import_question(
 import_helper_page = ElementImportHelper(
     model="questions.page",
     import_method=import_page,
-    validators= (PageLockedValidator, PageUniqueURIValidator),
-    lang_fields=('help', 'title', 'verbose_name')
+    validators=(PageLockedValidator, PageUniqueURIValidator),
+    lang_fields=('help', 'title', 'verbose_name'),
+    foreign_fields=('attribute',)
 )
 
 import_helper_questionset = ElementImportHelper(
     model="questions.questionset",
     import_method=import_questionset,
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
-    lang_fields=('help', 'title', 'verbose_name')
+    lang_fields=('help', 'title', 'verbose_name'),
+    foreign_fields=('attribute',)
 )
 
 import_helper_question = ElementImportHelper(
     model="questions.question",
     import_method=import_question,
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
-    lang_fields=('text', 'help', 'default_text', 'verbose_name')
+    lang_fields=('text', 'help', 'default_text', 'verbose_name'),
+    foreign_fields=('attribute','default_option')
 )
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 5d50ab26cd..21de7d73ca 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -6,8 +6,6 @@
 
 from rdmo.core.imports import (
     ElementImportHelper,
-    check_permissions,
-    set_foreign_field,
     set_m2m_instances,
     validate_instance,
 )
@@ -26,10 +24,10 @@ def import_task(
         user: models.Model = None
     ):
     # lang_fields are already set in management/import.py
-    instance.order = element.get('order') or 0
+    # set_foreign_field are already set in management/import.py
+    # check_permissions already done in management/import.py
 
-    set_foreign_field(instance, 'start_attribute', element)
-    set_foreign_field(instance, 'end_attribute', element)
+    instance.order = element.get('order') or 0
 
     instance.days_before = element.get('days_before')
     instance.days_after = element.get('days_after')
@@ -38,8 +36,6 @@ def import_task(
 
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 
@@ -57,5 +53,6 @@ def import_task(
     model="tasks.task",
     import_method=import_task,
     validators=(TaskLockedValidator, TaskUniqueURIValidator),
-    lang_fields=('title', 'text')
+    lang_fields=('title', 'text'),
+    foreign_fields=('start_attribute', 'end_attribute')
 )
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 01c0991198..62857a2068 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -2,11 +2,9 @@
 from typing import Callable, Tuple
 
 from django.contrib.sites.models import Site
-from django.db import models
 
 from rdmo.core.imports import (
     ElementImportHelper,
-    check_permissions,
     set_m2m_instances,
     validate_instance,
 )
@@ -22,8 +20,8 @@ def import_view(
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
-        user: models.Model = None
     ):
+    # check_permissions already done in management/import.py
 
     instance.order = element.get('order') or 0
     instance.template = element.get('template')
@@ -32,8 +30,6 @@ def import_view(
 
     validate_instance(instance, element, *validators)
 
-    check_permissions(instance, element, user)
-
     if element.get('errors'):
         return instance
 

From 448adf6df51922a5162deeee5dd03bd20ef1cc67 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 15 Jan 2024 19:01:00 +0100
Subject: [PATCH 042/205] refactor: import_elements arg

---
 rdmo/management/management/commands/import.py | 4 ++--
 rdmo/management/viewsets.py                   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/rdmo/management/management/commands/import.py b/rdmo/management/management/commands/import.py
index 3438164220..32eae1d5ab 100644
--- a/rdmo/management/management/commands/import.py
+++ b/rdmo/management/management/commands/import.py
@@ -25,6 +25,6 @@ def handle(self, *args, **options):
             elements = flat_xml_to_elements(root)
             elements = convert_elements(elements, version)
             elements = order_elements(elements)
-            elements = elements.values()
+            parsed_elements = list(elements.values())
 
-            import_elements(elements)
+            import_elements(parsed_elements)
diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index 4f5757b7b4..f1db91c56c 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -70,10 +70,10 @@ def create(self, request, *args, **kwargs):
         elements = order_elements(elements)
 
         # step 6: convert elements to a list
-        elements = list(elements.values())
+        _elements = list(elements.values())
 
         # step 8: import the elements if save=True is set
-        imported_elements = import_elements(elements, save=is_truthy(request.POST.get('import')), user=request.user)
+        imported_elements = import_elements(_elements, save=is_truthy(request.POST.get('import')), user=request.user)
 
         # step 9: return the list of, json-serializable, elements
         return Response(imported_elements)

From 2f500a55228dd1f8d45a995e09ae5f6550e91e56 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 15 Jan 2024 19:06:06 +0100
Subject: [PATCH 043/205] chore: update import functions, add
 updated_and_changed field

---
 rdmo/management/imports.py | 61 ++++++++++++++++++++++++++++++--------
 1 file changed, 49 insertions(+), 12 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index bf2f39cec3..99aa899429 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -1,7 +1,7 @@
 import copy
 import logging
 from collections import defaultdict
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Sequence
 
 from django.db import models
 from django.forms.models import model_to_dict
@@ -12,6 +12,7 @@
     get_lang_field_values,
     get_or_return_instance,
     make_import_info_msg,
+    set_foreign_field,
     set_lang_field,
 )
 from rdmo.domain.imports import import_helper_attribute
@@ -47,10 +48,11 @@
 
 IMPORT_ELEMENT_INIT_DICT = {
         'warnings': defaultdict(list),
-        'errors': None,
+        'errors': [],
         'created': False,
         'updated': False,
-        'original': defaultdict()
+        'original': defaultdict(),
+        'updated_and_changed': defaultdict(),
     }
 
 
@@ -58,8 +60,7 @@ def import_elements(uploaded_elements: List[Dict], save: bool = True, user: Opti
     imported_elements = []
     uploaded_uris = {i.get('uri') for i in uploaded_elements}
     for element in uploaded_elements:
-        model_path = element.pop('model')
-        element = import_element(model_path=model_path, element=element, save=save, user=user)
+        element = import_element(element=element, save=save, user=user, uploaded_uris=imported_elements)
         # replace warnings with filtered list of warnings
         warnings = element.pop('warnings')
         element['warnings'] = filter_warnings(warnings, uploaded_uris)
@@ -79,26 +80,28 @@ def filter_warnings(warnings: Dict, uploaded_uris: List[Dict]) -> List[str]:
 
 
 def import_element(
-        model_path: Optional[str] = None,
         element: Optional[Dict] = None,
         save: bool = True,
-        user: Optional[models.Model] = None
+        user: Optional[models.Model] = None,
+        uploaded_uris: Optional[Sequence[str]] = None
     ):
 
     if element is None:
         return element
 
-    element.update(IMPORT_ELEMENT_INIT_DICT)
-    element['errors'] = []
-
+    model_path = element.get('model')
     if model_path is None:
         return element
 
+    # initialize element dict with default values
+    element.update(IMPORT_ELEMENT_INIT_DICT)
+
     model = RDMO_MODEL_PATH_MAPPER[model_path]
     import_helper = ELEMENT_IMPORT_HELPERS[model_path]
     import_method = import_helper.import_method
     validators = import_helper.validators
     lang_field_names = import_helper.lang_fields if import_helper.lang_fields is not None else []
+    foreign_field_names = import_helper.foreign_fields if import_helper.foreign_fields is not None else []
     uri = element.get('uri')
 
     # get or create instance from uri and model_path
@@ -117,14 +120,24 @@ def import_element(
     if _perms_error_msg:
         # when there is an error msg, the import could be stopped and return
         element["errors"].append(_perms_error_msg)
+        return element
 
     # prepare original element when updated (maybe rename into lookup)
     _updated = not _created
     original_element = {}
+    filtered_ffnames = filter(lambda x: x in original_element, foreign_field_names)
     if _updated:
         original_element = model_to_dict(original_instance)
         original_element = {k: original_element.get(k, element.get(k))
                             for k in element.keys() if k not in IMPORT_ELEMENT_INIT_DICT}
+        for _field in filtered_ffnames:
+            if original_element[_field] is None:
+                continue
+            try:
+                # set the uri for foreign fields, instead of id
+                original_element[_field] = {'uri': getattr(original_instance, _field).uri}
+            except AttributeError:
+                pass
     # start to set values on the instance
     # set common field values from element on instance
     for common_field in import_helper.common_fields:
@@ -136,17 +149,41 @@ def import_element(
             # add the lang_code fields from the original instance
             lang_field_values = get_lang_field_values(lang_field_name, instance=original_instance)
             original_element.update(lang_field_values)
+    # set foreign fields
+    for foreign_field in foreign_field_names:
+        set_foreign_field(instance, foreign_field, element, uploaded_uris=uploaded_uris)
+
     # call the element specific import method
-    instance = import_method(instance, element, validators, save, user)
+    instance = import_method(instance, element, validators, save)
 
     if element.get('errors'):
         return element
 
-    if _updated:
+    if _updated and not _created:
         element['updated'] = _updated
+        # and instance is not original_instance
         # keep only strings, make json serializable
         original_element_json = {k: val for k, val in original_element.items() if isinstance(val, str)}
         element['original'] = original_element_json
+        # add updated and changed
+        instance_field_names = {i.name for i in instance._meta.local_concrete_fields}
+        updated_and_changed = {}
+        for k, val in filter(lambda x: x[0] in instance_field_names, original_element.items()):
+
+            new_val = getattr(instance, k, None)
+            if k in foreign_field_names and new_val is not None:
+                try:
+                    # set the uri for foreign fields, instead of id
+                    new_val = {'uri': getattr(instance, k).uri}
+                except AttributeError:
+                    pass
+
+            if new_val is None:
+                continue
+            if new_val != val:
+                updated_and_changed[k] = {"current": val, "uploaded": new_val}
+
+        element['updated_and_changed'] = updated_and_changed
 
     if save:
         logger.info(_msg)

From 355d047a3c58c83ad0ca06686a929c3fd44772c0 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 15 Jan 2024 19:08:03 +0100
Subject: [PATCH 044/205] tests: refactor to read_xml_and_parse_to_elements
 function

---
 rdmo/management/tests/__init__.py             |  28 +++
 .../tests/test_import_conditions.py           |  80 ++++----
 rdmo/management/tests/test_import_domain.py   |  65 +++----
 rdmo/management/tests/test_import_options.py  |  84 +++-----
 .../management/tests/test_import_questions.py | 183 ++++++------------
 rdmo/management/tests/test_import_tasks.py    |  65 +++----
 rdmo/management/tests/test_import_views.py    |  65 +++----
 7 files changed, 225 insertions(+), 345 deletions(-)

diff --git a/rdmo/management/tests/__init__.py b/rdmo/management/tests/__init__.py
index e69de29bb2..063402d4ca 100644
--- a/rdmo/management/tests/__init__.py
+++ b/rdmo/management/tests/__init__.py
@@ -0,0 +1,28 @@
+from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
+
+
+def read_xml_and_parse_to_elements(xml_file):
+    root = read_xml_file(xml_file)
+    version = root.attrib.get('version')
+    elements = flat_xml_to_elements(root)
+    elements = convert_elements(elements, version)
+    elements = order_elements(elements)
+    parsed_elements = list(elements.values())
+    return parsed_elements, root
+
+def change_fields_elements(elements, update_dict=None, n=3):
+
+    _default_update_dict = {'comment':  "this is a test comment {}"}
+
+    if len(elements) < n:
+        raise ValueError("Length of elements should not be smaller than n.")
+    _new_elements = []
+    _changed_elements = []
+    for _n,_element in enumerate(elements):
+        if _n <= n-1:
+            _element['comment'] = _default_update_dict['comment'].format(_n)
+            if update_dict is not None:
+                _element.update (**update_dict)
+            _changed_elements.append(_element)
+        _new_elements.append(_element)
+    return _new_elements, _changed_elements
diff --git a/rdmo/management/tests/test_import_conditions.py b/rdmo/management/tests/test_import_conditions.py
index 4c97ced3fd..8cf6d30dd3 100644
--- a/rdmo/management/tests/test_import_conditions.py
+++ b/rdmo/management/tests/test_import_conditions.py
@@ -1,73 +1,67 @@
 from pathlib import Path
 
 from rdmo.conditions.models import Condition
-from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
 from rdmo.management.imports import import_elements
 
+from . import change_fields_elements, read_xml_and_parse_to_elements
+
 
 def test_create_conditions(db, settings):
     Condition.objects.all().delete()
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == Condition.objects.count() == 15
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert len(root) == len(imported_elements) == Condition.objects.count() == 15
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_conditions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+
+    assert len(root) == len(imported_elements) == 15
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+
+
+def test_update_conditions_with_changed_comments(db, settings):
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
+
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, changed_elements = change_fields_elements(elements, n=3)
+    # breakpoint()
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements)
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(imported_elements) == 15
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len([i for i in elements if i['updated_and_changed']]) == len(changed_elements)
 
 
 def test_create_legacy_conditions(db, settings):
     Condition.objects.all().delete()
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'conditions.xml'
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
-
-    assert len(root) == len(elements) == Condition.objects.count() == 15
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert len(root) == len(imported_elements) == Condition.objects.count() == 15
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_legacy_conditions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'conditions.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
-
-    assert len(root) == len(elements) == 15
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+
+    assert len(root) == len(imported_elements) == 15
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_domain.py b/rdmo/management/tests/test_import_domain.py
index b9f032b92f..ac286fc0a7 100644
--- a/rdmo/management/tests/test_import_domain.py
+++ b/rdmo/management/tests/test_import_domain.py
@@ -1,42 +1,33 @@
 from pathlib import Path
 
-from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
 from rdmo.domain.models import Attribute
 from rdmo.management.imports import import_elements
 
+from . import read_xml_and_parse_to_elements
+
 
 def test_create_domain(db, settings):
     Attribute.objects.all().delete()
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == Attribute.objects.count() == 86
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert len(root) == len(imported_elements) == Attribute.objects.count() == 86
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_domain(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements)
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(imported_elements)
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
 
 def test_create_legacy_domain(db, settings):
@@ -44,31 +35,21 @@ def test_create_legacy_domain(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'domain.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 86
+    assert len(root) == len(imported_elements) == 86
     assert Attribute.objects.count() == 86
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_legacy_domain(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'domain.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
-
-    assert len(root) == len(elements) == 86
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+
+    assert len(root) == len(imported_elements) == 86
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 7139e92f85..9abcdebbb7 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -1,75 +1,59 @@
 from pathlib import Path
 
-from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
 from rdmo.management.imports import import_elements
 from rdmo.options.models import Option, OptionSet
 
+from . import read_xml_and_parse_to_elements
+
 
 def test_create_optionsets(db, settings):
     OptionSet.objects.all().delete()
     Option.objects.all().delete()
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    parsed_elements = elements.values()
-    imported_elements = import_elements(parsed_elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == 13
     assert OptionSet.objects.count() == 4
     assert Option.objects.count() == 9
-    assert all(element['created'] is True for element in parsed_elements)
-    assert all(element['updated'] is False for element in parsed_elements)
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_optionsets(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    parsed_elements = elements.values()
-    imported_elements = import_elements(parsed_elements)
+
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == 13
-    assert all(element['created'] is False for element in parsed_elements)
-    assert all(element['updated'] is True for element in parsed_elements)
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
 
 def test_create_options(db, settings):
     Option.objects.all().delete()
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    parsed_elements = elements.values()
-    imported_elements = import_elements(parsed_elements)
+
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == Option.objects.count() == 9
-    assert all(element['created'] is True for element in parsed_elements)
-    assert all(element['updated'] is False for element in parsed_elements)
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_options(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    parsed_elements = elements.values()
-    imported_elements = import_elements(parsed_elements)
+
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == 9
-    assert all(element['created'] is False for element in parsed_elements)
-    assert all(element['updated'] is True for element in parsed_elements)
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
 
 def test_create_legacy_options(db, settings):
@@ -78,32 +62,22 @@ def test_create_legacy_options(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'options.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    parsed_elements = elements.values()
-    imported_elements = import_elements(parsed_elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == 12
     assert OptionSet.objects.count() == 4
     assert Option.objects.count() == 8
-    assert all(element['created'] is True for element in parsed_elements)
-    assert all(element['updated'] is False for element in parsed_elements)
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_legacy_options(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'options.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    parsed_elements = elements.values()
-    imported_elements = import_elements(parsed_elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == 12
-    assert all(element['created'] is False for element in parsed_elements)
-    assert all(element['updated'] is True for element in parsed_elements)
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py
index 7bb275fce2..a282f5b56c 100644
--- a/rdmo/management/tests/test_import_questions.py
+++ b/rdmo/management/tests/test_import_questions.py
@@ -1,9 +1,10 @@
 from pathlib import Path
 
-from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
 from rdmo.management.imports import import_elements
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
+from . import read_xml_and_parse_to_elements
+
 
 def test_create_catalogs(db, settings):
     Catalog.objects.all().delete()
@@ -14,38 +15,28 @@ def test_create_catalogs(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 148
+    assert len(root) == len(imported_elements) == 148
     assert Catalog.objects.count() == 2
     assert Section.objects.count() == 6
     assert Page.objects.count() == 48
     assert QuestionSet.objects.count() == 3
     assert Question.objects.count() == 89
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_catalogs(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 148
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(imported_elements) == 148
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
 
 def test_create_sections(db, settings):
@@ -56,37 +47,27 @@ def test_create_sections(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 146
+    assert len(root) == len(imported_elements) == 146
     assert Section.objects.count() == 6
     assert Page.objects.count() == 48
     assert QuestionSet.objects.count() == 3
     assert Question.objects.count() == 89
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_sections(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 146
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(imported_elements) == 146
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
 
 def test_create_pages(db, settings):
@@ -96,36 +77,26 @@ def test_create_pages(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 140
+    assert len(root) == len(imported_elements) == 140
     assert Page.objects.count() == 48
     assert QuestionSet.objects.count() == 3
     assert Question.objects.count() == 89
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_pages(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 140
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(imported_elements) == 140
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
 
 def test_create_questionsets(db, settings):
@@ -135,36 +106,26 @@ def test_create_questionsets(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
     assert len(root) == 10  # two questionsets appear twice in the export file
-    assert len(elements) == 8
+    assert len(imported_elements) == 8
     assert QuestionSet.objects.count() == 3
     assert Question.objects.count() == 5
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_questionsets(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
     assert len(root) == 10  # two questionsets appear twice in the export file
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
 
 def test_create_questions(db, settings):
@@ -174,34 +135,24 @@ def test_create_questions(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 89
+    assert len(root) == len(imported_elements) == 89
     assert Question.objects.count() == 89
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_questions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 89
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(imported_elements) == 89
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
 
 def test_create_legacy_questions(db, settings):
@@ -213,22 +164,17 @@ def test_create_legacy_questions(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'questions.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 147
+    assert len(root) == len(imported_elements) == 147
     assert Catalog.objects.count() == 1
     assert Section.objects.count() == 6
     assert Page.objects.count() == 48
     assert QuestionSet.objects.count() == 3
     assert Question.objects.count() == 89
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
     # check that all elements ended up in the catalog
     catalog = Catalog.objects.prefetch_elements().first()
@@ -240,17 +186,12 @@ def test_create_legacy_questions(db, settings):
 def test_update_legacy_questions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'questions.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
-
-    assert len(root) == len(elements) == 147
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+
+    assert len(root) == len(imported_elements) == 147
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
     # check that all elements ended up in the catalog
     catalog = Catalog.objects.prefetch_elements().first()
diff --git a/rdmo/management/tests/test_import_tasks.py b/rdmo/management/tests/test_import_tasks.py
index 08071990cf..3e9a87831b 100644
--- a/rdmo/management/tests/test_import_tasks.py
+++ b/rdmo/management/tests/test_import_tasks.py
@@ -1,42 +1,33 @@
 from pathlib import Path
 
-from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
 from rdmo.management.imports import import_elements
 from rdmo.tasks.models import Task
 
+from . import read_xml_and_parse_to_elements
+
 
 def test_create_tasks(db, settings):
     Task.objects.all().delete()
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'tasks.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == Task.objects.count() == 2
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert len(root) == len(imported_elements) == Task.objects.count() == 2
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'tasks.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 2
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(imported_elements) == 2
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
 
 def test_create_legacy_tasks(db, settings):
@@ -44,30 +35,20 @@ def test_create_legacy_tasks(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'tasks.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == Task.objects.count() == 2
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert len(root) == len(imported_elements) == Task.objects.count() == 2
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_legacy_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'tasks.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
-
-    assert len(root) == len(elements) == 2
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+
+    assert len(root) == len(imported_elements) == 2
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_views.py b/rdmo/management/tests/test_import_views.py
index a6c7bc7d06..bf169c69e2 100644
--- a/rdmo/management/tests/test_import_views.py
+++ b/rdmo/management/tests/test_import_views.py
@@ -1,42 +1,33 @@
 from pathlib import Path
 
-from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
 from rdmo.management.imports import import_elements
 from rdmo.views.models import View
 
+from . import read_xml_and_parse_to_elements
+
 
 def test_create_tasks(db, settings):
     View.objects.all().delete()
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'views.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == View.objects.count() == 3
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert len(root) == len(imported_elements) == View.objects.count() == 3
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'views.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == 3
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    assert len(root) == len(imported_elements) == 3
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
 
 
 def test_create_legacy_tasks(db, settings):
@@ -44,30 +35,20 @@ def test_create_legacy_tasks(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'views.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
 
-    assert len(root) == len(elements) == View.objects.count() == 3
-    assert all(element['created'] is True for element in elements)
-    assert all(element['updated'] is False for element in elements)
+    assert len(root) == len(imported_elements) == View.objects.count() == 3
+    assert all(element['created'] is True for element in imported_elements)
+    assert all(element['updated'] is False for element in imported_elements)
 
 
 def test_update_legacy_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'views.xml'
 
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    elements = elements.values()
-    import_elements(elements)
-
-    assert len(root) == len(elements) == 3
-    assert all(element['created'] is False for element in elements)
-    assert all(element['updated'] is True for element in elements)
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+
+    assert len(root) == len(imported_elements) == 3
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)

From 6b188b0cd8a993be432c0540ac6048ab4679e7fe Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 16 Jan 2024 18:28:15 +0100
Subject: [PATCH 045/205] chore: rename to import_func

---
 rdmo/conditions/imports.py |  2 +-
 rdmo/core/imports.py       |  2 +-
 rdmo/domain/imports.py     |  2 +-
 rdmo/options/imports.py    |  4 ++--
 rdmo/questions/imports.py  | 10 +++++-----
 rdmo/tasks/imports.py      |  2 +-
 rdmo/views/imports.py      |  2 +-
 7 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index 981e038f98..eb19428dc6 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -40,7 +40,7 @@ def import_condition(
 
 import_helper_condition = ElementImportHelper(
     model="conditions.condition",
-    import_method=import_condition,
+    import_func=import_condition,
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
     lang_fields=[],
     foreign_fields=('source', 'target_option')
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 006a12a5fa..6305b2699a 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -71,7 +71,7 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
 @dataclass
 class ElementImportHelper:
     model: str
-    import_method: Callable
+    import_func: Callable
     validators: Iterable[Callable]
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
     lang_fields: Sequence[str] = field(default_factory=list)
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 0c47a85876..b17ff29b50 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -37,7 +37,7 @@ def import_attribute(
 
 import_helper_attribute = ElementImportHelper(
     model="domain.attribute",
-    import_method=import_attribute,
+    import_func=import_attribute,
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     lang_fields=[],
     foreign_fields=('parent',)
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 5f08ded56e..380d344de8 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -51,7 +51,7 @@ def import_option(
 
 import_helper_option = ElementImportHelper(
     model="options.option",
-    import_method=import_option,
+    import_func=import_option,
     validators=(OptionLockedValidator, OptionUniqueURIValidator),
     lang_fields=('text',)
 )
@@ -85,7 +85,7 @@ def import_optionset(
 
 import_helper_optionset = ElementImportHelper(
     model="options.optionset",
-    import_method=import_optionset,
+    import_func=import_optionset,
     validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
     lang_fields=[]
 )
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 2c235f794f..5a92a32db3 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -187,21 +187,21 @@ def import_question(
 
 import_helper_catalog = ElementImportHelper(
     model="questions.catalog",
-    import_method=import_catalog,
+    import_func=import_catalog,
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
     lang_fields=('help', 'title')
 )
 
 import_helper_section = ElementImportHelper(
     model="questions.section",
-    import_method=import_section,
+    import_func=import_section,
     validators=(SectionLockedValidator, SectionUniqueURIValidator),
     lang_fields=('title',)
 )
 
 import_helper_page = ElementImportHelper(
     model="questions.page",
-    import_method=import_page,
+    import_func=import_page,
     validators=(PageLockedValidator, PageUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',)
@@ -209,7 +209,7 @@ def import_question(
 
 import_helper_questionset = ElementImportHelper(
     model="questions.questionset",
-    import_method=import_questionset,
+    import_func=import_questionset,
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',)
@@ -217,7 +217,7 @@ def import_question(
 
 import_helper_question = ElementImportHelper(
     model="questions.question",
-    import_method=import_question,
+    import_func=import_question,
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
     lang_fields=('text', 'help', 'default_text', 'verbose_name'),
     foreign_fields=('attribute','default_option')
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 21de7d73ca..2175848e31 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -51,7 +51,7 @@ def import_task(
 
 import_helper_task = ElementImportHelper(
     model="tasks.task",
-    import_method=import_task,
+    import_func=import_task,
     validators=(TaskLockedValidator, TaskUniqueURIValidator),
     lang_fields=('title', 'text'),
     foreign_fields=('start_attribute', 'end_attribute')
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 62857a2068..6f5b9c597c 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -44,7 +44,7 @@ def import_view(
 
 import_helper_view = ElementImportHelper(
     model="views.view",
-    import_method=import_view,
+    import_func=import_view,
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
     lang_fields=( 'help', 'title')
 )

From 632239be847057359791a176f6036c437ecf3fb6 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 18 Jan 2024 18:58:58 +0100
Subject: [PATCH 046/205] chore: fix element init dict and refactor original
 funcs

---
 rdmo/core/imports.py       | 48 ++++++++++++++++++++-
 rdmo/management/imports.py | 87 +++++++++++---------------------------
 2 files changed, 71 insertions(+), 64 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 6305b2699a..31596f23f6 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -5,10 +5,11 @@
 from os.path import join as pj
 from pathlib import Path
 from random import randint
-from typing import Callable, Iterable, Optional, Sequence, Tuple
+from typing import Callable, Dict, Iterable, List, Optional, Sequence, Tuple
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
+from django.forms.models import model_to_dict
 
 from rest_framework.utils import model_meta
 
@@ -23,7 +24,6 @@
     'key',
     'comment',
 )
-IMPORT_INFO_MSG = 'Importing {model} {uri} from {filename}.'
 
 
 def handle_uploaded_file(filedata):
@@ -313,3 +313,47 @@ def check_permissions(instance: models.Model, element_uri: str, user: models.Mod
         message = f'You have no permissions to import {instance._meta.object_name} {element_uri}.'
         logger.info(message)
         return message
+
+
+def prepare_element_from_original_instance(element: Dict,
+                                        original_instance: models.Model,
+                                        lang_field_names: List[str],
+                                        foreign_field_names: List[str]):
+    original_element = model_to_dict(original_instance)
+
+    filtered_ffnames = filter(lambda x: x in original_element, foreign_field_names)
+    for _field in filtered_ffnames:
+        if original_element[_field] is None:
+            continue
+        try:
+            # set the uri for foreign fields, instead of id
+            original_element[_field] = {'uri': getattr(original_instance, _field).uri}
+        except AttributeError:
+            pass
+    for lang_field_name in lang_field_names:
+        # add the lang_code fields from the original instance
+        lang_field_values = get_lang_field_values(lang_field_name, instance=original_instance)
+        original_element.update(lang_field_values)
+    return original_element
+
+
+def get_original_and_updated(original_element: Dict, instance: models.Model, foreign_field_names: List[str]) -> Dict:
+
+    # add updated and changed
+    instance_field_names = {i.name for i in instance._meta.local_concrete_fields}
+    updated_and_changed = {}
+    for k, val in filter(lambda x: x[0] in instance_field_names, original_element.items()):
+
+        new_val = getattr(instance, k, None)
+        if k in foreign_field_names and new_val is not None:
+            try:
+                # set the uri for foreign fields, instead of id
+                new_val = {'uri': getattr(instance, k).uri}
+            except AttributeError:
+                pass
+
+        if new_val is None:
+            continue
+        if new_val != val:
+            updated_and_changed[k] = {"current": val, "uploaded": new_val}
+    return updated_and_changed
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 99aa899429..dfbf11c35c 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -4,14 +4,14 @@
 from typing import Dict, List, Optional, Sequence
 
 from django.db import models
-from django.forms.models import model_to_dict
 
 from rdmo.conditions.imports import import_helper_condition
 from rdmo.core.imports import (
     check_permissions,
-    get_lang_field_values,
     get_or_return_instance,
+    get_original_and_updated,
     make_import_info_msg,
+    prepare_element_from_original_instance,
     set_foreign_field,
     set_lang_field,
 )
@@ -47,37 +47,27 @@
     }
 
 IMPORT_ELEMENT_INIT_DICT = {
-        'warnings': defaultdict(list),
-        'errors': [],
-        'created': False,
-        'updated': False,
-        'original': defaultdict(),
-        'updated_and_changed': defaultdict(),
+        'warnings': lambda: defaultdict(list),
+        'errors': list,
+        'created': bool,
+        'updated': bool,
+        'original': dict,
+        'updated_and_changed': dict,
     }
 
 
 def import_elements(uploaded_elements: List[Dict], save: bool = True, user: Optional[models.Model] = None):
     imported_elements = []
     uploaded_uris = {i.get('uri') for i in uploaded_elements}
-    for element in uploaded_elements:
-        element = import_element(element=element, save=save, user=user, uploaded_uris=imported_elements)
-        # replace warnings with filtered list of warnings
-        warnings = element.pop('warnings')
-        element['warnings'] = filter_warnings(warnings, uploaded_uris)
+    for uploaded_element in uploaded_elements:
+        element = import_element(element=uploaded_element, save=save, user=user, uploaded_uris=imported_elements)
+        # replace warnings with filtered flat list of warnings
+        filtered_warnings = set(filter(lambda k: k not in uploaded_uris, element['warnings'].keys()))
+        element['warnings'] = [val for k,val in element['warnings'].items() if k not in filtered_warnings]
         imported_elements.append(element)
     return imported_elements
 
 
-def filter_warnings(warnings: Dict, uploaded_uris: List[Dict]) -> List[str]:
-    # remove warnings regarding elements which are in the elements list
-    ret = []
-    if not warnings:
-        return ret
-    for uri, messages in warnings.items():
-        if uri not in uploaded_uris:
-            ret += messages
-    return ret
-
 
 def import_element(
         element: Optional[Dict] = None,
@@ -94,12 +84,14 @@ def import_element(
         return element
 
     # initialize element dict with default values
-    element.update(IMPORT_ELEMENT_INIT_DICT)
+    for _k,_val in IMPORT_ELEMENT_INIT_DICT.items():
+        element[_k] = _val()
 
     model = RDMO_MODEL_PATH_MAPPER[model_path]
     import_helper = ELEMENT_IMPORT_HELPERS[model_path]
-    import_method = import_helper.import_method
+    import_func = import_helper.import_func
     validators = import_helper.validators
+    common_fields = import_helper.common_fields
     lang_field_names = import_helper.lang_fields if import_helper.lang_fields is not None else []
     foreign_field_names = import_helper.foreign_fields if import_helper.foreign_fields is not None else []
     uri = element.get('uri')
@@ -124,64 +116,35 @@ def import_element(
 
     # prepare original element when updated (maybe rename into lookup)
     _updated = not _created
-    original_element = {}
-    filtered_ffnames = filter(lambda x: x in original_element, foreign_field_names)
-    if _updated:
-        original_element = model_to_dict(original_instance)
-        original_element = {k: original_element.get(k, element.get(k))
-                            for k in element.keys() if k not in IMPORT_ELEMENT_INIT_DICT}
-        for _field in filtered_ffnames:
-            if original_element[_field] is None:
-                continue
-            try:
-                # set the uri for foreign fields, instead of id
-                original_element[_field] = {'uri': getattr(original_instance, _field).uri}
-            except AttributeError:
-                pass
+
     # start to set values on the instance
     # set common field values from element on instance
-    for common_field in import_helper.common_fields:
+    for common_field in common_fields:
         setattr(instance, common_field, element.get(common_field) or '')
     # set language fields
     for lang_field_name in lang_field_names:
         set_lang_field(instance, lang_field_name, element)
-        if original_instance is not None:
-            # add the lang_code fields from the original instance
-            lang_field_values = get_lang_field_values(lang_field_name, instance=original_instance)
-            original_element.update(lang_field_values)
     # set foreign fields
     for foreign_field in foreign_field_names:
         set_foreign_field(instance, foreign_field, element, uploaded_uris=uploaded_uris)
 
     # call the element specific import method
-    instance = import_method(instance, element, validators, save)
+    instance = import_func(instance, element, validators, save)
 
     if element.get('errors'):
         return element
 
     if _updated and not _created:
         element['updated'] = _updated
+        original_element = {}
+
         # and instance is not original_instance
         # keep only strings, make json serializable
+        original_element = prepare_element_from_original_instance(element, original_instance,
+                                                                    lang_field_names, foreign_field_names)
         original_element_json = {k: val for k, val in original_element.items() if isinstance(val, str)}
         element['original'] = original_element_json
-        # add updated and changed
-        instance_field_names = {i.name for i in instance._meta.local_concrete_fields}
-        updated_and_changed = {}
-        for k, val in filter(lambda x: x[0] in instance_field_names, original_element.items()):
-
-            new_val = getattr(instance, k, None)
-            if k in foreign_field_names and new_val is not None:
-                try:
-                    # set the uri for foreign fields, instead of id
-                    new_val = {'uri': getattr(instance, k).uri}
-                except AttributeError:
-                    pass
-
-            if new_val is None:
-                continue
-            if new_val != val:
-                updated_and_changed[k] = {"current": val, "uploaded": new_val}
+        updated_and_changed = get_original_and_updated(original_element, instance, foreign_field_names)
 
         element['updated_and_changed'] = updated_and_changed
 

From 1dacf0922c3f357014615bf203b30831be606384 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 19 Jan 2024 12:14:57 +0100
Subject: [PATCH 047/205] chore: import js rename obj to element

---
 .../js/components/import/ImportElement.js     | 34 +++++++++----------
 .../components/import/ImportSuccessElement.js | 20 +++++------
 .../assets/js/components/main/Import.js       |  4 +--
 3 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index c670272f44..3558f9b5b8 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -11,35 +11,35 @@ import Warnings from './common/Warnings'
 import { codeClass, verboseNames } from '../../constants/elements'
 import { isEmpty } from 'lodash'
 
-const ImportElement = ({ config, instance, importActions }) => {
-  const showFields = () => importActions.updateElement(instance, {show: !instance.show})
-  const toggleImport = () => importActions.updateElement(instance, {import: !instance.import})
-  const updateInstance = (key, value) => importActions.updateElement(instance, {[key]: value})
+const ImportElement = ({ config, element, importActions }) => {
+  const showFields = () => importActions.updateElement(element, {show: !element.show})
+  const toggleImport = () => importActions.updateElement(element, {import: !element.import})
+  const updateElement = (key, value) => importActions.updateElement(element, {[key]: value})
 
   return (
     <li className="list-group-item">
       <div className="pull-right">
-        <WarningLink element={instance} onClick={showFields} />
-        <ErrorLink element={instance} onClick={showFields} />
-        <ShowLink element={instance} onClick={showFields} />
+        <WarningLink element={element} onClick={showFields} />
+        <ErrorLink element={element} onClick={showFields} />
+        <ShowLink element={element} onClick={showFields} />
         {
-          instance.updated && !isEmpty(instance.updated_and_changed) && !instance.created &&
+          element.updated && !isEmpty(element.updated_and_changed) && !element.created &&
           <p className="element-link fa fa-pencil"></p>
         }
       </div>
       <div className="checkbox">
         <label className="mr-5">
-          <input type="checkbox" checked={instance.import} onChange={toggleImport} />
-          <strong>{verboseNames[instance.model]}{' '}</strong>
+          <input type="checkbox" checked={element.import} onChange={toggleImport} />
+          <strong>{verboseNames[element.model]}{' '}</strong>
         </label>
-        <CodeLink className={codeClass[instance.model]} uri={instance.uri} onClick={showFields} />
+        <CodeLink className={codeClass[element.model]} uri={element.uri} onClick={showFields} />
       </div>
       {
-        instance.show && <>
-          <Form config={config} element={instance} updateElement={updateInstance} />
-          <Fields element={instance} />
-          <Warnings element={instance} />
-          <Errors element={instance} />
+        element.show && <>
+          <Form config={config} element={element} updateElement={updateElement} />
+          <Fields element={element} />
+          <Warnings element={element} />
+          <Errors element={element} />
         </>
       }
     </li>
@@ -48,7 +48,7 @@ const ImportElement = ({ config, instance, importActions }) => {
 
 ImportElement.propTypes = {
   config: PropTypes.object.isRequired,
-  instance: PropTypes.object.isRequired,
+  element: PropTypes.object.isRequired,
   importActions: PropTypes.object.isRequired
 }
 
diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index 20440e7dc9..4a8446db1d 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -5,32 +5,32 @@ import uniqueId from 'lodash/uniqueId'
 import { codeClass, verboseNames } from '../../constants/elements'
 import { isEmpty } from 'lodash'
 
-const ImportSuccessElement = ({ instance }) => {
+const ImportSuccessElement = ({ element }) => {
   return (
     <li className="list-group-item">
       <p>
-        <strong>{verboseNames[instance.model]}{' '}</strong>
-        <code className={codeClass[instance.model]}>{instance.uri}</code>
-        {instance.created && <span className="text-success">{' '}{gettext('created')}  </span> && <span className="muted element-link fa fa-plus"></span>}
-        {instance.updated && <span className="text-info">{' '}{gettext('updated')} </span> && <span className="muted element-link fa fa-pencil"></span>}
+        <strong>{verboseNames[element.model]}{' '}</strong>
+        <code className={codeClass[element.model]}>{element.uri}</code>
+        {element.created && <span className="text-success">{' '}{gettext('created')}  </span> && <span className="muted element-link fa fa-plus"></span>}
+        {element.updated && <span className="text-info">{' '}{gettext('updated')} </span> && <span className="muted element-link fa fa-pencil"></span>}
         {
-          !isEmpty(instance.errors) && !(instance.created || instance.updated) &&
+          !isEmpty(element.errors) && !(element.created || element.updated) &&
           <span className="text-danger">{' '}{gettext('could not be imported')}</span>
         }
         {
-          !isEmpty(instance.errors) && (instance.created || instance.updated) &&
+          !isEmpty(element.errors) && (element.created || element.updated) &&
           <>{', '}<span className="text-danger">{gettext('but could not be added to parent element')}</span></>
         }
         {'.'}
       </p>
-      {instance.warnings.map(message => <p key={uniqueId()} className="text-warning">{message}</p>)}
-      {instance.errors.map(message => <p key={uniqueId()} className="text-danger">{message}</p>)}
+      {element.warnings.map(message => <p key={uniqueId()} className="text-warning">{message}</p>)}
+      {element.errors.map(message => <p key={uniqueId()} className="text-danger">{message}</p>)}
     </li>
   )
 }
 
 ImportSuccessElement.propTypes = {
-  instance: PropTypes.object.isRequired,
+  element: PropTypes.object.isRequired,
 }
 
 export default ImportSuccessElement
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 5b53ae903c..2a7c863a81 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -61,9 +61,9 @@ const Import = ({ config, imports, importActions }) => {
       {
         elements.map((element, index) => {
           if (success) {
-            return <ImportSuccesElement key={index} instance={element}/>
+            return <ImportSuccesElement key={index} element={element}/>
           } else {
-              return <ImportElement key={index} config={config} instance={element} importActions={importActions} />
+              return <ImportElement key={index} config={config} element={element} importActions={importActions} />
           }
         })
       }

From 0d5632c914b61375302e9164b04fe48b201c32b3 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 19 Jan 2024 12:15:31 +0100
Subject: [PATCH 048/205] feat: import sidebar add select and show
 ChangedElements

---
 .../assets/js/actions/importActions.js        |  6 ++++
 .../js/components/sidebar/ImportSidebar.js    | 35 +++++++++++++++++--
 .../assets/js/reducers/importsReducer.js      | 21 ++++++++++-
 3 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/rdmo/management/assets/js/actions/importActions.js b/rdmo/management/assets/js/actions/importActions.js
index 7abb1d1bdb..96dceb16a2 100644
--- a/rdmo/management/assets/js/actions/importActions.js
+++ b/rdmo/management/assets/js/actions/importActions.js
@@ -65,10 +65,16 @@ export function updateElement(element, values) {
 export function selectElements(value) {
   return {type: 'import/selectElements', value}
 }
+export function selectChangedElements(value) {
+  return {type: 'import/selectChangedElements', value}
+}
 
 export function showElements(value) {
   return {type: 'import/showElements', value}
 }
+export function showChangedElements(value) {
+  return {type: 'import/showChangedElements', value}
+}
 
 export function updateUriPrefix(uriPrefix) {
   return {type: 'import/updateUriPrefix', uriPrefix}
diff --git a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
index c1cecb8b40..73e2614d54 100644
--- a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
+++ b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
@@ -8,6 +8,7 @@ import Link from 'rdmo/core/assets/js/components/Link'
 const ImportSidebar = ({ config, imports, importActions }) => {
   const { elements, success } = imports
   const count = elements.filter(e => e.import).length
+  const updatedAndChangedElements = elements.filter(element => element.updated && !isEmpty(element.updated_and_changed))
   const [uriPrefix, setUriPrefix] = useState('')
   const disabled = isNil(uriPrefix) || isEmpty(uriPrefix)
 
@@ -33,7 +34,6 @@ const ImportSidebar = ({ config, imports, importActions }) => {
     return (
       <div className="import-sidebar">
         <h2>{gettext('Import elements')}</h2>
-
         <p className="import-buttons">
           <button className="btn btn-success" onClick={() => importActions.importElements()}>
             {interpolate(ngettext('Import one element', 'Import %s elements', count), [count])}
@@ -51,17 +51,48 @@ const ImportSidebar = ({ config, imports, importActions }) => {
               {gettext('Select all')}
             </Link>
           </li>
+          { updatedAndChangedElements.length > -1 &&
+            <ul className="list-unstyled">
+            <li>
+              <Link onClick={() => importActions.selectChangedElements(true)}>
+                {gettext('Select changed')}
+              </Link>
+            </li>
+            <li>
+              <Link onClick={() => importActions.selectChangedElements(false)}>
+                {gettext('Unselect changed')}
+              </Link>
+            </li>
+            </ul>
+          }
           <li>
             <Link onClick={() => importActions.selectElements(false)}>
               {gettext('Unselect all')}
             </Link>
           </li>
-          <br></br>
+          </ul>
+
+          <h2>{gettext('Show')}</h2>
+          <ul className="list-unstyled">
           <li>
             <Link onClick={() => importActions.showElements(true)}>
               {gettext('Show all')}
             </Link>
           </li>
+          { updatedAndChangedElements.length > -1 &&
+            <ul className="list-unstyled">
+            <li>
+              <Link onClick={() => importActions.showChangedElements(true)}>
+                {gettext('Show changed')}
+              </Link>
+            </li>
+            <li>
+              <Link onClick={() => importActions.showChangedElements(false)}>
+                {gettext('Hide changed')}
+              </Link>
+            </li>
+            </ul>
+          }
           <li>
             <Link onClick={() => importActions.showElements(false)}>
               {gettext('Hide all')}
diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index 909af7a103..8c37fcd396 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -1,6 +1,7 @@
 import isArray from 'lodash/isArray'
 import isNil from 'lodash/isNil'
 import isUndefined from 'lodash/isUndefined'
+import { isEmpty } from 'lodash'
 
 import { buildUri } from '../utils/elements'
 
@@ -51,12 +52,30 @@ export default function importsReducer(state = initialState, action) {
       }
     case 'import/selectElements':
       return {...state, elements: state.elements.map(element => {
-        return {...element, import: action.value}
+          return {...element, import: action.value}
       })}
+    case 'import/selectChangedElements':
+      return {...state, elements: state.elements.map(element => {
+        if (element.updated && !isEmpty(element.updated_and_changed) && !element.created ) {
+          return {...element, import: action.value}
+        }
+        else if (action.value) {return {...element, import: !action.value}}
+          else { return element }
+      }
+      )}
     case 'import/showElements':
       return {...state, elements: state.elements.map(element => {
         return {...element, show: action.value}
       })}
+    case 'import/showChangedElements':
+      return {...state, elements: state.elements.map(element => {
+        if (element.updated && !isEmpty(element.updated_and_changed) && !element.created ) {
+          return {...element, show: action.value}
+        }
+        else if (action.value) {return {...element, show: !action.value}}
+        else { return element }
+      }
+      )}
     case 'import/updateUriPrefix':
       elements = state.elements.map(element => {
         element.uri_prefix = action.uriPrefix

From c09a50cb33e25f81104063bbcc6b96f62f4c6095 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 22 Jan 2024 09:16:29 +0100
Subject: [PATCH 049/205] refactor: remove specific Import components, replaced
 by ImportElement

---
 .../js/components/import/ImportAttribute.js   | 54 -------------------
 .../js/components/import/ImportCatalog.js     | 52 ------------------
 .../js/components/import/ImportCondition.js   | 50 -----------------
 .../js/components/import/ImportOption.js      | 50 -----------------
 .../js/components/import/ImportOptionSet.js   | 50 -----------------
 .../assets/js/components/import/ImportPage.js | 50 -----------------
 .../js/components/import/ImportQuestion.js    | 50 -----------------
 .../js/components/import/ImportQuestionSet.js | 50 -----------------
 .../js/components/import/ImportSection.js     | 50 -----------------
 .../assets/js/components/import/ImportTask.js | 52 ------------------
 .../assets/js/components/import/ImportView.js | 52 ------------------
 11 files changed, 560 deletions(-)
 delete mode 100644 rdmo/management/assets/js/components/import/ImportAttribute.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportCatalog.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportCondition.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportOption.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportOptionSet.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportPage.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportQuestion.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportQuestionSet.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportSection.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportTask.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportView.js

diff --git a/rdmo/management/assets/js/components/import/ImportAttribute.js b/rdmo/management/assets/js/components/import/ImportAttribute.js
deleted file mode 100644
index c680d82a50..0000000000
--- a/rdmo/management/assets/js/components/import/ImportAttribute.js
+++ /dev/null
@@ -1,54 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportAttribute = ({ config, attribute, importActions }) => {
-  const showFields = () => importActions.updateElement(attribute, {show: !attribute.show})
-  const toggleImport = () => importActions.updateElement(attribute, {import: !attribute.import})
-  const updateAttribute = (key, value) => importActions.updateElement(attribute, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <WarningLink element={attribute} onClick={showFields} />
-        <ErrorLink element={attribute} onClick={showFields} />
-        <ShowLink element={attribute} onClick={showFields} />
-        {
-          attribute.updated && !attribute.created &&
-          <p className="element-link fa fa-pencil"></p>
-          }
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={attribute.import} onChange={toggleImport} />
-          <strong>{gettext('Attribute')}</strong>
-        </label>
-        <CodeLink className={codeClass[attribute.model]} uri={attribute.uri} onClick={showFields} />
-      </div>
-      {
-        attribute.show && <>
-          <Form config={config} element={attribute} updateElement={updateAttribute} />
-          <Fields element={attribute} />
-          <Warnings element={attribute} />
-          <Errors element={attribute} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportAttribute.propTypes = {
-  config: PropTypes.object.isRequired,
-  attribute: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportAttribute
diff --git a/rdmo/management/assets/js/components/import/ImportCatalog.js b/rdmo/management/assets/js/components/import/ImportCatalog.js
deleted file mode 100644
index 2e564e3b69..0000000000
--- a/rdmo/management/assets/js/components/import/ImportCatalog.js
+++ /dev/null
@@ -1,52 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { AvailableLink, CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportCatalog = ({ config, catalog, importActions }) => {
-  const showFields = () => importActions.updateElement(catalog, {show: !catalog.show})
-  const toggleImport = () => importActions.updateElement(catalog, {import: !catalog.import})
-  const toggleAvailable = () => importActions.updateElement(catalog, {available: !catalog.available})
-  const updateCatalog = (key, value) => importActions.updateElement(catalog, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <AvailableLink element={catalog} verboseName={gettext('catalog')} onClick={toggleAvailable} />
-        <WarningLink element={catalog} onClick={showFields} />
-        <ErrorLink element={catalog} onClick={showFields} />
-        <ShowLink element={catalog} onClick={showFields} />
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={catalog.import} onChange={toggleImport} />
-          <strong>{gettext('Catalog')}</strong>
-        </label>
-        <CodeLink className={codeClass[catalog.model]} uri={catalog.uri} onClick={showFields} />
-      </div>
-      {
-        catalog.show && <>
-          <Form config={config} element={catalog} updateElement={updateCatalog} />
-          <Fields element={catalog} />
-          <Warnings element={catalog} />
-          <Errors element={catalog} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportCatalog.propTypes = {
-  config: PropTypes.object.isRequired,
-  catalog: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportCatalog
diff --git a/rdmo/management/assets/js/components/import/ImportCondition.js b/rdmo/management/assets/js/components/import/ImportCondition.js
deleted file mode 100644
index 58598f2429..0000000000
--- a/rdmo/management/assets/js/components/import/ImportCondition.js
+++ /dev/null
@@ -1,50 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportCondition = ({ config, condition, importActions }) => {
-  const showFields = () => importActions.updateElement(condition, {show: !condition.show})
-  const toggleImport = () => importActions.updateElement(condition, {import: !condition.import})
-  const updateCondition = (key, value) => importActions.updateElement(condition, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <WarningLink element={condition} onClick={showFields} />
-        <ErrorLink element={condition} onClick={showFields} />
-        <ShowLink element={condition} onClick={showFields} />
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={condition.import} onChange={toggleImport} />
-          <strong>{gettext('Condition')}</strong>
-        </label>
-        <CodeLink className={codeClass[condition.model]} uri={condition.uri} onClick={showFields} />
-      </div>
-      {
-        condition.show && <>
-          <Form config={config} element={condition} updateElement={updateCondition} />
-          <Fields element={condition} />
-          <Warnings element={condition} />
-          <Errors element={condition} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportCondition.propTypes = {
-  config: PropTypes.object.isRequired,
-  condition: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportCondition
diff --git a/rdmo/management/assets/js/components/import/ImportOption.js b/rdmo/management/assets/js/components/import/ImportOption.js
deleted file mode 100644
index af8f0166ac..0000000000
--- a/rdmo/management/assets/js/components/import/ImportOption.js
+++ /dev/null
@@ -1,50 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportOption = ({ config, option, importActions }) => {
-  const showFields = () => importActions.updateElement(option, {show: !option.show})
-  const toggleImport = () => importActions.updateElement(option, {import: !option.import})
-  const updateOption = (key, value) => importActions.updateElement(option, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <WarningLink element={option} onClick={showFields} />
-        <ErrorLink element={option} onClick={showFields} />
-        <ShowLink element={option} onClick={showFields} />
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={option.import} onChange={toggleImport} />
-          <strong>{gettext('Option')}</strong>
-        </label>
-        <CodeLink className={codeClass[option.model]} uri={option.uri} onClick={showFields} />
-      </div>
-      {
-        option.show && <>
-          <Form config={config} element={option} updateElement={updateOption} />
-          <Fields element={option} />
-          <Warnings element={option} />
-          <Errors element={option} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportOption.propTypes = {
-  config: PropTypes.object.isRequired,
-  option: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportOption
diff --git a/rdmo/management/assets/js/components/import/ImportOptionSet.js b/rdmo/management/assets/js/components/import/ImportOptionSet.js
deleted file mode 100644
index e7798c051d..0000000000
--- a/rdmo/management/assets/js/components/import/ImportOptionSet.js
+++ /dev/null
@@ -1,50 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportOptionSet = ({ config, optionset, importActions }) => {
-  const showFields = () => importActions.updateElement(optionset, {show: !optionset.show})
-  const toggleImport = () => importActions.updateElement(optionset, {import: !optionset.import})
-  const updateOptionSet = (key, value) => importActions.updateElement(optionset, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <WarningLink element={optionset} onClick={showFields} />
-        <ErrorLink element={optionset} onClick={showFields} />
-        <ShowLink element={optionset} onClick={showFields} />
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={optionset.import} onChange={toggleImport} />
-          <strong>{gettext('Option set')}</strong>
-        </label>
-        <CodeLink className={codeClass[optionset.model]} uri={optionset.uri} onClick={showFields} />
-      </div>
-      {
-        optionset.show && <>
-          <Form config={config} element={optionset} updateElement={updateOptionSet} />
-          <Fields element={optionset} />
-          <Warnings element={optionset} />
-          <Errors element={optionset} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportOptionSet.propTypes = {
-  config: PropTypes.object.isRequired,
-  optionset: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportOptionSet
diff --git a/rdmo/management/assets/js/components/import/ImportPage.js b/rdmo/management/assets/js/components/import/ImportPage.js
deleted file mode 100644
index 2fb438f904..0000000000
--- a/rdmo/management/assets/js/components/import/ImportPage.js
+++ /dev/null
@@ -1,50 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportPage = ({ config, page, importActions }) => {
-  const showFields = () => importActions.updateElement(page, {show: !page.show})
-  const toggleImport = () => importActions.updateElement(page, {import: !page.import})
-  const updatePage = (key, value) => importActions.updateElement(page, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <WarningLink element={page} onClick={showFields} />
-        <ErrorLink element={page} onClick={showFields} />
-        <ShowLink element={page} onClick={showFields} />
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={page.import} onChange={toggleImport} />
-          <strong>{gettext('Page')}</strong>
-        </label>
-        <CodeLink className={codeClass[page.model]} uri={page.uri} onClick={showFields} />
-      </div>
-      {
-        page.show && <>
-          <Form config={config} element={page} updateElement={updatePage} />
-          <Fields element={page} />
-          <Warnings element={page} />
-          <Errors element={page} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportPage.propTypes = {
-  config: PropTypes.object.isRequired,
-  page: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportPage
diff --git a/rdmo/management/assets/js/components/import/ImportQuestion.js b/rdmo/management/assets/js/components/import/ImportQuestion.js
deleted file mode 100644
index 12c757ee28..0000000000
--- a/rdmo/management/assets/js/components/import/ImportQuestion.js
+++ /dev/null
@@ -1,50 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportQuestion = ({ config, question, importActions }) => {
-  const showFields = () => importActions.updateElement(question, {show: !question.show})
-  const toggleImport = () => importActions.updateElement(question, {import: !question.import})
-  const updateQuestion = (key, value) => importActions.updateElement(question, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <WarningLink element={question} onClick={showFields} />
-        <ErrorLink element={question} onClick={showFields} />
-        <ShowLink element={question} onClick={showFields} />
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={question.import} onChange={toggleImport} />
-          <strong>{gettext('Question')}</strong>
-        </label>
-        <CodeLink className={codeClass[question.model]} uri={question.uri} onClick={showFields} />
-      </div>
-      {
-        question.show && <>
-          <Form config={config} element={question} updateElement={updateQuestion} />
-          <Fields element={question} />
-          <Warnings element={question} />
-          <Errors element={question} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportQuestion.propTypes = {
-  config: PropTypes.object.isRequired,
-  question: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportQuestion
diff --git a/rdmo/management/assets/js/components/import/ImportQuestionSet.js b/rdmo/management/assets/js/components/import/ImportQuestionSet.js
deleted file mode 100644
index 8163015cbe..0000000000
--- a/rdmo/management/assets/js/components/import/ImportQuestionSet.js
+++ /dev/null
@@ -1,50 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportQuestionSet = ({ config, questionset, importActions }) => {
-  const showFields = () => importActions.updateElement(questionset, {show: !questionset.show})
-  const toggleImport = () => importActions.updateElement(questionset, {import: !questionset.import})
-  const updateQuestionSet = (key, value) => importActions.updateElement(questionset, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <WarningLink element={questionset} onClick={showFields} />
-        <ErrorLink element={questionset} onClick={showFields} />
-        <ShowLink element={questionset} onClick={showFields} />
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={questionset.import} onChange={toggleImport} />
-          <strong>{gettext('Question set')}</strong>
-        </label>
-        <CodeLink className={codeClass[questionset.model]} uri={questionset.uri} onClick={showFields} />
-      </div>
-      {
-        questionset.show && <>
-          <Form config={config} element={questionset} updateElement={updateQuestionSet} />
-          <Fields element={questionset} />
-          <Warnings element={questionset} />
-          <Errors element={questionset} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportQuestionSet.propTypes = {
-  config: PropTypes.object.isRequired,
-  questionset: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportQuestionSet
diff --git a/rdmo/management/assets/js/components/import/ImportSection.js b/rdmo/management/assets/js/components/import/ImportSection.js
deleted file mode 100644
index 92203a8771..0000000000
--- a/rdmo/management/assets/js/components/import/ImportSection.js
+++ /dev/null
@@ -1,50 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportSection = ({ config, section, importActions }) => {
-  const showFields = () => importActions.updateElement(section, {show: !section.show})
-  const toggleImport = () => importActions.updateElement(section, {import: !section.import})
-  const updateSection = (key, value) => importActions.updateElement(section, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <WarningLink element={section} onClick={showFields} />
-        <ErrorLink element={section} onClick={showFields} />
-        <ShowLink element={section} onClick={showFields} />
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={section.import} onChange={toggleImport} />
-          <strong>{gettext('Section')}</strong>
-        </label>
-        <CodeLink className={codeClass[section.model]} uri={section.uri} onClick={showFields} />
-      </div>
-      {
-        section.show && <>
-          <Form config={config} element={section} updateElement={updateSection} />
-          <Fields element={section} />
-          <Warnings element={section} />
-          <Errors element={section} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportSection.propTypes = {
-  config: PropTypes.object.isRequired,
-  section: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportSection
diff --git a/rdmo/management/assets/js/components/import/ImportTask.js b/rdmo/management/assets/js/components/import/ImportTask.js
deleted file mode 100644
index 72b2b73158..0000000000
--- a/rdmo/management/assets/js/components/import/ImportTask.js
+++ /dev/null
@@ -1,52 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { AvailableLink, CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportTask = ({ config, task, importActions }) => {
-  const showFields = () => importActions.updateElement(task, {show: !task.show})
-  const toggleImport = () => importActions.updateElement(task, {import: !task.import})
-  const toggleAvailable = () => importActions.updateElement(task, {available: !task.available})
-  const updateTask = (key, value) => importActions.updateElement(task, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <AvailableLink element={task} verboseName={gettext('task')} onClick={toggleAvailable} />
-        <WarningLink element={task} onClick={showFields} />
-        <ErrorLink element={task} onClick={showFields} />
-        <ShowLink element={task} onClick={showFields} />
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={task.import} onChange={toggleImport} />
-          <strong>{gettext('Task')}</strong>
-        </label>
-        <CodeLink className={codeClass[task.model]} uri={task.uri} onClick={showFields} />
-      </div>
-      {
-        task.show && <>
-          <Form config={config} element={task} updateElement={updateTask} />
-          <Fields element={task} />
-          <Warnings element={task} />
-          <Errors element={task} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportTask.propTypes = {
-  config: PropTypes.object.isRequired,
-  task: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportTask
diff --git a/rdmo/management/assets/js/components/import/ImportView.js b/rdmo/management/assets/js/components/import/ImportView.js
deleted file mode 100644
index e48196efb5..0000000000
--- a/rdmo/management/assets/js/components/import/ImportView.js
+++ /dev/null
@@ -1,52 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import { AvailableLink, CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
-
-import Errors from './common/Errors'
-import Fields from './common/Fields'
-import Form from './common/Form'
-import Warnings from './common/Warnings'
-
-import { codeClass } from '../../constants/elements'
-
-const ImportView = ({ config, view, importActions }) => {
-  const showFields = () => importActions.updateElement(view, {show: !view.show})
-  const toggleImport = () => importActions.updateElement(view, {import: !view.import})
-  const toggleAvailable = () => importActions.updateElement(view, {available: !view.available})
-  const updateView = (key, value) => importActions.updateElement(view, {[key]: value})
-
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <AvailableLink element={view} verboseName={gettext('view')} onClick={toggleAvailable} />
-        <WarningLink element={view} onClick={showFields} />
-        <ErrorLink element={view} onClick={showFields} />
-        <ShowLink element={view} onClick={showFields} />
-      </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={view.import} onChange={toggleImport} />
-          <strong>{gettext('View')}</strong>
-        </label>
-        <CodeLink className={codeClass[view.model]} uri={view.uri} onClick={showFields} />
-      </div>
-      {
-        view.show && <>
-          <Form config={config} element={view} updateElement={updateView} />
-          <Fields element={view} />
-          <Warnings element={view} />
-          <Errors element={view} />
-        </>
-      }
-    </li>
-  )
-}
-
-ImportView.propTypes = {
-  config: PropTypes.object.isRequired,
-  view: PropTypes.object.isRequired,
-  importActions: PropTypes.object.isRequired
-}
-
-export default ImportView

From 31ae41d7218494551b445cbaa26dbdfed50868b2 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 22 Jan 2024 18:39:26 +0100
Subject: [PATCH 050/205] chore: update js import Fields and FieldsDiff

---
 .../assets/js/components/import/common/Fields.js |  4 ++--
 .../js/components/import/common/FieldsDiffs.js   | 16 +++++++---------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index f0147d4910..a7c63716ce 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -56,8 +56,8 @@ const Fields = ({ element }) => {
                   }
                 </div>
                 {
-                  isEmpty(element.errors) && !isEmpty(element.original) && element.updated &&
-                  element.original[key] != value && <FieldsDiffs element={element} field={key}/>
+                  isEmpty(element.errors) && !isEmpty(element.updated_and_changed) && element.updated &&
+                  key in element.updated_and_changed && <FieldsDiffs element={element} field={key}/>
                   }
               </div>
             )
diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index 882a03c74e..020686f342 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -6,18 +6,16 @@ import { isUndefined } from 'lodash'
 
 
 const FieldsDiffs = ({ element, field }) => {
-
+  const newVal = element.updated_and_changed[field].uploaded
+  const oldVal = element.updated_and_changed[field].current
   return !isUndefined(element) &&
-          !isUndefined(element[field]) &&
-          !isEmpty(element.original) &&
-          !isEmpty(element.original[field]) &&
-          isEmpty(element.errors) &&
-          typeof(element[field]) === 'string' &&
-          typeof(element.original[field]) === 'string' &&
+        !isEmpty(oldVal) &&
+        !isEmpty(element.updated_and_changed) &&
+        !isUndefined(newVal) &&
    <div className="col-sm-12">
     <ReactDiffViewer
-        oldValue={element.original[field]}
-        newValue={element[field]}
+        oldValue={oldVal}
+        newValue={newVal}
         splitView={true}
         hideLineNumbers={true}
         leftTitle={gettext('Current')}

From 30280b6492fed37c86336c022c93716138b7865d Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 22 Jan 2024 18:46:09 +0100
Subject: [PATCH 051/205] refactor: add serializer arg to ElementImportHelper

---
 rdmo/conditions/imports.py |  6 +++--
 rdmo/core/imports.py       | 48 ++------------------------------------
 rdmo/domain/imports.py     |  6 +++--
 rdmo/options/imports.py    | 13 +++++++----
 rdmo/questions/imports.py  | 38 +++++++++++++++++++-----------
 rdmo/tasks/imports.py      |  6 +++--
 rdmo/views/imports.py      |  6 +++--
 7 files changed, 51 insertions(+), 72 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index eb19428dc6..c6a28448be 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -3,13 +3,14 @@
 
 from django.contrib.sites.models import Site
 
-from rdmo.conditions.validators import ConditionLockedValidator, ConditionUniqueURIValidator
 from rdmo.core.imports import (
     ElementImportHelper,
     validate_instance,
 )
 
 from .models import Condition
+from .serializers.v1 import ConditionSerializer
+from .validators import ConditionLockedValidator, ConditionUniqueURIValidator
 
 logger = logging.getLogger(__name__)
 
@@ -43,5 +44,6 @@ def import_condition(
     import_func=import_condition,
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
     lang_fields=[],
-    foreign_fields=('source', 'target_option')
+    foreign_fields=('source', 'target_option'),
+    serializer=ConditionSerializer
 )
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 31596f23f6..c1b63b3d1c 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -5,11 +5,10 @@
 from os.path import join as pj
 from pathlib import Path
 from random import randint
-from typing import Callable, Dict, Iterable, List, Optional, Sequence, Tuple
+from typing import Callable, Iterable, Optional, Sequence, Tuple
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
-from django.forms.models import model_to_dict
 
 from rest_framework.utils import model_meta
 
@@ -73,6 +72,7 @@ class ElementImportHelper:
     model: str
     import_func: Callable
     validators: Iterable[Callable]
+    serializer: Callable
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
     lang_fields: Sequence[str] = field(default_factory=list)
     foreign_fields: Sequence[str] = field(default_factory=list)
@@ -313,47 +313,3 @@ def check_permissions(instance: models.Model, element_uri: str, user: models.Mod
         message = f'You have no permissions to import {instance._meta.object_name} {element_uri}.'
         logger.info(message)
         return message
-
-
-def prepare_element_from_original_instance(element: Dict,
-                                        original_instance: models.Model,
-                                        lang_field_names: List[str],
-                                        foreign_field_names: List[str]):
-    original_element = model_to_dict(original_instance)
-
-    filtered_ffnames = filter(lambda x: x in original_element, foreign_field_names)
-    for _field in filtered_ffnames:
-        if original_element[_field] is None:
-            continue
-        try:
-            # set the uri for foreign fields, instead of id
-            original_element[_field] = {'uri': getattr(original_instance, _field).uri}
-        except AttributeError:
-            pass
-    for lang_field_name in lang_field_names:
-        # add the lang_code fields from the original instance
-        lang_field_values = get_lang_field_values(lang_field_name, instance=original_instance)
-        original_element.update(lang_field_values)
-    return original_element
-
-
-def get_original_and_updated(original_element: Dict, instance: models.Model, foreign_field_names: List[str]) -> Dict:
-
-    # add updated and changed
-    instance_field_names = {i.name for i in instance._meta.local_concrete_fields}
-    updated_and_changed = {}
-    for k, val in filter(lambda x: x[0] in instance_field_names, original_element.items()):
-
-        new_val = getattr(instance, k, None)
-        if k in foreign_field_names and new_val is not None:
-            try:
-                # set the uri for foreign fields, instead of id
-                new_val = {'uri': getattr(instance, k).uri}
-            except AttributeError:
-                pass
-
-        if new_val is None:
-            continue
-        if new_val != val:
-            updated_and_changed[k] = {"current": val, "uploaded": new_val}
-    return updated_and_changed
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index b17ff29b50..910cc24680 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -7,9 +7,10 @@
     ElementImportHelper,
     validate_instance,
 )
-from rdmo.domain.validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
 
 from .models import Attribute
+from .serializers.v1 import BaseAttributeSerializer
+from .validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
 
 logger = logging.getLogger(__name__)
 
@@ -40,5 +41,6 @@ def import_attribute(
     import_func=import_attribute,
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     lang_fields=[],
-    foreign_fields=('parent',)
+    foreign_fields=('parent',),
+    serializer=BaseAttributeSerializer
 )
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 380d344de8..e1b4104aaa 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -11,15 +11,16 @@
     set_reverse_m2m_through_instance,
     validate_instance,
 )
-from rdmo.options.validators import (
+
+from .models import Option, OptionSet
+from .serializers.v1 import OptionSerializer, OptionSetSerializer
+from .validators import (
     OptionLockedValidator,
     OptionSetLockedValidator,
     OptionSetUniqueURIValidator,
     OptionUniqueURIValidator,
 )
 
-from .models import Option, OptionSet
-
 logger = logging.getLogger(__name__)
 
 
@@ -53,7 +54,8 @@ def import_option(
     model="options.option",
     import_func=import_option,
     validators=(OptionLockedValidator, OptionUniqueURIValidator),
-    lang_fields=('text',)
+    lang_fields=('text',),
+    serializer = OptionSerializer
 )
 
 
@@ -87,5 +89,6 @@ def import_optionset(
     model="options.optionset",
     import_func=import_optionset,
     validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
-    lang_fields=[]
+    lang_fields=[],
+    serializer = OptionSetSerializer
 )
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 5a92a32db3..b9557e735d 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -11,7 +11,21 @@
     set_reverse_m2m_through_instance,
     validate_instance,
 )
-from rdmo.questions.validators import (
+
+from .models.catalog import Catalog
+from .models.page import Page
+from .models.question import Question
+from .models.questionset import QuestionSet
+from .models.section import Section
+from .serializers.v1 import (
+    CatalogSerializer,
+    PageSerializer,
+    QuestionSerializer,
+    QuestionSetSerializer,
+    SectionSerializer,
+)
+from .utils import get_widget_types
+from .validators import (
     CatalogLockedValidator,
     CatalogUniqueURIValidator,
     PageLockedValidator,
@@ -24,13 +38,6 @@
     SectionUniqueURIValidator,
 )
 
-from .models.catalog import Catalog
-from .models.page import Page
-from .models.question import Question
-from .models.questionset import QuestionSet
-from .models.section import Section
-from .utils import get_widget_types
-
 logger = logging.getLogger(__name__)
 
 
@@ -189,14 +196,16 @@ def import_question(
     model="questions.catalog",
     import_func=import_catalog,
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
-    lang_fields=('help', 'title')
+    lang_fields=('help', 'title'),
+    serializer = CatalogSerializer
 )
 
 import_helper_section = ElementImportHelper(
     model="questions.section",
     import_func=import_section,
     validators=(SectionLockedValidator, SectionUniqueURIValidator),
-    lang_fields=('title',)
+    lang_fields=('title',),
+    serializer = SectionSerializer
 )
 
 import_helper_page = ElementImportHelper(
@@ -204,7 +213,8 @@ def import_question(
     import_func=import_page,
     validators=(PageLockedValidator, PageUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
-    foreign_fields=('attribute',)
+    foreign_fields=('attribute',),
+    serializer = PageSerializer
 )
 
 import_helper_questionset = ElementImportHelper(
@@ -212,7 +222,8 @@ def import_question(
     import_func=import_questionset,
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
-    foreign_fields=('attribute',)
+    foreign_fields=('attribute',),
+    serializer = QuestionSetSerializer
 )
 
 import_helper_question = ElementImportHelper(
@@ -220,5 +231,6 @@ def import_question(
     import_func=import_question,
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
     lang_fields=('text', 'help', 'default_text', 'verbose_name'),
-    foreign_fields=('attribute','default_option')
+    foreign_fields=('attribute','default_option'),
+    serializer = QuestionSerializer
 )
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 2175848e31..9c29efd1ea 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -9,9 +9,10 @@
     set_m2m_instances,
     validate_instance,
 )
-from rdmo.tasks.validators import TaskLockedValidator, TaskUniqueURIValidator
 
 from .models import Task
+from .serializers.v1 import TaskSerializer
+from .validators import TaskLockedValidator, TaskUniqueURIValidator
 
 logger = logging.getLogger(__name__)
 
@@ -54,5 +55,6 @@ def import_task(
     import_func=import_task,
     validators=(TaskLockedValidator, TaskUniqueURIValidator),
     lang_fields=('title', 'text'),
-    foreign_fields=('start_attribute', 'end_attribute')
+    foreign_fields=('start_attribute', 'end_attribute'),
+    serializer=TaskSerializer
 )
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 6f5b9c597c..a9ca0cb1ad 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -8,9 +8,10 @@
     set_m2m_instances,
     validate_instance,
 )
-from rdmo.views.validators import ViewLockedValidator, ViewUniqueURIValidator
 
 from .models import View
+from .serializers.v1 import ViewSerializer
+from .validators import ViewLockedValidator, ViewUniqueURIValidator
 
 logger = logging.getLogger(__name__)
 
@@ -46,5 +47,6 @@ def import_view(
     model="views.view",
     import_func=import_view,
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
-    lang_fields=( 'help', 'title')
+    lang_fields=( 'help', 'title'),
+    serializer=ViewSerializer
 )

From 71b42d3e8adf8d3701942b0a8d6e070a0af48f12 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 22 Jan 2024 18:47:30 +0100
Subject: [PATCH 052/205] chore: let read only serializer handle request is
 None

---
 rdmo/core/serializers.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/rdmo/core/serializers.py b/rdmo/core/serializers.py
index 1c06cc8a50..74b060d7da 100644
--- a/rdmo/core/serializers.py
+++ b/rdmo/core/serializers.py
@@ -216,7 +216,10 @@ def construct_object_permission(model, action_name: str) -> str:
         return perm
 
     def get_read_only(self, obj) -> bool:
-        user = self.context['request'].user
+        request = self.context.get('request')
+        if request is None:
+            return False
+        user = request.user
         perms = (self.construct_object_permission(self.Meta.model, action_name)
                  for action_name in self.OBJECT_PERMISSION_ACTION_NAMES)
         return not all(user.has_perm(perm, obj) for perm in perms)

From 5ad91e2a41c273e9ad7ca2a070a482f409885cd9 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 22 Jan 2024 18:48:04 +0100
Subject: [PATCH 053/205] refactor: pass request instead user to
 import_elements

---
 rdmo/management/viewsets.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index f1db91c56c..889ef5b1aa 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -73,7 +73,7 @@ def create(self, request, *args, **kwargs):
         _elements = list(elements.values())
 
         # step 8: import the elements if save=True is set
-        imported_elements = import_elements(_elements, save=is_truthy(request.POST.get('import')), user=request.user)
+        imported_elements = import_elements(_elements, save=is_truthy(request.POST.get('import')), request=request)
 
         # step 9: return the list of, json-serializable, elements
         return Response(imported_elements)
@@ -93,7 +93,7 @@ def create(self, request, *args, **kwargs):
             raise ValidationError({'elements': [_('This is not a valid RDMO import JSON.')]}) from e
 
         # step 3: import the elements
-        imported_elements = import_elements(elements, user=request.user)
+        imported_elements = import_elements(elements, request=request)
 
         # step 4: return the list of elements
         return Response(imported_elements)

From b129f158007dbb1fdc52e9ff447a84cdfd89243f Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 22 Jan 2024 18:50:46 +0100
Subject: [PATCH 054/205] refactor: use serializers to make diff check
 updated_and_changed, fix warnings and use request

---
 rdmo/management/imports.py | 51 ++++++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index dfbf11c35c..460c3b1d6d 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -3,15 +3,13 @@
 from collections import defaultdict
 from typing import Dict, List, Optional, Sequence
 
-from django.db import models
+from django.http import HttpRequest
 
 from rdmo.conditions.imports import import_helper_condition
 from rdmo.core.imports import (
     check_permissions,
     get_or_return_instance,
-    get_original_and_updated,
     make_import_info_msg,
-    prepare_element_from_original_instance,
     set_foreign_field,
     set_lang_field,
 )
@@ -56,14 +54,12 @@
     }
 
 
-def import_elements(uploaded_elements: List[Dict], save: bool = True, user: Optional[models.Model] = None):
+def import_elements(uploaded_elements: List[Dict], save: bool = True, request: Optional[HttpRequest] = None):
     imported_elements = []
     uploaded_uris = {i.get('uri') for i in uploaded_elements}
     for uploaded_element in uploaded_elements:
-        element = import_element(element=uploaded_element, save=save, user=user, uploaded_uris=imported_elements)
-        # replace warnings with filtered flat list of warnings
-        filtered_warnings = set(filter(lambda k: k not in uploaded_uris, element['warnings'].keys()))
-        element['warnings'] = [val for k,val in element['warnings'].items() if k not in filtered_warnings]
+        element = import_element(element=uploaded_element, save=save, request=request, uploaded_uris=imported_elements)
+        element['warnings'] = [val for k, val in element['warnings'].items() if k not in uploaded_uris]
         imported_elements.append(element)
     return imported_elements
 
@@ -72,12 +68,12 @@ def import_elements(uploaded_elements: List[Dict], save: bool = True, user: Opti
 def import_element(
         element: Optional[Dict] = None,
         save: bool = True,
-        user: Optional[models.Model] = None,
+        request: Optional[HttpRequest] = None,
         uploaded_uris: Optional[Sequence[str]] = None
     ):
 
     if element is None:
-        return element
+        return
 
     model_path = element.get('model')
     if model_path is None:
@@ -88,6 +84,7 @@ def import_element(
         element[_k] = _val()
 
     model = RDMO_MODEL_PATH_MAPPER[model_path]
+    user = request.user if request is not None else None
     import_helper = ELEMENT_IMPORT_HELPERS[model_path]
     import_func = import_helper.import_func
     validators = import_helper.validators
@@ -133,18 +130,36 @@ def import_element(
 
     if element.get('errors'):
         return element
-
     if _updated and not _created:
         element['updated'] = _updated
-        original_element = {}
-
         # and instance is not original_instance
         # keep only strings, make json serializable
-        original_element = prepare_element_from_original_instance(element, original_instance,
-                                                                    lang_field_names, foreign_field_names)
-        original_element_json = {k: val for k, val in original_element.items() if isinstance(val, str)}
-        element['original'] = original_element_json
-        updated_and_changed = get_original_and_updated(original_element, instance, foreign_field_names)
+        original_serializer = import_helper.serializer(original_instance, context={'request': request})
+        original_data = original_serializer.data
+        original_element = {k: val for k,val in original_data.items() if k in element}
+        uploaded_serializer = import_helper.serializer(instance, context={'request': request})
+        uploaded_data = uploaded_serializer.data
+        uploaded_element = {k: val for k,val in uploaded_data.items() if k in original_element}
+
+        updated_and_changed = {}
+        for k, old_val in original_element.items():
+            new_val = uploaded_element[k]
+            if old_val != new_val:
+                updated_and_changed[k] = {"current": old_val, "uploaded": new_val}
+        # overwrite the "normal" element name with the value from element_uri
+        uri_keys = {k for k in list(original_data.keys())+list(uploaded_data.keys())
+                    if k.endswith('_uri') or k.endswith('_uris')}
+        for uri_key in uri_keys:
+            element_name, uri_field = uri_key.split('_')
+            if uri_key in updated_and_changed:
+                # eg. set attribute as key instead of attribute_uri
+                uri_key_val = updated_and_changed[uri_key].pop()
+                updated_and_changed[element_name] = uri_key_val
+            if element_name in element and uri_key in original_data:
+                old_val = original_data[uri_key]
+                new_val = element[element_name].get('uri')
+                if old_val != new_val:
+                    updated_and_changed[element_name] = {"current": old_val, "uploaded": new_val}
 
         element['updated_and_changed'] = updated_and_changed
 

From 296030c51947ecdb5ac77695865349185827ad61 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 24 Jan 2024 11:45:30 +0100
Subject: [PATCH 055/205] refactor: move current site adders to management
 import

---
 rdmo/conditions/imports.py |  4 +-
 rdmo/core/imports.py       |  8 ++--
 rdmo/domain/imports.py     |  8 ++--
 rdmo/management/imports.py | 77 ++++++++++++++++++++++----------------
 rdmo/options/imports.py    |  5 +--
 rdmo/questions/imports.py  | 15 ++++----
 rdmo/tasks/imports.py      |  4 +-
 rdmo/views/imports.py      |  5 +--
 8 files changed, 65 insertions(+), 61 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index c6a28448be..dcc89bbe30 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -1,8 +1,6 @@
 import logging
 from typing import Callable, Tuple
 
-from django.contrib.sites.models import Site
-
 from rdmo.core.imports import (
     ElementImportHelper,
     validate_instance,
@@ -34,7 +32,7 @@ def import_condition(
 
     if save:
         instance.save()
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index c1b63b3d1c..eb0fbdb572 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -5,7 +5,7 @@
 from os.path import join as pj
 from pathlib import Path
 from random import randint
-from typing import Callable, Iterable, Optional, Sequence, Tuple
+from typing import Any, Callable, Iterable, Optional, Sequence, Tuple
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
@@ -51,7 +51,7 @@ def generate_tempfile_name():
     return fn
 
 
-def get_or_return_instance(model: models.Model, uri: Optional[str]=None) -> Tuple[models.Model, bool]:
+def get_or_return_instance(model: Callable, uri: Optional[str]=None) -> Tuple[models.Model, bool]:
     if uri is None:
         return model(), True
     try:
@@ -76,7 +76,9 @@ class ElementImportHelper:
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
     lang_fields: Sequence[str] = field(default_factory=list)
     foreign_fields: Sequence[str] = field(default_factory=list)
-
+    extra_fields: Sequence[Tuple[str, Any]] = field(default_factory=list)
+    add_current_site_editors: bool = True
+    add_current_site_sites: bool = False
 
 
 def get_lang_field_values(field_name: str,
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 910cc24680..4f76eb0849 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -1,8 +1,6 @@
 import logging
 from typing import Callable, Tuple
 
-from django.contrib.sites.models import Site
-
 from rdmo.core.imports import (
     ElementImportHelper,
     validate_instance,
@@ -31,7 +29,7 @@ def import_attribute(
 
     if save:
         instance.save()
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 
@@ -40,7 +38,7 @@ def import_attribute(
     model="domain.attribute",
     import_func=import_attribute,
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
-    lang_fields=[],
     foreign_fields=('parent',),
-    serializer=BaseAttributeSerializer
+    serializer=BaseAttributeSerializer,
+
 )
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 460c3b1d6d..ce29ffc446 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -3,6 +3,7 @@
 from collections import defaultdict
 from typing import Dict, List, Optional, Sequence
 
+from django.contrib.sites.shortcuts import get_current_site
 from django.http import HttpRequest
 
 from rdmo.conditions.imports import import_helper_condition
@@ -49,7 +50,6 @@
         'errors': list,
         'created': bool,
         'updated': bool,
-        'original': dict,
         'updated_and_changed': dict,
     }
 
@@ -57,8 +57,10 @@
 def import_elements(uploaded_elements: List[Dict], save: bool = True, request: Optional[HttpRequest] = None):
     imported_elements = []
     uploaded_uris = {i.get('uri') for i in uploaded_elements}
+    current_site = get_current_site(request)
     for uploaded_element in uploaded_elements:
-        element = import_element(element=uploaded_element, save=save, request=request, uploaded_uris=imported_elements)
+        element = import_element(element=uploaded_element, save=save, uploaded_uris=imported_elements,
+                                    request=request, current_site=current_site)
         element['warnings'] = [val for k, val in element['warnings'].items() if k not in uploaded_uris]
         imported_elements.append(element)
     return imported_elements
@@ -69,8 +71,9 @@ def import_element(
         element: Optional[Dict] = None,
         save: bool = True,
         request: Optional[HttpRequest] = None,
-        uploaded_uris: Optional[Sequence[str]] = None
-    ):
+        uploaded_uris: Optional[Sequence[str]] = None,
+        current_site = None
+    ) -> Dict:
 
     if element is None:
         return
@@ -134,38 +137,48 @@ def import_element(
         element['updated'] = _updated
         # and instance is not original_instance
         # keep only strings, make json serializable
-        original_serializer = import_helper.serializer(original_instance, context={'request': request})
-        original_data = original_serializer.data
-        original_element = {k: val for k,val in original_data.items() if k in element}
-        uploaded_serializer = import_helper.serializer(instance, context={'request': request})
-        uploaded_data = uploaded_serializer.data
-        uploaded_element = {k: val for k,val in uploaded_data.items() if k in original_element}
-
-        updated_and_changed = {}
-        for k, old_val in original_element.items():
-            new_val = uploaded_element[k]
-            if old_val != new_val:
-                updated_and_changed[k] = {"current": old_val, "uploaded": new_val}
-        # overwrite the "normal" element name with the value from element_uri
-        uri_keys = {k for k in list(original_data.keys())+list(uploaded_data.keys())
-                    if k.endswith('_uri') or k.endswith('_uris')}
-        for uri_key in uri_keys:
-            element_name, uri_field = uri_key.split('_')
-            if uri_key in updated_and_changed:
-                # eg. set attribute as key instead of attribute_uri
-                uri_key_val = updated_and_changed[uri_key].pop()
-                updated_and_changed[element_name] = uri_key_val
-            if element_name in element and uri_key in original_data:
-                old_val = original_data[uri_key]
-                new_val = element[element_name].get('uri')
-                if old_val != new_val:
-                    updated_and_changed[element_name] = {"current": old_val, "uploaded": new_val}
-
-        element['updated_and_changed'] = updated_and_changed
+        serializer = import_helper.serializer
+        changes = get_updated_changes(element, instance, original_instance, serializer, request=None)
+        element['updated_and_changed'] = changes
 
     if save:
         logger.info(_msg)
         element['created'] = _created
         element['updated'] = _updated
+        if import_helper.add_current_site_editors:
+            instance.editors.add(current_site)
+        if import_helper.add_current_site_sites:
+            instance.sites.add(current_site)
 
     return element
+
+
+def get_updated_changes(element, new_instance,
+                        original_instance, serializer, request=None) -> Dict[str, str]:
+    original_serializer = serializer(original_instance, context={'request': request})
+    original_data = original_serializer.data
+    original_element = {k: val for k,val in original_data.items() if k in element}
+    uploaded_serializer = serializer(new_instance, context={'request': request})
+    uploaded_data = uploaded_serializer.data
+    uploaded_element = {k: val for k,val in uploaded_data.items() if k in element}
+
+    updated_and_changed = {}
+    for k, old_val in original_element.items():
+        new_val = uploaded_element[k]
+        if old_val != new_val and any([old_val,new_val]):
+            updated_and_changed[k] = {"current": old_val, "uploaded": new_val}
+    # overwrite the normal "element" name with the value from "element_uri"
+    uri_keys = {k for k in list(original_data.keys())+list(uploaded_data.keys())
+                if k.endswith('_uri') or k.endswith('_uris')}
+    for uri_key in uri_keys:
+        element_name, uri_field = uri_key.split('_')
+        if uri_key in updated_and_changed:
+            # eg. set attribute as key instead of attribute_uri
+            uri_key_val = updated_and_changed[uri_key].pop()
+            updated_and_changed[element_name] = uri_key_val
+        if element_name in element and uri_key in original_data:
+            old_val = original_data[uri_key]
+            new_val = element[element_name].get('uri')
+            if old_val != new_val and any([old_val,new_val]):
+                updated_and_changed[element_name] = {"current": old_val, "uploaded": new_val}
+    return updated_and_changed
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index e1b4104aaa..ddb55637b9 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -1,7 +1,6 @@
 import logging
 from typing import Callable, Tuple
 
-from django.contrib.sites.models import Site
 from django.db import models
 
 from rdmo.core.imports import (
@@ -45,7 +44,7 @@ def import_option(
         instance.save()
         set_m2m_instances(instance, 'conditions', element)
         set_m2m_through_instances(instance, 'options', element, 'optionset', 'option', 'optionset_options')
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 
@@ -80,7 +79,7 @@ def import_optionset(
     if save:
         instance.save()
         set_reverse_m2m_through_instance(instance, 'optionset', element, 'option', 'optionset', 'option_optionsets')
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index b9557e735d..97e22084fe 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -1,7 +1,6 @@
 import logging
 from typing import Callable, Tuple
 
-from django.contrib.sites.models import Site
 from django.db import models
 
 from rdmo.core.imports import (
@@ -61,8 +60,7 @@ def import_catalog(
     if save:
         instance.save()
         set_m2m_through_instances(instance, 'sections', element, 'catalog', 'section', 'catalog_sections')
-        instance.sites.add(Site.objects.get_current())
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 
@@ -84,7 +82,7 @@ def import_section(
         instance.save()
         set_reverse_m2m_through_instance(instance, 'catalog', element, 'section', 'catalog', 'section_catalogs')
         set_m2m_through_instances(instance, 'pages', element, 'section', 'page', 'section_pages')
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 
@@ -113,7 +111,7 @@ def import_page(
         set_reverse_m2m_through_instance(instance, 'section', element, 'page', 'section', 'page_sections')
         set_m2m_through_instances(instance, 'questionsets', element, 'page', 'questionset', 'page_questionsets')
         set_m2m_through_instances(instance, 'questions', element, 'page', 'question', 'page_questions')
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 
@@ -143,7 +141,7 @@ def import_questionset(
         set_reverse_m2m_through_instance(instance, 'questionset', element, 'questionset', 'parent', 'questionset_parents')  # noqa: E501
         set_m2m_through_instances(instance, 'questionsets', element, 'parent', 'questionset', 'questionset_questionsets')  # noqa: E501
         set_m2m_through_instances(instance, 'questions', element, 'questionset', 'question', 'questionset_questions')
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 
@@ -187,7 +185,7 @@ def import_question(
         set_reverse_m2m_through_instance(instance, 'questionset', element, 'question', 'questionset', 'question_questionsets')  # noqa: E501
         set_m2m_instances(instance, 'conditions', element)
         set_m2m_instances(instance, 'optionsets', element)
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 
@@ -197,7 +195,8 @@ def import_question(
     import_func=import_catalog,
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
     lang_fields=('help', 'title'),
-    serializer = CatalogSerializer
+    serializer = CatalogSerializer,
+    add_current_site_sites = True
 )
 
 import_helper_section = ElementImportHelper(
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 9c29efd1ea..9be1352096 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -1,7 +1,6 @@
 import logging
 from typing import Callable, Tuple
 
-from django.contrib.sites.models import Site
 from django.db import models
 
 from rdmo.core.imports import (
@@ -44,8 +43,7 @@ def import_task(
         instance.save()
         set_m2m_instances(instance, 'catalogs', element)
         set_m2m_instances(instance, 'conditions', element)
-        instance.sites.add(Site.objects.get_current())
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index a9ca0cb1ad..52639fe789 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -1,8 +1,6 @@
 import logging
 from typing import Callable, Tuple
 
-from django.contrib.sites.models import Site
-
 from rdmo.core.imports import (
     ElementImportHelper,
     set_m2m_instances,
@@ -37,8 +35,7 @@ def import_view(
     if save:
         instance.save()
         set_m2m_instances(instance, 'catalogs', element)
-        instance.sites.add(Site.objects.get_current())
-        instance.editors.add(Site.objects.get_current())
+        # sites and editors are added in management/import.py
 
     return instance
 

From 35b97e687f97397a2e5a4b976b694a3f9fc8268c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 24 Jan 2024 11:55:37 +0100
Subject: [PATCH 056/205] js remove original field

---
 rdmo/management/assets/js/components/import/common/Fields.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index a7c63716ce..3d543d395a 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -25,7 +25,6 @@ const excludeKeys = [
   'uri_prefix',
   'valid',
   'warnings',
-  'original',
   'updated_and_changed',
 ]
 

From ec0eefeb3b99f4dd3bc834454ff068e56f993b3f Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 24 Jan 2024 13:18:50 +0100
Subject: [PATCH 057/205] refactor: move extra fields setter to management
 import

---
 rdmo/conditions/imports.py |  8 +++---
 rdmo/core/constants.py     | 23 ++++++++++++++++
 rdmo/core/imports.py       | 26 +++++++++++-------
 rdmo/domain/imports.py     |  2 --
 rdmo/management/imports.py | 18 ++++++++++---
 rdmo/options/imports.py    | 21 +++++----------
 rdmo/questions/imports.py  | 54 +++++++++-----------------------------
 rdmo/tasks/imports.py      | 15 +++--------
 rdmo/views/imports.py      | 12 +++------
 9 files changed, 82 insertions(+), 97 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index dcc89bbe30..c15f4ccba4 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -22,9 +22,7 @@ def import_condition(
 
     # set_foreign_field are already set in management/import.py
     # check_permissions already done in management/import.py
-    instance.relation = element.get('relation') or ''
-    instance.target_text = element.get('target_text') or ''
-
+    # extra_fields are set in in management/import.py
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
@@ -41,7 +39,7 @@ def import_condition(
     model="conditions.condition",
     import_func=import_condition,
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
-    lang_fields=[],
     foreign_fields=('source', 'target_option'),
-    serializer=ConditionSerializer
+    serializer=ConditionSerializer,
+    extra_fields=('relation', 'target_text')
 )
diff --git a/rdmo/core/constants.py b/rdmo/core/constants.py
index 1e1cbc3a78..0453c4ca65 100644
--- a/rdmo/core/constants.py
+++ b/rdmo/core/constants.py
@@ -77,3 +77,26 @@
     "tib": {"base": 1024, "power": 4},
     "pib": {"base": 1024, "power": 5},
 }
+
+ELEMENT_COMMON_FIELDS = (
+    'uri_prefix',
+    'uri_path',
+    'key',
+    'comment',
+)
+
+ELEMENT_IMPORT_EXTRA_FIELDS_DEFAULTS = {
+    'order': 0,
+    'available': True,
+    'template': '',
+    'relation': '',
+    'target_text': '',
+    'provider_key': '',
+    'additional_input': '',
+    'is_collection': False,
+    'is_optional': False,
+    'default_external_id': '',
+    'value_type': '',
+    'unit': '',
+    'widget_type': 'text',
+}
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index eb0fbdb572..669d74e69e 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -5,26 +5,19 @@
 from os.path import join as pj
 from pathlib import Path
 from random import randint
-from typing import Any, Callable, Iterable, Optional, Sequence, Tuple
+from typing import Callable, Iterable, Optional, Sequence, Tuple
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
 
 from rest_framework.utils import model_meta
 
+from rdmo.core.constants import ELEMENT_COMMON_FIELDS, ELEMENT_IMPORT_EXTRA_FIELDS_DEFAULTS
 from rdmo.core.utils import get_languages
 
 logger = logging.getLogger(__name__)
 
 
-ELEMENT_COMMON_FIELDS = (
-    'uri_prefix',
-    'uri_path',
-    'key',
-    'comment',
-)
-
-
 def handle_uploaded_file(filedata):
     tempfilename = generate_tempfile_name()
     with open(tempfilename, 'wb+') as destination:
@@ -76,7 +69,7 @@ class ElementImportHelper:
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
     lang_fields: Sequence[str] = field(default_factory=list)
     foreign_fields: Sequence[str] = field(default_factory=list)
-    extra_fields: Sequence[Tuple[str, Any]] = field(default_factory=list)
+    extra_fields: Sequence[str] = field(default_factory=list)
     add_current_site_editors: bool = True
     add_current_site_sites: bool = False
 
@@ -141,6 +134,19 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None) -> None
     logger.info(message)
     element['warnings'][foreign_uri].append(message)
 
+def set_extra_field(instance, field_name, element, questions_widget_types=None) -> None:
+
+    element_value = element.get(field_name)
+    default_value = ELEMENT_IMPORT_EXTRA_FIELDS_DEFAULTS.get(field_name)
+    extra_value = element_value or default_value
+    if field_name == 'widget_type':
+        if element_value in questions_widget_types:
+            extra_value = element_value
+        else:
+            extra_value = default_value
+
+    setattr(instance, field_name, extra_value)
+
 
 def set_m2m_instances(instance, field_name, element):
     if field_name not in element:
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 4f76eb0849..9e6b6ad708 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -21,7 +21,6 @@ def import_attribute(
     # set_foreign_field are already set in management/import.py
     # check_permissions already done in management/import.py
     instance.path = instance.build_path(instance.key, instance.parent)
-
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
@@ -40,5 +39,4 @@ def import_attribute(
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     foreign_fields=('parent',),
     serializer=BaseAttributeSerializer,
-
 )
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index ce29ffc446..16886d7b0a 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -11,6 +11,7 @@
     check_permissions,
     get_or_return_instance,
     make_import_info_msg,
+    set_extra_field,
     set_foreign_field,
     set_lang_field,
 )
@@ -23,6 +24,7 @@
     import_helper_questionset,
     import_helper_section,
 )
+from rdmo.questions.utils import get_widget_types
 from rdmo.tasks.imports import import_helper_task
 from rdmo.views.imports import import_helper_view
 
@@ -58,9 +60,11 @@ def import_elements(uploaded_elements: List[Dict], save: bool = True, request: O
     imported_elements = []
     uploaded_uris = {i.get('uri') for i in uploaded_elements}
     current_site = get_current_site(request)
+    questions_widget_types = get_widget_types()
     for uploaded_element in uploaded_elements:
         element = import_element(element=uploaded_element, save=save, uploaded_uris=imported_elements,
-                                    request=request, current_site=current_site)
+                                    request=request, current_site=current_site,
+                                    questions_widget_types=questions_widget_types)
         element['warnings'] = [val for k, val in element['warnings'].items() if k not in uploaded_uris]
         imported_elements.append(element)
     return imported_elements
@@ -72,7 +76,8 @@ def import_element(
         save: bool = True,
         request: Optional[HttpRequest] = None,
         uploaded_uris: Optional[Sequence[str]] = None,
-        current_site = None
+        current_site = None,
+        questions_widget_types = None
     ) -> Dict:
 
     if element is None:
@@ -92,8 +97,10 @@ def import_element(
     import_func = import_helper.import_func
     validators = import_helper.validators
     common_fields = import_helper.common_fields
-    lang_field_names = import_helper.lang_fields if import_helper.lang_fields is not None else []
-    foreign_field_names = import_helper.foreign_fields if import_helper.foreign_fields is not None else []
+    lang_field_names = import_helper.lang_fields
+    foreign_field_names = import_helper.foreign_fields
+    extra_field_names = import_helper.extra_fields
+
     uri = element.get('uri')
 
     # get or create instance from uri and model_path
@@ -127,6 +134,9 @@ def import_element(
     # set foreign fields
     for foreign_field in foreign_field_names:
         set_foreign_field(instance, foreign_field, element, uploaded_uris=uploaded_uris)
+    # set extra fields
+    for extra_field in extra_field_names:
+        set_extra_field(instance, extra_field, element, questions_widget_types=questions_widget_types)
 
     # call the element specific import method
     instance = import_func(instance, element, validators, save)
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index ddb55637b9..44ffbed4ac 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -1,8 +1,6 @@
 import logging
 from typing import Callable, Tuple
 
-from django.db import models
-
 from rdmo.core.imports import (
     ElementImportHelper,
     set_m2m_instances,
@@ -28,13 +26,9 @@ def import_option(
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
-        user: models.Model = None
     ):
     # check_permissions already done in management/import.py
-    instance.order = element.get('order') or 0
-    instance.provider_key = element.get('provider_key') or ''
-    instance.additional_input = element.get('additional_input') or ""
-
+    # extra_fields are set in in management/import.py
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
@@ -54,7 +48,8 @@ def import_option(
     import_func=import_option,
     validators=(OptionLockedValidator, OptionUniqueURIValidator),
     lang_fields=('text',),
-    serializer = OptionSerializer
+    serializer = OptionSerializer,
+    extra_fields = ('order', 'provider_key', 'additional_input')
 )
 
 
@@ -63,14 +58,10 @@ def import_optionset(
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
-        user: models.Model = None
     ):
-
     # lang_fields are already set in management/import.py
     # check_permissions already done in management/import.py
-
-    instance.additional_input = element.get('additional_input') or ""
-
+    # extra_fields are set in in management/import.py
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
@@ -88,6 +79,6 @@ def import_optionset(
     model="options.optionset",
     import_func=import_optionset,
     validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
-    lang_fields=[],
-    serializer = OptionSetSerializer
+    serializer = OptionSetSerializer,
+    extra_fields=('additional_input',)
 )
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 97e22084fe..162fcb4fce 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -1,8 +1,6 @@
 import logging
 from typing import Callable, Tuple
 
-from django.db import models
-
 from rdmo.core.imports import (
     ElementImportHelper,
     set_m2m_instances,
@@ -23,7 +21,6 @@
     QuestionSetSerializer,
     SectionSerializer,
 )
-from .utils import get_widget_types
 from .validators import (
     CatalogLockedValidator,
     CatalogUniqueURIValidator,
@@ -45,13 +42,9 @@ def import_catalog(
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
-        user: models.Model = None
     ):
     # check_permissions already done in management/import.py
-    instance.order = element.get('order') or 0
-
-    instance.available = element.get('available', True)
-
+    # extra_fields are set in management/import.py
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
@@ -70,7 +63,6 @@ def import_section(
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
-        user: models.Model = None
     ):
     # check_permissions already done in management/import.py
     validate_instance(instance, element, *validators)
@@ -92,14 +84,11 @@ def import_page(
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
-        user: models.Model = None
     ):
     # lang_fields are already set in management/import.py
     # set_foreign_field are already set in management/import.py
     # check_permissions already done in management/import.py
-
-    instance.is_collection = element.get('is_collection') or False
-
+    # extra_fields are set in management/import.py
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
@@ -121,14 +110,11 @@ def import_questionset(
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
-        user: models.Model = None
     ):
     # lang_fields are already set in management/import.py
     # set_foreign_field are already set in management/import.py
     # check_permissions already done in management/import.py
-
-    instance.is_collection = element.get('is_collection') or False
-
+    # extra_fields are set in management/import.py
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
@@ -151,31 +137,12 @@ def import_question(
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
-        user: models.Model = None
     ):
     # lang_fields are already set in management/import.py
     # set_foreign_fields are already set in management/import.py
     # check_permissions already done in management/import.py
-
-    instance.is_collection = element.get('is_collection') or False
-    instance.is_optional = element.get('is_optional') or False
-
-    instance.default_external_id = element.get('default_external_id') or ''
-
-    if element.get('widget_type') in get_widget_types():
-        instance.widget_type = element.get('widget_type')
-    else:
-        instance.widget_type = 'text'
-
-    instance.value_type = element.get('value_type') or ''
-    instance.maximum = element.get('maximum')
-    instance.minimum = element.get('minimum')
-    instance.step = element.get('step')
-    instance.unit = element.get('unit') or ''
-    instance.width = element.get('width')
-
+    # extra_fields are set in management/import.py
     validate_instance(instance, element, *validators)
-
     if element.get('errors'):
         return instance
 
@@ -196,7 +163,8 @@ def import_question(
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
     lang_fields=('help', 'title'),
     serializer = CatalogSerializer,
-    add_current_site_sites = True
+    add_current_site_sites = True,
+    extra_fields = ('order', 'available')
 )
 
 import_helper_section = ElementImportHelper(
@@ -213,7 +181,8 @@ def import_question(
     validators=(PageLockedValidator, PageUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
-    serializer = PageSerializer
+    serializer = PageSerializer,
+    extra_fields = ('is_collection',)
 )
 
 import_helper_questionset = ElementImportHelper(
@@ -222,7 +191,8 @@ def import_question(
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
-    serializer = QuestionSetSerializer
+    serializer = QuestionSetSerializer,
+    extra_fields = ('is_collection',)
 )
 
 import_helper_question = ElementImportHelper(
@@ -231,5 +201,7 @@ def import_question(
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
     lang_fields=('text', 'help', 'default_text', 'verbose_name'),
     foreign_fields=('attribute','default_option'),
-    serializer = QuestionSerializer
+    serializer = QuestionSerializer,
+    extra_fields = ('is_collection','is_optional', 'default_external_id', 'widget_type',
+                    'value_type', 'maximum', 'minimum', 'step', 'unit','width')
 )
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 9be1352096..278687613c 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -1,8 +1,6 @@
 import logging
 from typing import Callable, Tuple
 
-from django.db import models
-
 from rdmo.core.imports import (
     ElementImportHelper,
     set_m2m_instances,
@@ -21,19 +19,11 @@ def import_task(
         element: dict,
         validators: Tuple[Callable],
         save: bool = False,
-        user: models.Model = None
     ):
     # lang_fields are already set in management/import.py
     # set_foreign_field are already set in management/import.py
     # check_permissions already done in management/import.py
-
-    instance.order = element.get('order') or 0
-
-    instance.days_before = element.get('days_before')
-    instance.days_after = element.get('days_after')
-
-    instance.available = element.get('available', True)
-
+    # extra_fields are set in in management/import.py
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
@@ -54,5 +44,6 @@ def import_task(
     validators=(TaskLockedValidator, TaskUniqueURIValidator),
     lang_fields=('title', 'text'),
     foreign_fields=('start_attribute', 'end_attribute'),
-    serializer=TaskSerializer
+    serializer=TaskSerializer,
+    extra_fields=('order', 'days_before', 'days_after', 'available')
 )
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 52639fe789..524d69b58e 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -21,12 +21,7 @@ def import_view(
         save: bool = False,
     ):
     # check_permissions already done in management/import.py
-
-    instance.order = element.get('order') or 0
-    instance.template = element.get('template')
-
-    instance.available = element.get('available', True)
-
+    # extra_fields are set in in management/import.py
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
@@ -44,6 +39,7 @@ def import_view(
     model="views.view",
     import_func=import_view,
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
-    lang_fields=( 'help', 'title'),
-    serializer=ViewSerializer
+    lang_fields=('help', 'title'),
+    serializer=ViewSerializer,
+    extra_fields=('order', 'template', 'available')
 )

From c75bb7d33f331155f1960be3b012c062faca0d70 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 24 Jan 2024 17:11:46 +0100
Subject: [PATCH 058/205] tests: add import tests for updated and changed
 fields

---
 rdmo/management/tests/__init__.py             |  16 ++-
 .../tests/test_import_conditions.py           |  22 ++-
 rdmo/management/tests/test_import_domain.py   |  18 ++-
 rdmo/management/tests/test_import_options.py  |  49 ++++++-
 .../management/tests/test_import_questions.py | 134 +++++++++++++++++-
 rdmo/management/tests/test_import_tasks.py    |  23 ++-
 rdmo/management/tests/test_import_views.py    |  23 ++-
 7 files changed, 271 insertions(+), 14 deletions(-)

diff --git a/rdmo/management/tests/__init__.py b/rdmo/management/tests/__init__.py
index 063402d4ca..697d6309a9 100644
--- a/rdmo/management/tests/__init__.py
+++ b/rdmo/management/tests/__init__.py
@@ -12,7 +12,9 @@ def read_xml_and_parse_to_elements(xml_file):
 
 def change_fields_elements(elements, update_dict=None, n=3):
 
+    update_dict = update_dict if update_dict is not None else {}
     _default_update_dict = {'comment':  "this is a test comment {}"}
+    update_dict.update(**_default_update_dict)
 
     if len(elements) < n:
         raise ValueError("Length of elements should not be smaller than n.")
@@ -20,9 +22,15 @@ def change_fields_elements(elements, update_dict=None, n=3):
     _changed_elements = []
     for _n,_element in enumerate(elements):
         if _n <= n-1:
-            _element['comment'] = _default_update_dict['comment'].format(_n)
-            if update_dict is not None:
-                _element.update (**update_dict)
-            _changed_elements.append(_element)
+            updated_and_changed = {}
+            changed_element = _element
+            for k,val in update_dict.items():
+                if isinstance(val, str):
+                    val = val.format(_n)
+                updated_and_changed[k]= {'current': _element[k], 'uploaded': val}
+                _element[k] = val
+            if updated_and_changed:
+                changed_element['updated_and_changed'] = updated_and_changed
+            _changed_elements.append(changed_element)
         _new_elements.append(_element)
     return _new_elements, _changed_elements
diff --git a/rdmo/management/tests/test_import_conditions.py b/rdmo/management/tests/test_import_conditions.py
index 8cf6d30dd3..c001754c83 100644
--- a/rdmo/management/tests/test_import_conditions.py
+++ b/rdmo/management/tests/test_import_conditions.py
@@ -1,10 +1,19 @@
 from pathlib import Path
 
+import pytest
+
 from rdmo.conditions.models import Condition
 from rdmo.management.imports import import_elements
 
 from . import change_fields_elements, read_xml_and_parse_to_elements
 
+imported_update_changes = [
+    None,
+    {
+    'target_text' : 'test target_text {}',
+    'relation': 'notempty'
+    }
+]
 
 def test_create_conditions(db, settings):
     Condition.objects.all().delete()
@@ -30,18 +39,21 @@ def test_update_conditions(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-def test_update_conditions_with_changed_comments(db, settings):
+@pytest.mark.parametrize('update_dict', imported_update_changes)
+def test_update_conditions_with_changed_fields(db, settings, update_dict):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
     elements, root = read_xml_and_parse_to_elements(xml_file)
-    elements, changed_elements = change_fields_elements(elements, n=3)
-    # breakpoint()
+    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=7)
     imported_elements = import_elements(elements)
-
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
     assert len(root) == len(imported_elements) == 15
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
-    assert len([i for i in elements if i['updated_and_changed']]) == len(changed_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
 
 
 def test_create_legacy_conditions(db, settings):
diff --git a/rdmo/management/tests/test_import_domain.py b/rdmo/management/tests/test_import_domain.py
index ac286fc0a7..eda3e222dc 100644
--- a/rdmo/management/tests/test_import_domain.py
+++ b/rdmo/management/tests/test_import_domain.py
@@ -3,7 +3,7 @@
 from rdmo.domain.models import Attribute
 from rdmo.management.imports import import_elements
 
-from . import read_xml_and_parse_to_elements
+from . import change_fields_elements, read_xml_and_parse_to_elements
 
 
 def test_create_domain(db, settings):
@@ -30,6 +30,22 @@ def test_update_domain(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
+def test_update_attributes_with_changed_fields(db, settings):
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
+
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, changed_elements = change_fields_elements(elements, n=50)
+    imported_elements = import_elements(elements)
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    assert len(root) == len(imported_elements) == 86
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
+
+
 def test_create_legacy_domain(db, settings):
     Attribute.objects.all().delete()
 
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 9abcdebbb7..e6d93c4cec 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -1,10 +1,13 @@
 from pathlib import Path
 
+import pytest
+
 from rdmo.management.imports import import_elements
 from rdmo.options.models import Option, OptionSet
 
-from . import read_xml_and_parse_to_elements
+from . import change_fields_elements, read_xml_and_parse_to_elements
 
+imported_update_changes = [None]
 
 def test_create_optionsets(db, settings):
     OptionSet.objects.all().delete()
@@ -32,6 +35,28 @@ def test_update_optionsets(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
+@pytest.mark.parametrize('update_dict', imported_update_changes)
+def test_update_optionsets_with_changed_fields(db, settings, update_dict):
+    OptionSet.objects.all().delete()
+    Option.objects.all().delete()
+
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+    assert len(root) == len(imported_elements) == 13
+    # start test with fresh options in db
+    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=7)
+    imported_elements = import_elements(elements)
+    assert len(root) == len(imported_elements) == 13
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
+
+
 def test_create_options(db, settings):
     Option.objects.all().delete()
 
@@ -56,6 +81,28 @@ def test_update_options(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
+@pytest.mark.parametrize('update_dict', imported_update_changes)
+def test_update_options_with_changed_fields(db, settings, update_dict):
+    OptionSet.objects.all().delete()
+    Option.objects.all().delete()
+
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+    assert len(root) == len(imported_elements) == 9
+    # start test with fresh options in db
+    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=4)
+    imported_elements = import_elements(elements)
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    assert len(root) == len(imported_elements) == 9
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
+
+
 def test_create_legacy_options(db, settings):
     OptionSet.objects.all().delete()
     Option.objects.all().delete()
diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py
index a282f5b56c..1bfe24aacf 100644
--- a/rdmo/management/tests/test_import_questions.py
+++ b/rdmo/management/tests/test_import_questions.py
@@ -1,9 +1,13 @@
 from pathlib import Path
 
+import pytest
+
 from rdmo.management.imports import import_elements
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
-from . import read_xml_and_parse_to_elements
+from . import change_fields_elements, read_xml_and_parse_to_elements
+
+imported_update_changes = [None]
 
 
 def test_create_catalogs(db, settings):
@@ -39,6 +43,32 @@ def test_update_catalogs(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
+@pytest.mark.parametrize('update_dict', imported_update_changes)
+def test_update_catalogs_with_changed_fields(db, settings, update_dict):
+    Catalog.objects.all().delete()
+    Section.objects.all().delete()
+    Page.objects.all().delete()
+    QuestionSet.objects.all().delete()
+    Question.objects.all().delete()
+
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+    assert len(root) == len(imported_elements) == 148
+    # start test with fresh elements in db
+
+    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=75)
+    imported_elements = import_elements(elements)
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    assert len(imported_elements) == 148
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
+
+
 def test_create_sections(db, settings):
     Section.objects.all().delete()
     Page.objects.all().delete()
@@ -70,6 +100,31 @@ def test_update_sections(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
+@pytest.mark.parametrize('update_dict', imported_update_changes)
+def test_update_sections_with_changed_fields(db, settings, update_dict):
+    Catalog.objects.all().delete()
+    Section.objects.all().delete()
+    Page.objects.all().delete()
+    QuestionSet.objects.all().delete()
+    Question.objects.all().delete()
+
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+    assert len(root) == len(imported_elements) == 146
+    # start test with fresh elements in db
+
+    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=75)
+    imported_elements = import_elements(elements)
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
+
+
 def test_create_pages(db, settings):
     Page.objects.all().delete()
     QuestionSet.objects.all().delete()
@@ -99,6 +154,31 @@ def test_update_pages(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
+@pytest.mark.parametrize('update_dict', imported_update_changes)
+def test_update_pages_with_changed_fields(db, settings, update_dict):
+    Catalog.objects.all().delete()
+    Section.objects.all().delete()
+    Page.objects.all().delete()
+    QuestionSet.objects.all().delete()
+    Question.objects.all().delete()
+
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+    assert len(root) == len(imported_elements) == 140
+    # start test with fresh elements in db
+    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=75)
+    imported_elements = import_elements(elements)
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    assert len(imported_elements) == 140
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
+
+
 def test_create_questionsets(db, settings):
     Page.objects.all().delete()
     QuestionSet.objects.all().delete()
@@ -124,10 +204,37 @@ def test_update_questionsets(db, settings):
     imported_elements = import_elements(elements)
 
     assert len(root) == 10  # two questionsets appear twice in the export file
+    assert len(imported_elements) == 8
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
 
 
+@pytest.mark.parametrize('update_dict', imported_update_changes)
+def test_update_questionsets_with_changed_fields(db, settings, update_dict):
+    Catalog.objects.all().delete()
+    Section.objects.all().delete()
+    Page.objects.all().delete()
+    QuestionSet.objects.all().delete()
+    Question.objects.all().delete()
+
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+    assert len(root) == 10  # two questionsets appear twice in the export file
+    assert len(imported_elements) == 8
+    # start test with fresh elements in db
+    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=5)
+    imported_elements = import_elements(elements)
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    assert len(imported_elements) == 8
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
+
+
 def test_create_questions(db, settings):
     Page.objects.all().delete()
     QuestionSet.objects.all().delete()
@@ -155,6 +262,31 @@ def test_update_questions(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
+@pytest.mark.parametrize('update_dict', imported_update_changes)
+def test_update_questions_with_changed_fields(db, settings, update_dict):
+    Catalog.objects.all().delete()
+    Section.objects.all().delete()
+    Page.objects.all().delete()
+    QuestionSet.objects.all().delete()
+    Question.objects.all().delete()
+
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+    assert len(root) == len(imported_elements) == 89
+    # start test with fresh elements in db
+    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=45)
+    imported_elements = import_elements(elements)
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    assert len(imported_elements) == 89
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
+
+
 def test_create_legacy_questions(db, settings):
     Catalog.objects.all().delete()
     Section.objects.all().delete()
diff --git a/rdmo/management/tests/test_import_tasks.py b/rdmo/management/tests/test_import_tasks.py
index 3e9a87831b..5e0c5c426c 100644
--- a/rdmo/management/tests/test_import_tasks.py
+++ b/rdmo/management/tests/test_import_tasks.py
@@ -1,9 +1,13 @@
 from pathlib import Path
 
+import pytest
+
 from rdmo.management.imports import import_elements
 from rdmo.tasks.models import Task
 
-from . import read_xml_and_parse_to_elements
+from . import change_fields_elements, read_xml_and_parse_to_elements
+
+imported_update_changes = [None]
 
 
 def test_create_tasks(db, settings):
@@ -30,6 +34,23 @@ def test_update_tasks(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
+@pytest.mark.parametrize('update_dict', imported_update_changes)
+def test_update_tasks_with_changed_fields(db, settings, update_dict):
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'tasks.xml'
+
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=1)
+    imported_elements = import_elements(elements)
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    assert len(root) == len(imported_elements) == 2
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
+
+
 def test_create_legacy_tasks(db, settings):
     Task.objects.all().delete()
 
diff --git a/rdmo/management/tests/test_import_views.py b/rdmo/management/tests/test_import_views.py
index bf169c69e2..72a1873cce 100644
--- a/rdmo/management/tests/test_import_views.py
+++ b/rdmo/management/tests/test_import_views.py
@@ -1,9 +1,13 @@
 from pathlib import Path
 
+import pytest
+
 from rdmo.management.imports import import_elements
 from rdmo.views.models import View
 
-from . import read_xml_and_parse_to_elements
+from . import change_fields_elements, read_xml_and_parse_to_elements
+
+imported_update_changes = [None]
 
 
 def test_create_tasks(db, settings):
@@ -30,6 +34,23 @@ def test_update_tasks(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
+@pytest.mark.parametrize('update_dict', imported_update_changes)
+def test_update_views_with_changed_fields(db, settings, update_dict):
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'views.xml'
+
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=2)
+    imported_elements = import_elements(elements)
+    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    assert len(root) == len(imported_elements) == 3
+    assert all(element['created'] is False for element in imported_elements)
+    assert all(element['updated'] is True for element in imported_elements)
+    assert len(imported_and_changed) == len(changed_elements)
+    # compare two ordered lists with "updated_and_changed" dicts
+    for test, imported in zip(changed_elements, imported_and_changed):
+        assert test['updated_and_changed'] == imported['updated_and_changed']
+
+
 def test_create_legacy_tasks(db, settings):
     View.objects.all().delete()
 

From a1e69b917487e406b4c86195ce41e70b82686525 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Jan 2024 11:55:51 +0100
Subject: [PATCH 059/205] chore: add file to importActions.js and
 importsReducer.js

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/js/actions/importActions.js   | 6 +++---
 rdmo/management/assets/js/reducers/importsReducer.js | 4 +++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/rdmo/management/assets/js/actions/importActions.js b/rdmo/management/assets/js/actions/importActions.js
index 96dceb16a2..b65dc32cbd 100644
--- a/rdmo/management/assets/js/actions/importActions.js
+++ b/rdmo/management/assets/js/actions/importActions.js
@@ -8,7 +8,7 @@ import { fetchElements, fetchElement } from './elementActions'
 
 export function uploadFile(file) {
   return function(dispatch) {
-    dispatch(uploadFileInit())
+    dispatch(uploadFileInit(file))
 
     return ManagementApi.uploadFile(file)
       .then(elements => dispatch(uploadFileSuccess(elements)))
@@ -18,8 +18,8 @@ export function uploadFile(file) {
   }
 }
 
-export function uploadFileInit() {
-  return {type: 'import/uploadFileInit'}
+export function uploadFileInit(file) {
+  return {type: 'import/uploadFileInit', file: file}
 }
 
 export function uploadFileSuccess(elements) {
diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index 8c37fcd396..cfb4273251 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -9,7 +9,8 @@ import { buildUri } from '../utils/elements'
 const initialState = {
   elements: [],
   errors: [],
-  success: false
+  success: false,
+  file: null
 }
 
 export default function importsReducer(state = initialState, action) {
@@ -18,6 +19,7 @@ export default function importsReducer(state = initialState, action) {
   switch(action.type) {
     // upload file
     case 'import/uploadFileInit':
+      return {...state, file: action.file}
     case 'elements/fetchElementsInit':
     case 'elements/fetchElementInit':
       return {...state, elements: [], errors: [], success: false}

From 45e9e0cf5c046462ce2a2e3082bc57f2ece151f4 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Jan 2024 11:57:04 +0100
Subject: [PATCH 060/205] chore: add file configActions arg to Import component

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/js/containers/Main.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/rdmo/management/assets/js/containers/Main.js b/rdmo/management/assets/js/containers/Main.js
index 4cd6049d3f..532b1fdb75 100644
--- a/rdmo/management/assets/js/containers/Main.js
+++ b/rdmo/management/assets/js/containers/Main.js
@@ -25,7 +25,7 @@ const Main = ({ config, elements, imports, configActions, elementActions, import
     return null
   }
 
-  // check if an an error occurred
+  // check if an error occurred
   if (!isNil(elements.errors.api)) {
     return <MainErrors errors={elements.errors.api} />
   } else if (get(elements, 'element.errors.api')) {
@@ -35,11 +35,12 @@ const Main = ({ config, elements, imports, configActions, elementActions, import
   }
 
   if (!isEmpty(imports.elements)) {
-    return <Import config={config} imports={imports} importActions={importActions} />
+    return <Import config={config} imports={imports}
+                   configActions={configActions} importActions={importActions} />
   }
 
   // check if the nested components should be displayed
-  if (!isNil(element) && elementAction == 'nested') {
+  if (!isNil(element) && elementAction === 'nested') {
     return <Nested config={config} elements={elements}
                    configActions={configActions} elementActions={elementActions} />
   }

From bf2a7fe340df159844ff1021eafe2e03e816f04c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Jan 2024 14:03:25 +0100
Subject: [PATCH 061/205] feat: add filters to Import component

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/main/Import.js       | 116 ++++++++++++------
 1 file changed, 79 insertions(+), 37 deletions(-)

diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 2a7c863a81..0e6f46fbc1 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -7,66 +7,107 @@ import { getUriPrefixes } from '../../utils/filter'
 import { FilterString, FilterUriPrefix } from '../common/Filter'
 
 import ImportElement from '../import/ImportElement'
-import ImportSuccesElement from '../import/ImportSuccessElement'
+import ImportSuccessElement from '../import/ImportSuccessElement'
+import {Checkbox} from '../common/Checkboxes'
 
-const Import = ({ config, imports, importActions }) => {
-  const { filename, elements, success } = imports
-  const updateFilterString = (value) => importActions.updateConfig('filter.import.elements.search', value)
-  const updateFilterUriPrefix = (value) => importActions.updateConfig('filter.import.elements.uri_prefix', value)
+
+const Import = ({ config, imports, configActions, importActions }) => {
+  const { file, elements, success } = imports
+  const updateFilterString = (value) => configActions.updateConfig('filter.import.elements.search', value)
+  const updateFilterUriPrefix = (value) => configActions.updateConfig('filter.import.elements.uri_prefix', value)
+  const updateFilterChanged = (value) => configActions.updateConfig('filter.import.elements.changed', value)
   const updatedElements = elements.filter(element => element.updated)
   const createdElements = elements.filter(element => element.created)
   const updatedAndChangedElements = elements.filter(element => element.updated && !isEmpty(element.updated_and_changed))
-  const updatedAndSameElements = elements.filter(element => element.updated && isEmpty(element.updated_and_changed))
+  // const updatedAndSameElements = elements.filter(element => element.updated && isEmpty(element.updated_and_changed))
   const importErrors = elements.filter(element => !isEmpty(element.errors))
-
+  const searchString = get(config, 'filter.import.elements.search', '')
+  const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
+  const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
+  const filterByChanged = (elements, selectFilterChanged) => {
+    if (selectFilterChanged) {
+    return updatedAndChangedElements
+  } else {
+    return elements
+  }}
+  const filterByUriSearch = (elements, searchString) => {
+    if (searchString) {
+      const lowercaseSearch = searchString.toLowerCase()
+      return elements.filter((element) =>
+        element.uri.toLowerCase().includes(lowercaseSearch)
+          // || element.title.toLowerCase().includes(lowercaseSearch)
+      )
+    } else {
+      return elements
+    }
+  }
+    const filterByUriPrefix = (elements, searchUriPrefix) => {
+    if (searchUriPrefix) {
+      return elements.filter((element) =>
+        element.uri_prefix.toLowerCase().includes(searchUriPrefix)
+          // || element.title.toLowerCase().includes(lowercaseSearch)
+      )
+    } else {
+      return elements
+    }
+  }
+  const filteredElements = filterByUriSearch(
+                                  filterByUriPrefix(
+                                        filterByChanged(elements, selectFilterChanged),
+                                  selectedUriPrefix),
+                          searchString)
 
   return (
     <div className="panel panel-default panel-import">
       <div className="panel-heading">
-        <strong>{gettext('Import')} from file {filename}</strong>
+        <strong>{gettext('Import')} from: {file.name}</strong>
         <div className="pull-right">
           {
-            elements.length > -1 && <span>{gettext('Total')}: {elements.length} </span>
+            elements.length > 0 && <span>{gettext('Total')}: {elements.length} </span>
           }
           {
-            updatedElements.length > -1 && <span> {gettext('Updated')}: {updatedElements.length}
-            {' ('}{gettext('Changed')}: {updatedAndChangedElements.length}
-            {' '}{gettext('Same')}: {updatedAndSameElements.length}{') '}
+            updatedElements.length > 0 && <span> {gettext('Updated')}: {updatedElements.length}
+            {' ('}{gettext('Changed')}: {updatedAndChangedElements.length}{') '}
               </span>
             }
           {
-            createdElements.length > -1 && <span>{gettext('Created')}: {createdElements.length} </span>
+            createdElements.length > 0 && <span>{gettext('Created')}: {createdElements.length} </span>
           }
           {
-            importErrors.length > -1 && <span>{gettext('Errors')}: {importErrors.length} </span>
+            importErrors.length > 0 && <span>{gettext('Errors')}: {importErrors.length} </span>
           }
         </div>
       </div>
-    <div className="panel-body">
-    {/* TODO: still to implement functions for filter, uri_prefix dropdown. */}
-    <div className="row">
-      <div className={'col-sm-8'}>
-        <FilterString value={get(config, 'filter.import.elements.search', '')} onChange={updateFilterString}
-                      placeholder={gettext('Filter uri')} />
-      </div>
-      <div className="col-sm-4">
-      {/* TODO: add update of the filter to elements */}
-        <FilterUriPrefix value={get(config, 'filter.import.elements.uri_prefix', '')} onChange={updateFilterUriPrefix}
-                        options={getUriPrefixes(elements)} />
+      <div className="panel-body">
+        <div className="row">
+          <div className={'col-sm-8'}>
+            <FilterString value={get(config, 'filter.import.elements.search', '')} onChange={updateFilterString}
+                          placeholder={gettext('Filter uri')}/>
+          </div>
+          <div className="col-sm-4">
+            <FilterUriPrefix value={get(config, 'filter.import.elements.uri_prefix', '')}
+                             onChange={updateFilterUriPrefix}
+                             options={getUriPrefixes(elements)}/>
+          </div>
+        </div>
+        {
+        updatedAndChangedElements.length > 0 && <div className="checkboxes">
+          <span className="mr-10">{gettext('Changed:')}</span>
+          <Checkbox label={<code className="code-questions">{gettext('Changed')}</code>}
+                    value={get(config, 'filter.import.elements.changed', true)} onChange={updateFilterChanged}/>
+        </div>
+          }
       </div>
-
-    </div>
-    </div>
       <ul className="list-group">
-      {
-        elements.map((element, index) => {
-          if (success) {
-            return <ImportSuccesElement key={index} element={element}/>
-          } else {
-              return <ImportElement key={index} config={config} element={element} importActions={importActions} />
-          }
-        })
-      }
+        {
+          filteredElements.map((element, index) => {
+            if (success) {
+              return <ImportSuccessElement key={index} element={element}/>
+            } else {
+              return <ImportElement key={index} config={config} element={element} importActions={importActions}/>
+            }
+          })
+        }
       </ul>
     </div>
   )
@@ -75,6 +116,7 @@ const Import = ({ config, imports, importActions }) => {
 Import.propTypes = {
   config: PropTypes.object.isRequired,
   imports: PropTypes.object.isRequired,
+  configActions: PropTypes.object.isRequired,
   importActions: PropTypes.object.isRequired
 }
 

From 1c8622f95315b310280d60dba1d9216365c320f9 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Jan 2024 14:10:10 +0100
Subject: [PATCH 062/205] chore: fix typos in ImportSidebar.js

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/sidebar/ImportSidebar.js         | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
index 73e2614d54..140105c1df 100644
--- a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
+++ b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
@@ -51,7 +51,7 @@ const ImportSidebar = ({ config, imports, importActions }) => {
               {gettext('Select all')}
             </Link>
           </li>
-          { updatedAndChangedElements.length > -1 &&
+          {updatedAndChangedElements.length > 0 &&
             <ul className="list-unstyled">
             <li>
               <Link onClick={() => importActions.selectChangedElements(true)}>
@@ -79,16 +79,16 @@ const ImportSidebar = ({ config, imports, importActions }) => {
               {gettext('Show all')}
             </Link>
           </li>
-          { updatedAndChangedElements.length > -1 &&
+          {updatedAndChangedElements.length > 0 &&
             <ul className="list-unstyled">
             <li>
               <Link onClick={() => importActions.showChangedElements(true)}>
-                {gettext('Show changed')}
+                {gettext('Show changes')}
               </Link>
             </li>
             <li>
               <Link onClick={() => importActions.showChangedElements(false)}>
-                {gettext('Hide changed')}
+                {gettext('Hide changes')}
               </Link>
             </li>
             </ul>

From 110ea1211a6901968badd527288f58e34a085e9e Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Jan 2024 15:01:55 +0100
Subject: [PATCH 063/205] chore: fix typing typo and arg

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/imports.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 16886d7b0a..55d0fc33c9 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -1,7 +1,7 @@
 import copy
 import logging
 from collections import defaultdict
-from typing import Dict, List, Optional, Sequence
+from typing import Dict, List, Optional
 
 from django.contrib.sites.shortcuts import get_current_site
 from django.http import HttpRequest
@@ -62,7 +62,7 @@ def import_elements(uploaded_elements: List[Dict], save: bool = True, request: O
     current_site = get_current_site(request)
     questions_widget_types = get_widget_types()
     for uploaded_element in uploaded_elements:
-        element = import_element(element=uploaded_element, save=save, uploaded_uris=imported_elements,
+        element = import_element(element=uploaded_element, save=save, uploaded_uris=uploaded_uris,
                                     request=request, current_site=current_site,
                                     questions_widget_types=questions_widget_types)
         element['warnings'] = [val for k, val in element['warnings'].items() if k not in uploaded_uris]
@@ -75,13 +75,13 @@ def import_element(
         element: Optional[Dict] = None,
         save: bool = True,
         request: Optional[HttpRequest] = None,
-        uploaded_uris: Optional[Sequence[str]] = None,
+        uploaded_uris: Optional[set[str]] = None,
         current_site = None,
         questions_widget_types = None
     ) -> Dict:
 
     if element is None:
-        return
+        return {}
 
     model_path = element.get('model')
     if model_path is None:
@@ -183,7 +183,7 @@ def get_updated_changes(element, new_instance,
     for uri_key in uri_keys:
         element_name, uri_field = uri_key.split('_')
         if uri_key in updated_and_changed:
-            # eg. set attribute as key instead of attribute_uri
+            # e.g. set attribute as key instead of attribute_uri
             uri_key_val = updated_and_changed[uri_key].pop()
             updated_and_changed[element_name] = uri_key_val
         if element_name in element and uri_key in original_data:

From feccc59bd6691095264efbfaf66d23262fec2265 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Jan 2024 18:16:40 +0100
Subject: [PATCH 064/205] chore: refactor import warning messages into dict of
 uris

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/ImportElement.js     |  2 +-
 .../components/import/ImportSuccessElement.js |  3 ++-
 .../js/components/import/common/Warnings.js   | 27 +++++++++++++++----
 .../assets/js/components/main/Import.js       | 14 ++++++++--
 rdmo/management/imports.py                    |  2 +-
 5 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index 3558f9b5b8..293900896e 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -38,7 +38,7 @@ const ImportElement = ({ config, element, importActions }) => {
         element.show && <>
           <Form config={config} element={element} updateElement={updateElement} />
           <Fields element={element} />
-          <Warnings element={element} />
+          <Warnings element={element} success={false}/>
           <Errors element={element} />
         </>
       }
diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index 4a8446db1d..263887fa27 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -4,6 +4,7 @@ import uniqueId from 'lodash/uniqueId'
 
 import { codeClass, verboseNames } from '../../constants/elements'
 import { isEmpty } from 'lodash'
+import Warnings from './common/Warnings'
 
 const ImportSuccessElement = ({ element }) => {
   return (
@@ -23,7 +24,7 @@ const ImportSuccessElement = ({ element }) => {
         }
         {'.'}
       </p>
-      {element.warnings.map(message => <p key={uniqueId()} className="text-warning">{message}</p>)}
+        <Warnings element={element} success={true}/>
       {element.errors.map(message => <p key={uniqueId()} className="text-danger">{message}</p>)}
     </li>
   )
diff --git a/rdmo/management/assets/js/components/import/common/Warnings.js b/rdmo/management/assets/js/components/import/common/Warnings.js
index c4c91bdb83..be8cba22ab 100644
--- a/rdmo/management/assets/js/components/import/common/Warnings.js
+++ b/rdmo/management/assets/js/components/import/common/Warnings.js
@@ -2,16 +2,32 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import isEmpty from 'lodash/isEmpty'
 import uniqueId from 'lodash/uniqueId'
+import {codeClass} from '../../../constants/elements'
 
-const Warnings = ({ element }) => {
-  return !isEmpty(element.warnings) && <div className="row text-warning mt-10">
-    <div className="col-sm-3 text-right">
+const Warnings = ({ element, success = false }) => {
+  return !isEmpty(element.warnings) && <div className="row mt-10">
+    { (success === true) &&
+    <div className="col-sm-3 text-warning text-right">
       {gettext('Warnings')}
     </div>
+  }
     <div className="col-sm-9">
       <ul className="list-unstyled">
         {
-          element.warnings.map(message => <li key={uniqueId('error-')}>{message}</li>)
+          Object.entries(element.warnings).map(([uri, messages]) => {
+              return (
+                  <li key={uniqueId('warning-uri-')}><code className={codeClass['domain.attribute']}>{uri}</code>
+                    <div className="col-sm-9">
+                      <ul className="list-unstyled">
+                        {
+                          messages.map(message => (
+                              <li className="text-warning" key={uniqueId('warning-uri-message')}>{message}</li>))
+                        }
+                      </ul>
+                    </div>
+                  </li>
+              )
+          })
         }
       </ul>
     </div>
@@ -19,7 +35,8 @@ const Warnings = ({ element }) => {
 }
 
 Warnings.propTypes = {
-  element: PropTypes.object.isRequired
+  element: PropTypes.object.isRequired,
+  success: PropTypes.bool.isRequired
 }
 
 export default Warnings
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 0e6f46fbc1..5fb86a8638 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -21,11 +21,21 @@ const Import = ({ config, imports, configActions, importActions }) => {
   const updatedAndChangedElements = elements.filter(element => element.updated && !isEmpty(element.updated_and_changed))
   // const updatedAndSameElements = elements.filter(element => element.updated && isEmpty(element.updated_and_changed))
   const importErrors = elements.filter(element => !isEmpty(element.errors))
+  // const importWarnings = elements.filter(element => !isEmpty(element.warnings))
   const searchString = get(config, 'filter.import.elements.search', '')
   const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
-  const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
+  const selectFilterChanged = () => {
+    const configBool = get(config, 'filter.import.elements.changed', false)
+    if (configBool === true && updatedAndChangedElements.length === 0) {
+      return false
+    }
+    else {
+      return configBool
+    }
+  }
+
   const filterByChanged = (elements, selectFilterChanged) => {
-    if (selectFilterChanged) {
+    if (selectFilterChanged === true) {
     return updatedAndChangedElements
   } else {
     return elements
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 55d0fc33c9..59888fd7f8 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -65,7 +65,7 @@ def import_elements(uploaded_elements: List[Dict], save: bool = True, request: O
         element = import_element(element=uploaded_element, save=save, uploaded_uris=uploaded_uris,
                                     request=request, current_site=current_site,
                                     questions_widget_types=questions_widget_types)
-        element['warnings'] = [val for k, val in element['warnings'].items() if k not in uploaded_uris]
+        element['warnings'] = {k: val  for k, val in element['warnings'].items() if k not in uploaded_uris}
         imported_elements.append(element)
     return imported_elements
 

From bb343d702eb175fa38093ac4f6e5fdcd403d0268 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 30 Jan 2024 09:32:42 +0100
Subject: [PATCH 065/205] chore: fix typing of Set for py38

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/imports.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 59888fd7f8..f8248ae14d 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -1,7 +1,7 @@
 import copy
 import logging
 from collections import defaultdict
-from typing import Dict, List, Optional
+from typing import AbstractSet, Dict, List, Optional
 
 from django.contrib.sites.shortcuts import get_current_site
 from django.http import HttpRequest
@@ -75,7 +75,7 @@ def import_element(
         element: Optional[Dict] = None,
         save: bool = True,
         request: Optional[HttpRequest] = None,
-        uploaded_uris: Optional[set[str]] = None,
+        uploaded_uris: Optional[AbstractSet[str]] = None,
         current_site = None,
         questions_widget_types = None
     ) -> Dict:

From 07963e235903c59869a7dc7b0d10a82f3be2b67b Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 30 Jan 2024 10:53:05 +0100
Subject: [PATCH 066/205] refactor: move all import function calls to
 management, leave only ElementImportHelper

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/conditions/imports.py |  35 +------
 rdmo/core/imports.py       |  20 +++-
 rdmo/domain/imports.py     |  26 +----
 rdmo/management/imports.py |  20 +++-
 rdmo/options/imports.py    |  71 ++-----------
 rdmo/questions/imports.py  | 200 ++++++++++---------------------------
 rdmo/tasks/imports.py      |  41 +-------
 rdmo/views/imports.py      |  38 +------
 8 files changed, 103 insertions(+), 348 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index c15f4ccba4..8229b8d13e 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -1,43 +1,10 @@
-import logging
-from typing import Callable, Tuple
+from rdmo.core.imports import ElementImportHelper
 
-from rdmo.core.imports import (
-    ElementImportHelper,
-    validate_instance,
-)
-
-from .models import Condition
 from .serializers.v1 import ConditionSerializer
 from .validators import ConditionLockedValidator, ConditionUniqueURIValidator
 
-logger = logging.getLogger(__name__)
-
-
-def import_condition(
-        instance: Condition,
-        element: dict,
-        validators: Tuple[Callable],
-        save: bool = False,
-    ):
-
-    # set_foreign_field are already set in management/import.py
-    # check_permissions already done in management/import.py
-    # extra_fields are set in in management/import.py
-    validate_instance(instance, element, *validators)
-
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
 import_helper_condition = ElementImportHelper(
     model="conditions.condition",
-    import_func=import_condition,
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
     foreign_fields=('source', 'target_option'),
     serializer=ConditionSerializer,
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 669d74e69e..da69937e0a 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -5,7 +5,7 @@
 from os.path import join as pj
 from pathlib import Path
 from random import randint
-from typing import Callable, Iterable, Optional, Sequence, Tuple
+from typing import Callable, Dict, Iterable, Optional, Sequence, Tuple
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
@@ -63,13 +63,15 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
 @dataclass
 class ElementImportHelper:
     model: str
-    import_func: Callable
     validators: Iterable[Callable]
     serializer: Callable
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
     lang_fields: Sequence[str] = field(default_factory=list)
     foreign_fields: Sequence[str] = field(default_factory=list)
     extra_fields: Sequence[str] = field(default_factory=list)
+    m2m_instance_fields: Sequence[str] = field(default_factory=list)
+    m2m_through_instance_fields: Sequence[Dict[str, str]] = field(default_factory=list)
+    reverse_m2m_through_instance_fields: Sequence[Dict[str, str]] = field(default_factory=list)
     add_current_site_editors: bool = True
     add_current_site_sites: bool = False
 
@@ -144,11 +146,13 @@ def set_extra_field(instance, field_name, element, questions_widget_types=None)
             extra_value = element_value
         else:
             extra_value = default_value
+    if field_name == "path" and hasattr(instance, "build_path"):
+        extra_value = instance.build_path(instance.key, instance.parent)
 
     setattr(instance, field_name, extra_value)
 
 
-def set_m2m_instances(instance, field_name, element):
+def set_m2m_instances(instance, element, field_name):
     if field_name not in element:
         return
 
@@ -182,9 +186,12 @@ def set_m2m_instances(instance, field_name, element):
     getattr(instance, field_name).set(foreign_instances)
 
 
-def set_m2m_through_instances(instance, field_name, element, source_name, target_name, through_name) -> None:
+def set_m2m_through_instances(instance, element, field_name=None, source_name=None,
+                              target_name=None, through_name=None) -> None:
     if field_name not in element:
         return
+    if not all([source_name, target_name, through_name]):
+        return
 
     target_elements = element.get(field_name) or []
 
@@ -236,9 +243,12 @@ def set_m2m_through_instances(instance, field_name, element, source_name, target
             through_instance.delete()
 
 
-def set_reverse_m2m_through_instance(instance, field_name, element, source_name, target_name, through_name) -> None:
+def set_reverse_m2m_through_instance(instance, element, field_name=None, source_name=None,
+                                     target_name=None, through_name=None) -> None:
     if field_name not in element:
         return
+    if not all([source_name, target_name, through_name]):
+        return
 
     target_element = element.get(field_name)
 
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 9e6b6ad708..9846193073 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -1,42 +1,18 @@
 import logging
-from typing import Callable, Tuple
 
 from rdmo.core.imports import (
     ElementImportHelper,
-    validate_instance,
 )
 
-from .models import Attribute
 from .serializers.v1 import BaseAttributeSerializer
 from .validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
 
 logger = logging.getLogger(__name__)
 
-
-def import_attribute(
-    instance: Attribute, element: dict,
-    validators: Tuple[Callable],
-    save: bool = False) -> Attribute:
-
-    # set_foreign_field are already set in management/import.py
-    # check_permissions already done in management/import.py
-    instance.path = instance.build_path(instance.key, instance.parent)
-    validate_instance(instance, element, *validators)
-
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
 import_helper_attribute = ElementImportHelper(
     model="domain.attribute",
-    import_func=import_attribute,
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     foreign_fields=('parent',),
+    extra_fields=('path',),
     serializer=BaseAttributeSerializer,
 )
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index f8248ae14d..c9f6d4cd65 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -14,6 +14,10 @@
     set_extra_field,
     set_foreign_field,
     set_lang_field,
+    set_m2m_instances,
+    set_m2m_through_instances,
+    set_reverse_m2m_through_instance,
+    validate_instance,
 )
 from rdmo.domain.imports import import_helper_attribute
 from rdmo.options.imports import import_helper_option, import_helper_optionset
@@ -94,7 +98,6 @@ def import_element(
     model = RDMO_MODEL_PATH_MAPPER[model_path]
     user = request.user if request is not None else None
     import_helper = ELEMENT_IMPORT_HELPERS[model_path]
-    import_func = import_helper.import_func
     validators = import_helper.validators
     common_fields = import_helper.common_fields
     lang_field_names = import_helper.lang_fields
@@ -138,23 +141,32 @@ def import_element(
     for extra_field in extra_field_names:
         set_extra_field(instance, extra_field, element, questions_widget_types=questions_widget_types)
 
-    # call the element specific import method
-    instance = import_func(instance, element, validators, save)
+    # call the validators on the instance
+    validate_instance(instance, element, *validators)
 
     if element.get('errors'):
         return element
+
     if _updated and not _created:
         element['updated'] = _updated
         # and instance is not original_instance
         # keep only strings, make json serializable
         serializer = import_helper.serializer
-        changes = get_updated_changes(element, instance, original_instance, serializer, request=None)
+        changes = get_updated_changes(element, instance, original_instance, serializer, request=request)
         element['updated_and_changed'] = changes
 
     if save:
         logger.info(_msg)
         element['created'] = _created
         element['updated'] = _updated
+        instance.save()
+        for m2m_field in import_helper.m2m_instance_fields:
+            set_m2m_instances(instance, element, m2m_field)
+        for m2m_through_fields in import_helper.m2m_through_instance_fields:
+            set_m2m_through_instances(instance, element, **m2m_through_fields)
+        for reverse_m2m_fields in import_helper.reverse_m2m_through_instance_fields:
+            set_reverse_m2m_through_instance(instance, element, **reverse_m2m_fields)
+
         if import_helper.add_current_site_editors:
             instance.editors.add(current_site)
         if import_helper.add_current_site_sites:
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 44ffbed4ac..dd4ad6cea3 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -1,15 +1,7 @@
-import logging
-from typing import Callable, Tuple
-
 from rdmo.core.imports import (
     ElementImportHelper,
-    set_m2m_instances,
-    set_m2m_through_instances,
-    set_reverse_m2m_through_instance,
-    validate_instance,
 )
 
-from .models import Option, OptionSet
 from .serializers.v1 import OptionSerializer, OptionSetSerializer
 from .validators import (
     OptionLockedValidator,
@@ -18,67 +10,26 @@
     OptionUniqueURIValidator,
 )
 
-logger = logging.getLogger(__name__)
-
-
-def import_option(
-        instance: Option,
-        element: dict,
-        validators: Tuple[Callable],
-        save: bool = False,
-    ):
-    # check_permissions already done in management/import.py
-    # extra_fields are set in in management/import.py
-    validate_instance(instance, element, *validators)
-
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        set_m2m_instances(instance, 'conditions', element)
-        set_m2m_through_instances(instance, 'options', element, 'optionset', 'option', 'optionset_options')
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
 import_helper_option = ElementImportHelper(
     model="options.option",
-    import_func=import_option,
     validators=(OptionLockedValidator, OptionUniqueURIValidator),
     lang_fields=('text',),
     serializer = OptionSerializer,
-    extra_fields = ('order', 'provider_key', 'additional_input')
+    extra_fields = ('order', 'provider_key', 'additional_input'),
+    m2m_instance_fields = ('conditions', ),
+    m2m_through_instance_fields = [
+        {'field_name': 'options', 'source_name': 'optionset',
+         'target_name': 'option', 'through_name': 'optionset_options'}
+    ]
 )
 
-
-def import_optionset(
-        instance: OptionSet,
-        element: dict,
-        validators: Tuple[Callable],
-        save: bool = False,
-    ):
-    # lang_fields are already set in management/import.py
-    # check_permissions already done in management/import.py
-    # extra_fields are set in in management/import.py
-    validate_instance(instance, element, *validators)
-
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        set_reverse_m2m_through_instance(instance, 'optionset', element, 'option', 'optionset', 'option_optionsets')
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
 import_helper_optionset = ElementImportHelper(
     model="options.optionset",
-    import_func=import_optionset,
     validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
     serializer = OptionSetSerializer,
-    extra_fields=('additional_input',)
+    extra_fields=('additional_input',),
+    reverse_m2m_through_instance_fields=[
+        {'field_name': 'optionset', 'source_name': 'option',
+         'target_name': 'optionset', 'through_name': 'option_optionsets'}
+    ]
 )
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 162fcb4fce..3275c532a3 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -1,19 +1,7 @@
-import logging
-from typing import Callable, Tuple
-
 from rdmo.core.imports import (
     ElementImportHelper,
-    set_m2m_instances,
-    set_m2m_through_instances,
-    set_reverse_m2m_through_instance,
-    validate_instance,
 )
 
-from .models.catalog import Catalog
-from .models.page import Page
-from .models.question import Question
-from .models.questionset import QuestionSet
-from .models.section import Section
 from .serializers.v1 import (
     CatalogSerializer,
     PageSerializer,
@@ -34,174 +22,92 @@
     SectionUniqueURIValidator,
 )
 
-logger = logging.getLogger(__name__)
-
-
-def import_catalog(
-        instance: Catalog,
-        element: dict,
-        validators: Tuple[Callable],
-        save: bool = False,
-    ):
-    # check_permissions already done in management/import.py
-    # extra_fields are set in management/import.py
-    validate_instance(instance, element, *validators)
-
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        set_m2m_through_instances(instance, 'sections', element, 'catalog', 'section', 'catalog_sections')
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
-def import_section(
-        instance: Section,
-        element: dict,
-        validators: Tuple[Callable],
-        save: bool = False,
-    ):
-    # check_permissions already done in management/import.py
-    validate_instance(instance, element, *validators)
-
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        set_reverse_m2m_through_instance(instance, 'catalog', element, 'section', 'catalog', 'section_catalogs')
-        set_m2m_through_instances(instance, 'pages', element, 'section', 'page', 'section_pages')
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
-def import_page(
-        instance: Page,
-        element: dict,
-        validators: Tuple[Callable],
-        save: bool = False,
-    ):
-    # lang_fields are already set in management/import.py
-    # set_foreign_field are already set in management/import.py
-    # check_permissions already done in management/import.py
-    # extra_fields are set in management/import.py
-    validate_instance(instance, element, *validators)
-
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        set_m2m_instances(instance, 'conditions', element)
-        set_reverse_m2m_through_instance(instance, 'section', element, 'page', 'section', 'page_sections')
-        set_m2m_through_instances(instance, 'questionsets', element, 'page', 'questionset', 'page_questionsets')
-        set_m2m_through_instances(instance, 'questions', element, 'page', 'question', 'page_questions')
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
-def import_questionset(
-        instance: QuestionSet,
-        element: dict,
-        validators: Tuple[Callable],
-        save: bool = False,
-    ):
-    # lang_fields are already set in management/import.py
-    # set_foreign_field are already set in management/import.py
-    # check_permissions already done in management/import.py
-    # extra_fields are set in management/import.py
-    validate_instance(instance, element, *validators)
-
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        set_m2m_instances(instance, 'conditions', element)
-        set_reverse_m2m_through_instance(instance, 'page', element, 'questionset', 'page', 'questionset_pages')
-        set_reverse_m2m_through_instance(instance, 'questionset', element, 'questionset', 'parent', 'questionset_parents')  # noqa: E501
-        set_m2m_through_instances(instance, 'questionsets', element, 'parent', 'questionset', 'questionset_questionsets')  # noqa: E501
-        set_m2m_through_instances(instance, 'questions', element, 'questionset', 'question', 'questionset_questions')
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
-def import_question(
-        instance: Question,
-        element: dict,
-        validators: Tuple[Callable],
-        save: bool = False,
-    ):
-    # lang_fields are already set in management/import.py
-    # set_foreign_fields are already set in management/import.py
-    # check_permissions already done in management/import.py
-    # extra_fields are set in management/import.py
-    validate_instance(instance, element, *validators)
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        set_reverse_m2m_through_instance(instance, 'page', element, 'question', 'page', 'question_pages')
-        set_reverse_m2m_through_instance(instance, 'questionset', element, 'question', 'questionset', 'question_questionsets')  # noqa: E501
-        set_m2m_instances(instance, 'conditions', element)
-        set_m2m_instances(instance, 'optionsets', element)
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
 import_helper_catalog = ElementImportHelper(
     model="questions.catalog",
-    import_func=import_catalog,
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
     lang_fields=('help', 'title'),
     serializer = CatalogSerializer,
+    extra_fields = ('order', 'available'),
+    m2m_through_instance_fields=[
+        {'field_name': 'sections', 'source_name': 'catalog',
+         'target_name': 'section', 'through_name': 'catalog_sections'}
+    ],
     add_current_site_sites = True,
-    extra_fields = ('order', 'available')
 )
 
 import_helper_section = ElementImportHelper(
     model="questions.section",
-    import_func=import_section,
     validators=(SectionLockedValidator, SectionUniqueURIValidator),
     lang_fields=('title',),
-    serializer = SectionSerializer
+    serializer = SectionSerializer,
+    m2m_through_instance_fields=[
+        {'field_name': 'pages', 'source_name': 'section',
+         'target_name': 'page', 'through_name': 'section_pages'}
+    ],
+    reverse_m2m_through_instance_fields=[
+        {'field_name': 'catalog', 'source_name': 'section',
+         'target_name': 'catalog', 'through_name': 'section_catalogs'}
+    ]
 )
 
+
 import_helper_page = ElementImportHelper(
     model="questions.page",
-    import_func=import_page,
     validators=(PageLockedValidator, PageUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
     serializer = PageSerializer,
-    extra_fields = ('is_collection',)
+    extra_fields = ('is_collection',),
+    m2m_instance_fields = ('conditions', ),
+    m2m_through_instance_fields=[
+        {'field_name': 'questionsets', 'source_name': 'page',
+         'target_name': 'questionset', 'through_name': 'page_questionsets'},
+        {'field_name': 'questions', 'source_name': 'page',
+         'target_name': 'question', 'through_name': 'page_questions'}
+    ],
+    reverse_m2m_through_instance_fields=[
+        {'field_name': 'section', 'source_name': 'page',
+         'target_name': 'section', 'through_name': 'page_sections'}
+    ]
 )
 
+
 import_helper_questionset = ElementImportHelper(
     model="questions.questionset",
-    import_func=import_questionset,
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
     serializer = QuestionSetSerializer,
-    extra_fields = ('is_collection',)
+    extra_fields = ('is_collection',),
+    m2m_instance_fields = ('conditions', ),
+    m2m_through_instance_fields=[
+        {'field_name': 'questionsets', 'source_name': 'parent',
+         'target_name': 'questionset', 'through_name': 'questionset_questionsets'},
+        {'field_name': 'questions', 'source_name': 'questionset',
+         'target_name': 'question', 'through_name': 'questionset_questions'}
+    ],
+    reverse_m2m_through_instance_fields=[
+        {'field_name': 'page', 'source_name': 'questionset',
+         'target_name': 'page', 'through_name': 'questionset_pages'},
+        {'field_name': 'questionset', 'source_name': 'questionset',
+         'target_name': 'parent', 'through_name': 'questionset_parents'}
+    ]
 )
 
+
 import_helper_question = ElementImportHelper(
     model="questions.question",
-    import_func=import_question,
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
     lang_fields=('text', 'help', 'default_text', 'verbose_name'),
     foreign_fields=('attribute','default_option'),
-    serializer = QuestionSerializer,
-    extra_fields = ('is_collection','is_optional', 'default_external_id', 'widget_type',
-                    'value_type', 'maximum', 'minimum', 'step', 'unit','width')
+    serializer=QuestionSerializer,
+    extra_fields=('is_collection','is_optional', 'default_external_id', 'widget_type',
+                    'value_type', 'maximum', 'minimum', 'step', 'unit','width'),
+    m2m_instance_fields=('conditions', 'optionsets'),
+    reverse_m2m_through_instance_fields=[
+        {'field_name': 'page', 'source_name': 'question',
+         'target_name': 'page', 'through_name': 'question_pages'},
+        {'field_name': 'questionset', 'source_name': 'question',
+         'target_name': 'questionset', 'through_name': 'question_questionsets'}
+    ]
 )
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 278687613c..c6d8986e33 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -1,49 +1,14 @@
-import logging
-from typing import Callable, Tuple
+from rdmo.core.imports import ElementImportHelper
 
-from rdmo.core.imports import (
-    ElementImportHelper,
-    set_m2m_instances,
-    validate_instance,
-)
-
-from .models import Task
 from .serializers.v1 import TaskSerializer
 from .validators import TaskLockedValidator, TaskUniqueURIValidator
 
-logger = logging.getLogger(__name__)
-
-
-def import_task(
-        instance: Task,
-        element: dict,
-        validators: Tuple[Callable],
-        save: bool = False,
-    ):
-    # lang_fields are already set in management/import.py
-    # set_foreign_field are already set in management/import.py
-    # check_permissions already done in management/import.py
-    # extra_fields are set in in management/import.py
-    validate_instance(instance, element, *validators)
-
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        set_m2m_instances(instance, 'catalogs', element)
-        set_m2m_instances(instance, 'conditions', element)
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
 import_helper_task = ElementImportHelper(
     model="tasks.task",
-    import_func=import_task,
     validators=(TaskLockedValidator, TaskUniqueURIValidator),
     lang_fields=('title', 'text'),
     foreign_fields=('start_attribute', 'end_attribute'),
     serializer=TaskSerializer,
-    extra_fields=('order', 'days_before', 'days_after', 'available')
+    extra_fields=('order', 'days_before', 'days_after', 'available'),
+    m2m_instance_fields=('catalogs', 'conditions'),
 )
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 524d69b58e..2ac455e31c 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -1,45 +1,13 @@
-import logging
-from typing import Callable, Tuple
+from rdmo.core.imports import ElementImportHelper
 
-from rdmo.core.imports import (
-    ElementImportHelper,
-    set_m2m_instances,
-    validate_instance,
-)
-
-from .models import View
 from .serializers.v1 import ViewSerializer
 from .validators import ViewLockedValidator, ViewUniqueURIValidator
 
-logger = logging.getLogger(__name__)
-
-
-def import_view(
-        instance: View,
-        element: dict,
-        validators: Tuple[Callable],
-        save: bool = False,
-    ):
-    # check_permissions already done in management/import.py
-    # extra_fields are set in in management/import.py
-    validate_instance(instance, element, *validators)
-
-    if element.get('errors'):
-        return instance
-
-    if save:
-        instance.save()
-        set_m2m_instances(instance, 'catalogs', element)
-        # sites and editors are added in management/import.py
-
-    return instance
-
-
 import_helper_view = ElementImportHelper(
     model="views.view",
-    import_func=import_view,
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
     lang_fields=('help', 'title'),
     serializer=ViewSerializer,
-    extra_fields=('order', 'template', 'available')
+    extra_fields=('order', 'template', 'available'),
+    m2m_instance_fields=('catalogs',),
 )

From 509feaa5a576b6f8b767377bb9c4f382f7e5acd6 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 31 Jan 2024 16:57:57 +0100
Subject: [PATCH 067/205] js: refactor Links, add ShowUpdatedLink

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/common/Links.js      | 38 ++++++++++++-------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/rdmo/management/assets/js/components/common/Links.js b/rdmo/management/assets/js/components/common/Links.js
index fd9d0b594a..ac5b57a6e4 100644
--- a/rdmo/management/assets/js/components/common/Links.js
+++ b/rdmo/management/assets/js/components/common/Links.js
@@ -1,7 +1,6 @@
 import React from 'react'
 import PropTypes from 'prop-types'
 import classNames from 'classnames'
-import isEmpty from 'lodash/isEmpty'
 import isNil from 'lodash/isNil'
 import isUndefined from 'lodash/isUndefined'
 
@@ -218,47 +217,58 @@ CodeLink.propTypes = {
   order: PropTypes.number
 }
 
-const ErrorLink = ({ element, onClick }) => {
+const ErrorLink = ({ show, onClick }) => {
   return (
-    !isEmpty(element.errors) &&
+    show &&
     <Link className="element-link fa fa-warning text-danger" onClick={onClick} />
   )
 }
 
 ErrorLink.propTypes = {
-  element: PropTypes.object.isRequired,
+  show: PropTypes.bool,
   onClick: PropTypes.func.isRequired
 }
 
-
-const WarningLink = ({ element, onClick }) => {
+const WarningLink = ({ show= false, onClick }) => {
   return (
-    !isEmpty(element.warnings) &&
+    show &&
     <Link className="element-link fa fa-warning text-warning" onClick={onClick} />
   )
 }
 
 WarningLink.propTypes = {
-  element: PropTypes.object.isRequired,
+  show: PropTypes.bool,
+  onClick: PropTypes.func.isRequired
+}
+
+const ShowUpdatedLink = ({ show= false, onClick }) => {
+  return (
+    show &&
+    <Link className="element-link fa fa-pencil" onClick={onClick} />
+  )
+}
+
+ShowUpdatedLink.propTypes = {
+  show: PropTypes.bool,
   onClick: PropTypes.func.isRequired
 }
 
 
-const ShowLink = ({ element, onClick }) => {
-  const title = element.show ? gettext('Hide') : gettext('Show')
+const ShowLink = ({ show = false, onClick }) => {
+  const title = show ? gettext('Hide') : gettext('Show')
   const className = classNames({
     'element-link fa': true,
-    'fa-eye-slash': element.show,
-    'fa-eye': !element.show
+    'fa-eye-slash': !show,
+    'fa-eye': show
   })
 
   return <Link className={className} title={title} onClick={onClick} />
 }
 
 ShowLink.propTypes = {
-  element: PropTypes.object.isRequired,
+  show: PropTypes.bool,
   onClick: PropTypes.func.isRequired
 }
 
 export { EditLink, CopyLink, AddLink, AvailableLink, ToggleCurrentSiteLink, LockedLink, ShowElementsLink,
-         NestedLink, ExportLink, ExtendLink, CodeLink, ErrorLink, WarningLink, ShowLink }
+         NestedLink, ExportLink, ExtendLink, CodeLink, ErrorLink, WarningLink, ShowUpdatedLink, ShowLink }

From ee6b3b2fc70cb1136c27820852553e941be972d6 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 31 Jan 2024 17:31:57 +0100
Subject: [PATCH 068/205] chore: import handle when uri prefix endswith "/"

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/imports.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index c9f6d4cd65..53dfce4a02 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -105,7 +105,6 @@ def import_element(
     extra_field_names = import_helper.extra_fields
 
     uri = element.get('uri')
-
     # get or create instance from uri and model_path
     instance, _created = get_or_return_instance(model, uri=uri)
 
@@ -130,7 +129,14 @@ def import_element(
     # start to set values on the instance
     # set common field values from element on instance
     for common_field in common_fields:
-        setattr(instance, common_field, element.get(common_field) or '')
+        common_value = element.get(common_field) or ''
+        # handle URI Prefix ending with slash
+        if common_field == 'uri_prefix' and common_value.endswith('/'):
+            common_value = common_value.rstrip('/')
+            element[common_field] = common_value
+            if original_instance:
+                original_instance.uri_prefix = original_instance.uri_prefix.rstrip('/')
+        setattr(instance, common_field, common_value)
     # set language fields
     for lang_field_name in lang_field_names:
         set_lang_field(instance, lang_field_name, element)

From 51f4d2f29461628b811abc2188d3f115418adcc6 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 31 Jan 2024 17:33:08 +0100
Subject: [PATCH 069/205] refactor: rename var in ImportElement.js

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/ImportElement.js      | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index 293900896e..1bf5ea208a 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -1,7 +1,7 @@
 import React from 'react'
 import PropTypes from 'prop-types'
 
-import { CodeLink, WarningLink, ErrorLink, ShowLink } from '../common/Links'
+import { CodeLink, WarningLink, ErrorLink, ShowLink, ShowUpdatedLink } from '../common/Links'
 
 import Errors from './common/Errors'
 import Fields from './common/Fields'
@@ -12,27 +12,25 @@ import { codeClass, verboseNames } from '../../constants/elements'
 import { isEmpty } from 'lodash'
 
 const ImportElement = ({ config, element, importActions }) => {
-  const showFields = () => importActions.updateElement(element, {show: !element.show})
+  const updateShowField = () => importActions.updateElement(element, {show: !element.show})
   const toggleImport = () => importActions.updateElement(element, {import: !element.import})
   const updateElement = (key, value) => importActions.updateElement(element, {[key]: value})
 
   return (
     <li className="list-group-item">
       <div className="pull-right">
-        <WarningLink element={element} onClick={showFields} />
-        <ErrorLink element={element} onClick={showFields} />
-        <ShowLink element={element} onClick={showFields} />
-        {
-          element.updated && !isEmpty(element.updated_and_changed) && !element.created &&
-          <p className="element-link fa fa-pencil"></p>
-        }
+        <WarningLink show={!isEmpty(element.warnings)} onClick={updateShowField} />
+        <ErrorLink show={!isEmpty(element.errors)} onClick={updateShowField} />
+        <ShowUpdatedLink show={(!isEmpty(element.updated_and_changed) && !element.created)} onClick={updateShowField} />
+        <ShowLink show={element.show} onClick={updateShowField} />
+
       </div>
       <div className="checkbox">
         <label className="mr-5">
           <input type="checkbox" checked={element.import} onChange={toggleImport} />
           <strong>{verboseNames[element.model]}{' '}</strong>
         </label>
-        <CodeLink className={codeClass[element.model]} uri={element.uri} onClick={showFields} />
+        <CodeLink className={codeClass[element.model]} uri={element.uri} onClick={updateShowField} />
       </div>
       {
         element.show && <>

From ed66128480920cbdfc34d1efbce43d91cb9ceef4 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 31 Jan 2024 17:50:43 +0100
Subject: [PATCH 070/205] feat: draft add ImportWarnings.js

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/ImportWarnings.js    |  49 +++++++
 .../assets/js/components/main/Import.js       | 120 +++++++++---------
 2 files changed, 112 insertions(+), 57 deletions(-)
 create mode 100644 rdmo/management/assets/js/components/import/ImportWarnings.js

diff --git a/rdmo/management/assets/js/components/import/ImportWarnings.js b/rdmo/management/assets/js/components/import/ImportWarnings.js
new file mode 100644
index 0000000000..0a2791d469
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/ImportWarnings.js
@@ -0,0 +1,49 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+
+import {ShowLink} from '../common/Links'
+
+import Warnings from './common/Warnings'
+
+import get from 'lodash/get'
+
+const ImportWarnings = ({ config, elements, configActions }) => {
+
+  const updateShowWarnings = () => {
+    const currentVal = get(config, 'filter.import.warnings.show', false)
+    configActions.updateConfig('filter.import.elements.changed', !currentVal)
+  }
+
+  const showWarnings = get(config, 'filter.import.warnings.show', false)
+  // const toggleImport = () => importActions.updateElement(element, {import: !element.import})
+  // const updateElement = (key, value) => importActions.updateElement(element, {[key]: value})
+  return (
+    <li className="list-group-item">
+      <div className="pull-right">
+        <ShowLink show={showWarnings} onClick={updateShowWarnings}/>
+      </div>
+
+      <div>
+        <p>
+          <strong>{gettext('Warnings')}{' '}({elements.length}){': '}</strong>
+        </p>
+      </div>
+
+      <ul className="list-group">
+        {showWarnings && (
+          elements.map((element, index) => {
+            <Warnings key={index} element={element} showWarningTitle={true}/>
+          })
+        )}
+      </ul>
+    </li>
+  )
+}
+
+ImportWarnings.propTypes = {
+  config: PropTypes.object.isRequired,
+  elements: PropTypes.array.isRequired,
+  configActions: PropTypes.object.isRequired
+}
+
+export default ImportWarnings
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 5fb86a8638..77e4c8af4b 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -8,8 +8,9 @@ import { FilterString, FilterUriPrefix } from '../common/Filter'
 
 import ImportElement from '../import/ImportElement'
 import ImportSuccessElement from '../import/ImportSuccessElement'
+import ImportWarnings from '../import/ImportWarnings'
 import {Checkbox} from '../common/Checkboxes'
-
+import Errors from '../import/common/Errors'
 
 const Import = ({ config, imports, configActions, importActions }) => {
   const { file, elements, success } = imports
@@ -19,23 +20,16 @@ const Import = ({ config, imports, configActions, importActions }) => {
   const updatedElements = elements.filter(element => element.updated)
   const createdElements = elements.filter(element => element.created)
   const updatedAndChangedElements = elements.filter(element => element.updated && !isEmpty(element.updated_and_changed))
-  // const updatedAndSameElements = elements.filter(element => element.updated && isEmpty(element.updated_and_changed))
+
+  const importWarnings = elements.filter(element => !isEmpty(element.warnings))
   const importErrors = elements.filter(element => !isEmpty(element.errors))
-  // const importWarnings = elements.filter(element => !isEmpty(element.warnings))
+
   const searchString = get(config, 'filter.import.elements.search', '')
   const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
-  const selectFilterChanged = () => {
-    const configBool = get(config, 'filter.import.elements.changed', false)
-    if (configBool === true && updatedAndChangedElements.length === 0) {
-      return false
-    }
-    else {
-      return configBool
-    }
-  }
+  const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
 
   const filterByChanged = (elements, selectFilterChanged) => {
-    if (selectFilterChanged === true) {
+    if (selectFilterChanged === true && updatedAndChangedElements.length > 0) {
     return updatedAndChangedElements
   } else {
     return elements
@@ -68,58 +62,70 @@ const Import = ({ config, imports, configActions, importActions }) => {
                           searchString)
 
   return (
-    <div className="panel panel-default panel-import">
-      <div className="panel-heading">
-        <strong>{gettext('Import')} from: {file.name}</strong>
-        <div className="pull-right">
-          {
-            elements.length > 0 && <span>{gettext('Total')}: {elements.length} </span>
-          }
-          {
-            updatedElements.length > 0 && <span> {gettext('Updated')}: {updatedElements.length}
-            {' ('}{gettext('Changed')}: {updatedAndChangedElements.length}{') '}
+      <div className="panel panel-default panel-import">
+        <div className="panel-heading">
+          <strong>{gettext('Import')} from: {file.name}</strong>
+          <div className="pull-right">
+            {
+                elements.length > 0 && <span>{gettext('Total')}: {elements.length} </span>
+            }
+            {
+                updatedElements.length > 0 && <span> {gettext('Updated')}: {updatedElements.length}
+                  {' ('}{gettext('Changed')}: {updatedAndChangedElements.length}{') '}
               </span>
             }
-          {
-            createdElements.length > 0 && <span>{gettext('Created')}: {createdElements.length} </span>
-          }
-          {
-            importErrors.length > 0 && <span>{gettext('Errors')}: {importErrors.length} </span>
-          }
-        </div>
-      </div>
-      <div className="panel-body">
-        <div className="row">
-          <div className={'col-sm-8'}>
-            <FilterString value={get(config, 'filter.import.elements.search', '')} onChange={updateFilterString}
-                          placeholder={gettext('Filter uri')}/>
-          </div>
-          <div className="col-sm-4">
-            <FilterUriPrefix value={get(config, 'filter.import.elements.uri_prefix', '')}
-                             onChange={updateFilterUriPrefix}
-                             options={getUriPrefixes(elements)}/>
+            {
+                createdElements.length > 0 && <span>{gettext('Created')}: {createdElements.length} </span>
+            }
+            {
+                importErrors.length > 0 && <span>{gettext('Errors')}: {importErrors.length} </span>
+            }
           </div>
         </div>
-        {
-        updatedAndChangedElements.length > 0 && <div className="checkboxes">
-          <span className="mr-10">{gettext('Changed:')}</span>
-          <Checkbox label={<code className="code-questions">{gettext('Changed')}</code>}
-                    value={get(config, 'filter.import.elements.changed', true)} onChange={updateFilterChanged}/>
-        </div>
+        <div className="panel-body">
+          <div className="row">
+            <div className={'col-sm-8'}>
+              <FilterString value={get(config, 'filter.import.elements.search', '')} onChange={updateFilterString}
+                            placeholder={gettext('Filter uri')}/>
+            </div>
+            <div className="col-sm-4">
+              <FilterUriPrefix value={get(config, 'filter.import.elements.uri_prefix', '')}
+                               onChange={updateFilterUriPrefix}
+                               options={getUriPrefixes(elements)}/>
+            </div>
+          </div>
+          {
+              updatedAndChangedElements.length > 0 && <div className="checkboxes">
+                <span className="mr-10">{gettext('Changed:')}</span>
+                <Checkbox label={<code className="code-questions">{gettext('Filter changed')}</code>}
+                          value={get(config, 'filter.import.elements.changed', false)} onChange={updateFilterChanged}/>
+              </div>
           }
-      </div>
-      <ul className="list-group">
+        </div>
+        <ul className="list-group">
+        { importWarnings.length > 0 &&
+            <ImportWarnings config={config} elements={importWarnings} importActions={importActions} configActions={configActions}/>
+        }
+        </ul>
+        <ul className="list-group">
         {
-          filteredElements.map((element, index) => {
-            if (success) {
-              return <ImportSuccessElement key={index} element={element}/>
-            } else {
-              return <ImportElement key={index} config={config} element={element} importActions={importActions}/>
-            }
+          importErrors.map((element, index) => {
+            return <Errors key={index} element={element}/>
           })
         }
-      </ul>
-    </div>
+        </ul>
+        <ul className="list-group">
+          {
+            filteredElements.map((element, index) => {
+              if (success) {
+                return <ImportSuccessElement key={index} element={element}/>
+              } else {
+                return <ImportElement key={index} config={config} element={element} importActions={importActions}/>
+              }
+            })
+          }
+        </ul>
+      </div>
   )
 }
 

From f5fa507c1d134173e45d841849fd4f3388f8ea00 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 1 Feb 2024 09:42:07 +0100
Subject: [PATCH 071/205] chore: add xml version validation to UploadViewSet

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/viewsets.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index 889ef5b1aa..335e3ebc97 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -9,6 +9,9 @@
 from rest_framework.response import Response
 from rest_framework.serializers import ValidationError
 
+from packaging.version import parse
+
+from rdmo import __version__
 from rdmo.core.imports import handle_uploaded_file
 from rdmo.core.permissions import CanToggleElementCurrentSite
 from rdmo.core.utils import get_model_field_meta, is_truthy
@@ -50,6 +53,14 @@ def create(self, request, *args, **kwargs):
                 _('The content of the xml file does not consist of well formed data or markup.')
             ]})
 
+        # step 2.1: validate parsed xml
+        root_version = root.attrib.get('version') or '1.11.0'
+        parsed_version, rdmo_version = parse(root_version), parse(__version__)
+        if parsed_version > rdmo_version:
+            logger.info(f'Import failed version validation ({parsed_version} > {rdmo_version})')
+            raise ValidationError({'file': [_('This RDMO XML file does not have a valid version number.'),
+                                            f'RDMO XML Version: {root_version}']})
+
         # step 3: create element dicts from xml
         try:
             elements = flat_xml_to_elements(root)
@@ -64,7 +75,7 @@ def create(self, request, *args, **kwargs):
             raise ValidationError({'file': [_('This is not a valid RDMO XML file.')]}) from e
 
         # step 4: convert elements from previous versions
-        elements = convert_elements(elements, root.attrib.get('version'))
+        elements = convert_elements(elements, root_version)
 
         # step 5: order the elements and return
         elements = order_elements(elements)

From 0fd7f3691473bff70c8f171b3ede3d0212bded55 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 7 Feb 2024 16:17:52 +0100
Subject: [PATCH 072/205] js: remove isEmpty(oldVal) from FieldsDiffs.js

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../components/import/common/FieldsDiffs.js   | 31 +++++++++----------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index 020686f342..39246d83b9 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -4,25 +4,24 @@ import isEmpty from 'lodash/isEmpty'
 import ReactDiffViewer from 'react-diff-viewer-continued'
 import { isUndefined } from 'lodash'
 
-
 const FieldsDiffs = ({ element, field }) => {
   const newVal = element.updated_and_changed[field].uploaded
   const oldVal = element.updated_and_changed[field].current
-  return !isUndefined(element) &&
-        !isEmpty(oldVal) &&
-        !isEmpty(element.updated_and_changed) &&
-        !isUndefined(newVal) &&
-   <div className="col-sm-12">
-    <ReactDiffViewer
-        oldValue={oldVal}
-        newValue={newVal}
-        splitView={true}
-        hideLineNumbers={true}
-        leftTitle={gettext('Current')}
-        rightTitle={gettext('Uploaded')}
-        >
-        </ReactDiffViewer>
-    </div>
+  return (!isUndefined(element) &&
+          !isEmpty(element.updated_and_changed) &&
+          !isUndefined(newVal) &&
+     <div className="col-sm-12">
+      <ReactDiffViewer
+          oldValue={oldVal}
+          newValue={newVal}
+          splitView={true}
+          hideLineNumbers={true}
+          leftTitle={gettext('Current')}
+          rightTitle={gettext('Uploaded')}
+          >
+          </ReactDiffViewer>
+      </div>
+  )
 }
 
 FieldsDiffs.propTypes = {

From 49425945e5e8914fc0b67b28a167db751ee01c1a Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 7 Feb 2024 18:34:23 +0100
Subject: [PATCH 073/205] js: fix typos in UriPath component

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/import/common/UriPath.js      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/UriPath.js b/rdmo/management/assets/js/components/import/common/UriPath.js
index 70ebe4ccaf..62e9d99be1 100644
--- a/rdmo/management/assets/js/components/import/common/UriPath.js
+++ b/rdmo/management/assets/js/components/import/common/UriPath.js
@@ -2,9 +2,9 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import uniqueId from 'lodash/uniqueId'
 
-const UriPrefix = ({ element, onChange }) => {
-  const id = uniqueId('uriPrefix-'),
-        value = element.uri_path
+const UriPath = ({ element, onChange }) => {
+  const id = uniqueId('uriPath-'),
+        value = element.uri_path ?? ''
 
   return (
     <div className="form-group mb-0">
@@ -18,9 +18,9 @@ const UriPrefix = ({ element, onChange }) => {
   )
 }
 
-UriPrefix.propTypes = {
+UriPath.propTypes = {
   element: PropTypes.object.isRequired,
   onChange: PropTypes.func.isRequired
 }
 
-export default UriPrefix
+export default UriPath

From 761337b7b0de336608cc427d67914d770c02a00e Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 7 Feb 2024 18:35:45 +0100
Subject: [PATCH 074/205] js: catch null values for key and buildUri

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/js/components/import/common/Key.js | 2 +-
 rdmo/management/assets/js/utils/elements.js               | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/Key.js b/rdmo/management/assets/js/components/import/common/Key.js
index e6396b0098..fe63bac327 100644
--- a/rdmo/management/assets/js/components/import/common/Key.js
+++ b/rdmo/management/assets/js/components/import/common/Key.js
@@ -4,7 +4,7 @@ import uniqueId from 'lodash/uniqueId'
 
 const Key = ({ element, onChange }) => {
   const id = uniqueId('key-'),
-        value = element.key
+        value = element.key ?? ''
 
   return (
     <div className="form-group mb-0">
diff --git a/rdmo/management/assets/js/utils/elements.js b/rdmo/management/assets/js/utils/elements.js
index 8c006c5c48..97ca6434ed 100644
--- a/rdmo/management/assets/js/utils/elements.js
+++ b/rdmo/management/assets/js/utils/elements.js
@@ -163,10 +163,9 @@ function findDescendants(element, elementType) {
 
 const buildUri = (element) => {
   let uri = element.uri_prefix + '/' + elementModules[element.model] + '/'
-
-  if (!isUndefined(element.uri_path)) {
+  if (!isUndefined(element.uri_path) && !isNil(element.uri_path)) {
     uri += element.uri_path
-  } else if (!isUndefined(element.path)) {
+  } else if (!isUndefined(element.path) && !isNil(element.path)) {
     uri += element.path
   } else {
     uri += element.key

From 1121f960e7fcbe5f05b1b69d690b7302df9ecf43 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 7 Feb 2024 18:40:50 +0100
Subject: [PATCH 075/205] js: refactor Import Info and Warnings parts into
 components

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/ImportErrorsPanel.js |  42 ++++++
 .../js/components/import/ImportWarnings.js    |  49 -------
 .../components/import/ImportWarningsPanel.js  |  44 ++++++
 .../js/components/import/common/ImportInfo.js |  39 ++++++
 .../js/components/import/common/Warnings.js   |  54 ++++----
 .../assets/js/components/main/Import.js       | 127 +++++-------------
 6 files changed, 183 insertions(+), 172 deletions(-)
 create mode 100644 rdmo/management/assets/js/components/import/ImportErrorsPanel.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportWarnings.js
 create mode 100644 rdmo/management/assets/js/components/import/ImportWarningsPanel.js
 create mode 100644 rdmo/management/assets/js/components/import/common/ImportInfo.js

diff --git a/rdmo/management/assets/js/components/import/ImportErrorsPanel.js b/rdmo/management/assets/js/components/import/ImportErrorsPanel.js
new file mode 100644
index 0000000000..4f35381c4b
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/ImportErrorsPanel.js
@@ -0,0 +1,42 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+import {ShowLink} from '../common/Links'
+import Errors from './common/Errors'
+import get from 'lodash/get'
+
+const ImportErrorsPanel = ({ config, elements, configActions }) => {
+
+  const updateShowErrors = () => {
+    const currentVal = get(config, 'filter.import.errors.show', false)
+    configActions.updateConfig('filter.import.errors.show', !currentVal)
+  }
+
+  const showErrors = get(config, 'filter.import.errors.show', false)
+    const listErrors = elements.map((element, index) => {
+              return (<Errors key={index} element={element} />)
+              })
+  // const toggleImport = () => importActions.updateElement(element, {import: !element.import})
+  // const updateElement = (key, value) => importActions.updateElement(element, {[key]: value})
+  return (
+    <div className="panel panel-danger">
+      <div className="panel-heading"><strong>{gettext('Errors')}{' '}({elements.length}){' : '}</strong>
+        <div className="pull-right">
+          <ShowLink show={showErrors} onClick={updateShowErrors}/>
+        </div>
+      </div>
+      <div className="panel-body">
+        { showErrors &&
+          <ul className="list-group">{listErrors}</ul>
+        }
+      </div>
+    </div>
+   )
+}
+
+ImportErrorsPanel.propTypes = {
+  config: PropTypes.object.isRequired,
+  elements: PropTypes.array.isRequired,
+  configActions: PropTypes.object.isRequired
+}
+
+export default ImportErrorsPanel
diff --git a/rdmo/management/assets/js/components/import/ImportWarnings.js b/rdmo/management/assets/js/components/import/ImportWarnings.js
deleted file mode 100644
index 0a2791d469..0000000000
--- a/rdmo/management/assets/js/components/import/ImportWarnings.js
+++ /dev/null
@@ -1,49 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import {ShowLink} from '../common/Links'
-
-import Warnings from './common/Warnings'
-
-import get from 'lodash/get'
-
-const ImportWarnings = ({ config, elements, configActions }) => {
-
-  const updateShowWarnings = () => {
-    const currentVal = get(config, 'filter.import.warnings.show', false)
-    configActions.updateConfig('filter.import.elements.changed', !currentVal)
-  }
-
-  const showWarnings = get(config, 'filter.import.warnings.show', false)
-  // const toggleImport = () => importActions.updateElement(element, {import: !element.import})
-  // const updateElement = (key, value) => importActions.updateElement(element, {[key]: value})
-  return (
-    <li className="list-group-item">
-      <div className="pull-right">
-        <ShowLink show={showWarnings} onClick={updateShowWarnings}/>
-      </div>
-
-      <div>
-        <p>
-          <strong>{gettext('Warnings')}{' '}({elements.length}){': '}</strong>
-        </p>
-      </div>
-
-      <ul className="list-group">
-        {showWarnings && (
-          elements.map((element, index) => {
-            <Warnings key={index} element={element} showWarningTitle={true}/>
-          })
-        )}
-      </ul>
-    </li>
-  )
-}
-
-ImportWarnings.propTypes = {
-  config: PropTypes.object.isRequired,
-  elements: PropTypes.array.isRequired,
-  configActions: PropTypes.object.isRequired
-}
-
-export default ImportWarnings
diff --git a/rdmo/management/assets/js/components/import/ImportWarningsPanel.js b/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
new file mode 100644
index 0000000000..d71bc668ff
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
@@ -0,0 +1,44 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+
+import {ShowLink} from '../common/Links'
+
+import Warnings from './common/Warnings'
+
+import get from 'lodash/get'
+
+const ImportWarningsPanel = ({ config, elements, configActions }) => {
+
+  const updateShowWarnings = () => {
+    const currentVal = get(config, 'filter.import.warnings.show', false)
+    configActions.updateConfig('filter.import.warnings.show', !currentVal)
+  }
+  const showWarnings = get(config, 'filter.import.warnings.show', false)
+  const listWarnings = elements.map((element, index) => {
+              return (<Warnings key={index} element={element} showWarningTitle={true} success={false} />)
+              })
+  // const toggleImport = () => importActions.updateElement(element, {import: !element.import})
+  // const updateElement = (key, value) => importActions.updateElement(element, {[key]: value})
+  return (
+    <div className="panel panel-warning">
+      <div className="panel-heading"><strong onClick={updateShowWarnings}>{gettext('Warnings')}{' '}({elements.length}){': '}</strong>
+        <div className="pull-right">
+          <ShowLink show={showWarnings} onClick={updateShowWarnings}/>
+        </div>
+      </div>
+      <div className="panel-body">
+        { showWarnings &&
+        <ul className="list-group">{listWarnings}</ul>
+        }
+      </div>
+    </div>
+   )
+}
+
+ImportWarningsPanel.propTypes = {
+  config: PropTypes.object.isRequired,
+  elements: PropTypes.array.isRequired,
+  configActions: PropTypes.object.isRequired
+}
+
+export default ImportWarningsPanel
diff --git a/rdmo/management/assets/js/components/import/common/ImportInfo.js b/rdmo/management/assets/js/components/import/common/ImportInfo.js
new file mode 100644
index 0000000000..6f74d8bc70
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/common/ImportInfo.js
@@ -0,0 +1,39 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+import { isUndefined } from 'lodash'
+
+const ImportInfo = ({ elementsLength, updatedLength, createdLength, changedLength, warningsLength, errorsLength }) => {
+  return ( !isUndefined(elementsLength) && elementsLength > 0 &&
+    <div className="pull-right">
+      {
+        elementsLength > 0 && <span>{gettext('Total')}: {elementsLength} </span>
+      }
+      {
+        updatedLength > 0 && <span>{gettext('Updated')}: {updatedLength} </span>
+      }
+      { updatedLength > 0 &&
+        <span>{' ('}{gettext('Changed')}: {changedLength}{') '}</span>
+      }
+      {
+        createdLength > 0 && <span>{gettext('Created')}: {createdLength} </span>
+      }
+       {
+        warningsLength > 0 && <span>{gettext('Warnings')}: {warningsLength} </span>
+      }
+      {
+        errorsLength > 0 && <span>{gettext('Errors')}: {errorsLength} </span>
+      }
+    </div>
+  )
+}
+
+ImportInfo.propTypes = {
+  elementsLength: PropTypes.number,
+  updatedLength: PropTypes.number,
+  createdLength: PropTypes.number,
+  changedLength: PropTypes.number,
+  warningsLength: PropTypes.number,
+  errorsLength: PropTypes.number,
+}
+
+export default ImportInfo
diff --git a/rdmo/management/assets/js/components/import/common/Warnings.js b/rdmo/management/assets/js/components/import/common/Warnings.js
index be8cba22ab..52520a9066 100644
--- a/rdmo/management/assets/js/components/import/common/Warnings.js
+++ b/rdmo/management/assets/js/components/import/common/Warnings.js
@@ -1,37 +1,39 @@
 import React from 'react'
 import PropTypes from 'prop-types'
-import isEmpty from 'lodash/isEmpty'
+// import isEmpty from 'lodash/isEmpty'
 import uniqueId from 'lodash/uniqueId'
 import {codeClass} from '../../../constants/elements'
 
-const Warnings = ({ element, success = false }) => {
-  return !isEmpty(element.warnings) && <div className="row mt-10">
-    { (success === true) &&
-    <div className="col-sm-3 text-warning text-right">
-      {gettext('Warnings')}
-    </div>
-  }
-    <div className="col-sm-9">
-      <ul className="list-unstyled">
-        {
-          Object.entries(element.warnings).map(([uri, messages]) => {
+const Warnings = ({element, success = false}) => {
+  const listWarningMessages = Object.entries(element.warnings).map(([uri, messages]) => {
+    return (
+      <li key={uniqueId('warning-uri-')}><code className={codeClass['domain.attribute']}>{uri}</code>
+        <ul className="list-unstyled">
+          {
+            messages.map(message => {
               return (
-                  <li key={uniqueId('warning-uri-')}><code className={codeClass['domain.attribute']}>{uri}</code>
-                    <div className="col-sm-9">
-                      <ul className="list-unstyled">
-                        {
-                          messages.map(message => (
-                              <li className="text-warning" key={uniqueId('warning-uri-message')}>{message}</li>))
-                        }
-                      </ul>
-                    </div>
-                  </li>
+                <li className="text-warning" key={uniqueId('warning-uri-message')}>{message}</li>
               )
-          })
-        }
-      </ul>
+            })
+          }
+        </ul>
+      </li>
+    )
+  })
+
+  return (
+    <div className="row text-warning mt-10">
+      {
+        success === true && listWarningMessages.length > 0 &&
+        <div className="col-sm-3 text-right">
+          {gettext('Warnings')}
+        </div>
+      }
+      <div className="col-sm-9">
+        <ul className="list-unstyled">{listWarningMessages}</ul>
+      </div>
     </div>
-  </div>
+  )
 }
 
 Warnings.propTypes = {
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 77e4c8af4b..d2a7411c62 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -1,119 +1,52 @@
 import React from 'react'
 import PropTypes from 'prop-types'
 import isEmpty from 'lodash/isEmpty'
-import get from 'lodash/get'
-
-import { getUriPrefixes } from '../../utils/filter'
-import { FilterString, FilterUriPrefix } from '../common/Filter'
 
 import ImportElement from '../import/ImportElement'
 import ImportSuccessElement from '../import/ImportSuccessElement'
-import ImportWarnings from '../import/ImportWarnings'
-import {Checkbox} from '../common/Checkboxes'
-import Errors from '../import/common/Errors'
+import ImportWarningsPanel from '../import/ImportWarningsPanel'
+import ImportErrorsPanel from '../import/ImportErrorsPanel'
+import ImportInfo from '../import/common/ImportInfo'
+import ImportFilters from '../import/common/ImportInfo'
+
 
 const Import = ({ config, imports, configActions, importActions }) => {
   const { file, elements, success } = imports
-  const updateFilterString = (value) => configActions.updateConfig('filter.import.elements.search', value)
-  const updateFilterUriPrefix = (value) => configActions.updateConfig('filter.import.elements.uri_prefix', value)
-  const updateFilterChanged = (value) => configActions.updateConfig('filter.import.elements.changed', value)
+
+  const updatedAndChangedElements = elements.filter(element => element.updated && !isEmpty(element.updated_and_changed))
+
   const updatedElements = elements.filter(element => element.updated)
   const createdElements = elements.filter(element => element.created)
-  const updatedAndChangedElements = elements.filter(element => element.updated && !isEmpty(element.updated_and_changed))
 
   const importWarnings = elements.filter(element => !isEmpty(element.warnings))
   const importErrors = elements.filter(element => !isEmpty(element.errors))
 
-  const searchString = get(config, 'filter.import.elements.search', '')
-  const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
-  const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
-
-  const filterByChanged = (elements, selectFilterChanged) => {
-    if (selectFilterChanged === true && updatedAndChangedElements.length > 0) {
-    return updatedAndChangedElements
-  } else {
-    return elements
-  }}
-  const filterByUriSearch = (elements, searchString) => {
-    if (searchString) {
-      const lowercaseSearch = searchString.toLowerCase()
-      return elements.filter((element) =>
-        element.uri.toLowerCase().includes(lowercaseSearch)
-          // || element.title.toLowerCase().includes(lowercaseSearch)
-      )
-    } else {
-      return elements
-    }
-  }
-    const filterByUriPrefix = (elements, searchUriPrefix) => {
-    if (searchUriPrefix) {
-      return elements.filter((element) =>
-        element.uri_prefix.toLowerCase().includes(searchUriPrefix)
-          // || element.title.toLowerCase().includes(lowercaseSearch)
-      )
-    } else {
-      return elements
-    }
-  }
-  const filteredElements = filterByUriSearch(
-                                  filterByUriPrefix(
-                                        filterByChanged(elements, selectFilterChanged),
-                                  selectedUriPrefix),
-                          searchString)
+  const filteredElements = elements
 
   return (
-      <div className="panel panel-default panel-import">
-        <div className="panel-heading">
-          <strong>{gettext('Import')} from: {file.name}</strong>
-          <div className="pull-right">
-            {
-                elements.length > 0 && <span>{gettext('Total')}: {elements.length} </span>
-            }
-            {
-                updatedElements.length > 0 && <span> {gettext('Updated')}: {updatedElements.length}
-                  {' ('}{gettext('Changed')}: {updatedAndChangedElements.length}{') '}
-              </span>
-            }
-            {
-                createdElements.length > 0 && <span>{gettext('Created')}: {createdElements.length} </span>
-            }
-            {
-                importErrors.length > 0 && <span>{gettext('Errors')}: {importErrors.length} </span>
-            }
-          </div>
-        </div>
-        <div className="panel-body">
-          <div className="row">
-            <div className={'col-sm-8'}>
-              <FilterString value={get(config, 'filter.import.elements.search', '')} onChange={updateFilterString}
-                            placeholder={gettext('Filter uri')}/>
-            </div>
-            <div className="col-sm-4">
-              <FilterUriPrefix value={get(config, 'filter.import.elements.uri_prefix', '')}
-                               onChange={updateFilterUriPrefix}
-                               options={getUriPrefixes(elements)}/>
-            </div>
-          </div>
-          {
-              updatedAndChangedElements.length > 0 && <div className="checkboxes">
-                <span className="mr-10">{gettext('Changed:')}</span>
-                <Checkbox label={<code className="code-questions">{gettext('Filter changed')}</code>}
-                          value={get(config, 'filter.import.elements.changed', false)} onChange={updateFilterChanged}/>
-              </div>
-          }
-        </div>
-        <ul className="list-group">
-        { importWarnings.length > 0 &&
-            <ImportWarnings config={config} elements={importWarnings} importActions={importActions} configActions={configActions}/>
+    <div className="panel panel-default panel-import">
+      <div className="panel-heading">
+        <strong>{gettext('Import')} from: {file.name}</strong>
+        <ImportInfo elementsLength={elements.length} createdLength={createdElements.length}
+                        updatedLength={updatedElements.length} changedLength={updatedAndChangedElements.length}
+                        warningsLength={importWarnings.length} errorsLength={importErrors.length}/>
+
+      </div>
+      <div className="panel-body">
+        <ImportFilters config={config} elements={elements} updatedAndChanged={updatedAndChangedElements}
+                         importActions={importActions} configActions={configActions}/>
+
+        {
+        importWarnings.length > 0 &&
+          <ImportWarningsPanel config={config} elements={importWarnings} importActions={importActions}
+                               configActions={configActions}/>
         }
-        </ul>
-        <ul className="list-group">
         {
-          importErrors.map((element, index) => {
-            return <Errors key={index} element={element}/>
-          })
+        importErrors.length > 0 &&
+          <ImportErrorsPanel config={config} elements={importErrors} importActions={importActions}
+                               configActions={configActions}/>
         }
-        </ul>
+      </div>
         <ul className="list-group">
           {
             filteredElements.map((element, index) => {
@@ -125,7 +58,7 @@ const Import = ({ config, imports, configActions, importActions }) => {
             })
           }
         </ul>
-      </div>
+    </div>
   )
 }
 

From 49f43c97ee64f2dca4fd13312fe287b837ee01af Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 7 Feb 2024 18:41:51 +0100
Subject: [PATCH 076/205] js: cast values to String in FieldsDiffs.js

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/import/common/FieldsDiffs.js     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index 39246d83b9..1e2ee9bd34 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -5,15 +5,15 @@ import ReactDiffViewer from 'react-diff-viewer-continued'
 import { isUndefined } from 'lodash'
 
 const FieldsDiffs = ({ element, field }) => {
-  const newVal = element.updated_and_changed[field].uploaded
-  const oldVal = element.updated_and_changed[field].current
+  const newVal = element.updated_and_changed[field].uploaded ?? ''
+  const oldVal = element.updated_and_changed[field].current ?? ''
   return (!isUndefined(element) &&
           !isEmpty(element.updated_and_changed) &&
           !isUndefined(newVal) &&
      <div className="col-sm-12">
       <ReactDiffViewer
-          oldValue={oldVal}
-          newValue={newVal}
+          oldValue={oldVal.toString()}
+          newValue={newVal.toString()}
           splitView={true}
           hideLineNumbers={true}
           leftTitle={gettext('Current')}

From a54d53510c0015c45dfa268c417efaf1050f7d97 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 7 Feb 2024 18:46:57 +0100
Subject: [PATCH 077/205] js: refactor Import Info and Warnings parts into
 components

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/xml.py            |  1 +
 rdmo/management/viewsets.py | 10 ++++++----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index 5327969433..03211df268 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -26,6 +26,7 @@ def read_xml_file(file_name):
         return ET.parse(file_name).getroot()
     except Exception as e:
         log.error('Xml parsing error: ' + str(e))
+        raise e from e
 
 
 def parse_xml_string(string):
diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index 335e3ebc97..cb40b222a4 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -46,12 +46,14 @@ def create(self, request, *args, **kwargs):
             import_tmpfile_name = handle_uploaded_file(uploaded_file)
 
         # step 2: parse xml
-        root = read_xml_file(import_tmpfile_name)
-        if root is None:
+        try:
+            root = read_xml_file(import_tmpfile_name)
+        except Exception as e:
             logger.info('XML parsing error. Import failed.')
             raise ValidationError({'file': [
-                _('The content of the xml file does not consist of well formed data or markup.')
-            ]})
+                _('The content of the xml file does not consist of well formed data or markup.'),
+                _('Error') + f': {e!s}'
+            ]}) from e
 
         # step 2.1: validate parsed xml
         root_version = root.attrib.get('version') or '1.11.0'

From 9680daf8b7904f27711eaa286db01619d7889606 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 7 Feb 2024 18:50:23 +0100
Subject: [PATCH 078/205] chore: add error message to import missing key when
 build_path

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index da69937e0a..2f0f58d3a0 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -9,6 +9,7 @@
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
+from django.utils.translation import gettext_lazy as _
 
 from rest_framework.utils import model_meta
 
@@ -84,7 +85,7 @@ def get_lang_field_values(field_name: str,
         raise ValueError("Please choose one of each")
 
     ret = {}
-    for lang_code, _, lang_field in get_languages():
+    for lang_code, lang_verbose_name, lang_field in get_languages():
         name_code = f'{field_name}_{lang_code}'
         name_field = f'{field_name}_{lang_field}'
         get_key = name_field if by_field else name_code
@@ -110,6 +111,8 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None) -> None
     if not foreign_element:
         setattr(instance, field_name, None)
         return
+    if not isinstance(foreign_element, dict):
+        raise TypeError("foreign_field must be a dictionary")
 
     foreign_uri = foreign_element.get('uri')
 
@@ -147,7 +150,17 @@ def set_extra_field(instance, field_name, element, questions_widget_types=None)
         else:
             extra_value = default_value
     if field_name == "path" and hasattr(instance, "build_path"):
-        extra_value = instance.build_path(instance.key, instance.parent)
+        if instance.key:
+            extra_value = instance.build_path(instance.key, instance.parent)
+        else:
+            exception_message = _('This field may not be blank.')
+            message = '{instance_model} {instance_uri} cannot be imported (key: {exception}) .'.format(
+                instance_model=instance._meta.object_name,
+                instance_uri=element.get('uri'),
+                exception=exception_message
+            )
+            logger.info(message)
+            element['errors'].append(message)
 
     setattr(instance, field_name, extra_value)
 
@@ -328,6 +341,6 @@ def check_permissions(instance: models.Model, element_uri: str, user: models.Mod
         perms = [f'{app_label}.add_{model_name}_object']
 
     if not user.has_perms(perms, instance):
-        message = f'You have no permissions to import {instance._meta.object_name} {element_uri}.'
+        message = _('You have no permissions to import') + f'{instance._meta.object_name} {element_uri}.'
         logger.info(message)
         return message

From 98407b3794d31a55b41e8cc73821cfe5358ae7b6 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 7 Feb 2024 18:53:53 +0100
Subject: [PATCH 079/205] js: draft add ImportFilters.js component

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../components/import/common/ImportFilters.js | 87 +++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 rdmo/management/assets/js/components/import/common/ImportFilters.js

diff --git a/rdmo/management/assets/js/components/import/common/ImportFilters.js b/rdmo/management/assets/js/components/import/common/ImportFilters.js
new file mode 100644
index 0000000000..4bbcdde7ee
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/common/ImportFilters.js
@@ -0,0 +1,87 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+import {FilterString, FilterUriPrefix} from '../../common/Filter'
+import get from 'lodash/get'
+import {getUriPrefixes} from '../../../utils/filter'
+import {Checkbox} from '../../common/Checkboxes'
+
+const ImportFilters = ({ config, elements, updatedAndChanged, configActions }) => {
+
+  const updateFilterString = (value) => configActions.updateConfig('filter.import.elements.search', value)
+  const updateFilterUriPrefix = (value) => configActions.updateConfig('filter.import.elements.uri_prefix', value)
+  const updateFilterChanged = (value) => configActions.updateConfig('filter.import.elements.changed', value)
+
+  const searchString = get(config, 'filter.import.elements.search', '')
+  const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
+  const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
+
+  const filterByChanged = (elements, selectFilterChanged) => {
+    if (selectFilterChanged === true && updatedAndChanged.length > 0) {
+    return updatedAndChanged
+  } else {
+    return elements
+  }}
+  const filterByUriSearch = (elements, searchString) => {
+    if (searchString) {
+      const lowercaseSearch = searchString.toLowerCase()
+      return elements.filter((element) =>
+        element.uri.toLowerCase().includes(lowercaseSearch)
+          // || element.title.toLowerCase().includes(lowercaseSearch)
+      )
+    } else {
+      return elements
+    }
+  }
+    const filterByUriPrefix = (elements, searchUriPrefix) => {
+    if (searchUriPrefix) {
+      return elements.filter((element) =>
+        element.uri_prefix.toLowerCase().includes(searchUriPrefix)
+          // || element.title.toLowerCase().includes(lowercaseSearch)
+      )
+    } else {
+      return elements
+    }
+  }
+  const filteredElements = filterByUriSearch(
+                                  filterByUriPrefix(
+                                        filterByChanged(elements, selectFilterChanged),
+                                  selectedUriPrefix),
+                          searchString)
+
+
+  return (
+    <div className="row">
+      <div className="row">
+        <div className={'col-sm-8'}>
+          <FilterString value={get(config, 'filter.import.elements.search', '')} onChange={updateFilterString}
+                        placeholder={gettext('Filter uri')}/>
+        </div>
+        <div className="col-sm-4">
+          <FilterUriPrefix value={get(config, 'filter.import.elements.uri_prefix', '')}
+                           onChange={updateFilterUriPrefix}
+                           options={getUriPrefixes(elements)}/>
+        </div>
+      </div>
+      <div className="row-checkbox">
+        {
+          updatedAndChanged.length > 0 && <div className="checkboxes">
+            <span className="mr-10">{gettext('Changed:')}</span>
+            <Checkbox label={<code className="code-questions">{gettext('Filter changed')}</code>}
+                      value={get(config, 'filter.import.elements.changed', false)} onChange={updateFilterChanged}/>
+          </div>
+        }
+        {filteredElements.length}
+      </div>
+    </div>
+
+)
+}
+
+ImportFilters.propTypes = {
+  config: PropTypes.object.isRequired,
+  elements: PropTypes.object.isRequired,
+  updatedAndChanged: PropTypes.array,
+  configActions: PropTypes.object.isRequired,
+}
+
+export default ImportFilters

From 993939e74561802cbb84c83ecc3e41b092f1d350 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 8 Feb 2024 11:54:12 +0100
Subject: [PATCH 080/205] tests: add tests for errors in xml upload

---
 rdmo/core/xml.py                             |  5 +++--
 rdmo/management/tests/test_viewset_upload.py | 21 +++++++++++++++-----
 rdmo/management/viewsets.py                  |  2 +-
 testing/xml/error-version.xml                |  3 +++
 4 files changed, 23 insertions(+), 8 deletions(-)
 create mode 100644 testing/xml/error-version.xml

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index 03211df268..d2b6dfc5c0 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -21,12 +21,13 @@
 }
 
 
-def read_xml_file(file_name):
+def read_xml_file(file_name, raise_exception=False):
     try:
         return ET.parse(file_name).getroot()
     except Exception as e:
         log.error('Xml parsing error: ' + str(e))
-        raise e from e
+        if raise_exception:
+            raise e from e
 
 
 def parse_xml_string(string):
diff --git a/rdmo/management/tests/test_viewset_upload.py b/rdmo/management/tests/test_viewset_upload.py
index 6132d67f16..ae731b3445 100644
--- a/rdmo/management/tests/test_viewset_upload.py
+++ b/rdmo/management/tests/test_viewset_upload.py
@@ -31,6 +31,11 @@
     'list': 'v1-management:upload-list'
 }
 
+xml_error_files = [
+    ('file-does-not-exist.xml', 'may not be blank'),
+    ('xml/error.xml', 'syntax error'),
+    ('xml/error-version.xml', 'RDMO XML Version: 99'),
+]
 
 @pytest.mark.parametrize('username,password', users)
 def test_list(db, client, username, password):
@@ -110,13 +115,19 @@ def test_create_empty(db, client, username, password):
 
 
 @pytest.mark.parametrize('username,password', users)
-def test_create_error(db, client, username, password):
+@pytest.mark.parametrize('xml_file_path, error_message', xml_error_files)
+def test_create_error(db, client, username, password, xml_file_path, error_message):
     client.login(username=username, password=password)
 
-    xml_file = Path(settings.BASE_DIR) / 'xml' / 'error.xml'
-
+    xml_file = Path(settings.BASE_DIR).joinpath(xml_file_path)
     url = reverse(urlnames['list'])
-    with open(xml_file, encoding='utf8') as f:
-        response = client.post(url, {'file': f})
+    try:
+        with open(xml_file, encoding='utf8') as f:
+            response = client.post(url, {'file': f})
+    except FileNotFoundError:
+        response = client.post(url)
 
     assert response.status_code == status_map['create_error'][username], response.json()
+    if response.status_code == 400:
+        response_msg = ",".join(response.json()['file'])
+        assert error_message in response_msg
diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index cb40b222a4..25d2df0480 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -47,7 +47,7 @@ def create(self, request, *args, **kwargs):
 
         # step 2: parse xml
         try:
-            root = read_xml_file(import_tmpfile_name)
+            root = read_xml_file(import_tmpfile_name, raise_exception=True)
         except Exception as e:
             logger.info('XML parsing error. Import failed.')
             raise ValidationError({'file': [
diff --git a/testing/xml/error-version.xml b/testing/xml/error-version.xml
new file mode 100644
index 0000000000..efaaccaf70
--- /dev/null
+++ b/testing/xml/error-version.xml
@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rdmo xmlns:dc="http://purl.org/dc/elements/1.1/" created="2023-04-20T09:39:10.351808+02:00" version="99.9.9">
+</rdmo>

From 7b06876687dedb632a3006901102657abdd03e50 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 15 Feb 2024 04:18:42 +0100
Subject: [PATCH 081/205] refactor xml parsing into XmlParser class for import

---
 rdmo/core/xml.py                              | 112 +++++++++++++++++-
 rdmo/management/imports.py                    |   6 +-
 rdmo/management/management/commands/import.py |  29 +++--
 rdmo/management/tests/__init__.py             |  44 ++++---
 rdmo/management/tests/test_commands.py        |   2 +-
 .../tests/test_import_conditions.py           |  10 +-
 rdmo/management/tests/test_import_domain.py   |  11 +-
 rdmo/management/tests/test_import_options.py  |  15 ++-
 .../management/tests/test_import_questions.py |  30 +++--
 rdmo/management/tests/test_import_tasks.py    |  10 +-
 rdmo/management/tests/test_import_views.py    |  10 +-
 rdmo/management/tests/test_viewset_upload.py  |  11 +-
 rdmo/management/viewsets.py                   |  63 +++-------
 .../xml/elements/legacy/catalog-error-key.xml |  12 ++
 14 files changed, 245 insertions(+), 120 deletions(-)
 create mode 100644 testing/xml/elements/legacy/catalog-error-key.xml

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index d2b6dfc5c0..c07ae75e19 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -1,10 +1,17 @@
 import logging
 import re
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from django.utils.translation import gettext_lazy as _
 
 import defusedxml.ElementTree as ET
 from packaging.version import parse
 
-log = logging.getLogger(__name__)
+from rdmo import __version__ as VERSION
+
+logger = logging.getLogger(__name__)
 
 models = {
   'catalog': 'questions.catalog',
@@ -20,12 +27,100 @@
   'view': 'views.view'
 }
 
+DEFAULT_RDMO_XML_VERSION = '1.11.0'
+
+
+@dataclass
+class XmlParser:
+
+    file_name:str = None
+    # post init attributes
+    file:Path = None  # will be set from file_name
+    root = None
+    errors: list = field(default_factory=list)
+    parsed_elements: OrderedDict = field(default_factory=OrderedDict)
+
+    def __post_init__(self):
+        if self.file_name is None:
+            raise ValueError("File name is required.")
+        self.file = Path(self.file_name).resolve()
+        if not self.file.exists():
+            raise ValueError(f"File does not exist. {self.file}")
+
+        elements = self.parse_xml_to_elements(self.file)
+        self.parsed_elements = elements
+        self.errors.reverse()
+
+    def is_valid(self, raise_exception: bool = False) -> bool:
+        if self.errors and raise_exception:  # raise for errors
+            raise ValueError(self.errors)
+        return not bool(self.errors)
+
+    def parse_xml_to_elements(self, xml_file: Path, raise_exception:bool=False) -> None:
+        root = None
+        # step 2: parse xml
+        try:
+            root = read_xml_file(self.file, raise_exception=True)
+        except Exception as e:
+            self.errors.append(_('XML Parsing Error') + f': {e!s}')
+            logger.info('XML parsing error. Import failed.')
+
+        if root is None:
+            self.errors.append(_('The content of the xml file does not consist of well formed data or markup.'))
+            return
+        elif root.tag != 'rdmo':
+            self.errors.append(_('This XML does not contain RDMO content.'))
+            return
+        self.root = root
+
+        # step 2.1: validate parsed xml
+        root_version = root.attrib.get('version') or DEFAULT_RDMO_XML_VERSION
+        parsed_version, rdmo_version = parse(root_version), parse(VERSION)
+        if parsed_version > rdmo_version:
+            logger.info(f'Import failed version validation ({parsed_version} > {rdmo_version})')
+            self.errors.append(_('This RDMO XML file does not have a valid version number.'))
+            self.errors.append(f'RDMO XML Version: {root_version}')
+            return
+
+        # step 3: create element dicts from xml
+        try:
+            elements = flat_xml_to_elements(root)
+        except KeyError as e:
+            logger.info('Import failed with KeyError (%s)' % e)
+            self.errors.append(_('This is not a valid RDMO XML file.'))
+        except TypeError as e:
+            logger.info('Import failed with TypeError (%s)' % e)
+            self.errors.append(_('This is not a valid RDMO XML file.'))
+        except AttributeError as e:
+            logger.info('Import failed with AttributeError (%s)' % e)
+            self.errors.append(_('This is not a valid RDMO XML file.'))
+        if self.errors:
+            return
+
+        # step 3.1: validate elements for legacy versions
+        try:
+            pre_conversion_validate_legacy_elements(elements, parsed_version)
+        except ValueError as e:
+            logger.info('Import failed with ValueError (%s)' % e)
+            self.errors.append(_('This is not a valid RDMO XML file.'))
+            self.errors.append(_('XML Parsing Error') + f': {e!s}')
+        if self.errors:
+            return
+        # step 4: convert elements from previous versions
+        elements = convert_elements(elements, root_version)
+
+        # step 5: order the elements and return
+        elements = order_elements(elements)
+
+        logger.info(f'XML parsing of {self.file.name} success (length: {len(elements)}).')
+        return elements
+
 
 def read_xml_file(file_name, raise_exception=False):
     try:
         return ET.parse(file_name).getroot()
     except Exception as e:
-        log.error('Xml parsing error: ' + str(e))
+        logger.error('Xml parsing error: ' + str(e))
         if raise_exception:
             raise e from e
 
@@ -34,7 +129,7 @@ def parse_xml_string(string):
     try:
         return ET.fromstring(string)
     except Exception as e:
-        log.error('Xml parsing error: ' + str(e))
+        logger.error('Xml parsing error: ' + str(e))
 
 
 def flat_xml_to_elements(root):
@@ -121,7 +216,7 @@ def strip_ns(tag, ns_map):
 
 def convert_elements(elements, version):
     parsed_version = parse('1.11.0') if version is None else parse(version)
-
+    pre_conversion_validate_legacy_elements(elements, parsed_version)
     if parsed_version < parse('2.0.0'):
         elements = convert_legacy_elements(elements)
 
@@ -131,6 +226,13 @@ def convert_elements(elements, version):
     return elements
 
 
+def pre_conversion_validate_legacy_elements(elements, parsed_version):
+    if parsed_version < parse('2.0.0'):
+        _keys_in_elements = list(filter(lambda x: 'key' in x, elements.values()))
+        if not _keys_in_elements:
+            raise ValueError(f"Missing legacy elements, elements containing 'key' were expected for this XML with version {parsed_version}.")   # noqa: E501
+
+
 def convert_legacy_elements(elements):
     # first pass: identify pages
     for uri, element in elements.items():
@@ -230,7 +332,7 @@ def convert_additional_input(elements):
 
 
 def order_elements(elements):
-    ordered_elements = {}
+    ordered_elements = OrderedDict()
     for uri, element in elements.items():
         append_element(ordered_elements, elements, uri, element)
     return ordered_elements
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 53dfce4a02..1acaad2eeb 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -60,12 +60,12 @@
     }
 
 
-def import_elements(uploaded_elements: List[Dict], save: bool = True, request: Optional[HttpRequest] = None):
+def import_elements(uploaded_elements: Dict, save: bool = True, request: Optional[HttpRequest] = None) -> List[Dict]:
     imported_elements = []
-    uploaded_uris = {i.get('uri') for i in uploaded_elements}
+    uploaded_uris = set(uploaded_elements.keys())
     current_site = get_current_site(request)
     questions_widget_types = get_widget_types()
-    for uploaded_element in uploaded_elements:
+    for _uri, uploaded_element in uploaded_elements.items():
         element = import_element(element=uploaded_element, save=save, uploaded_uris=uploaded_uris,
                                     request=request, current_site=current_site,
                                     questions_widget_types=questions_widget_types)
diff --git a/rdmo/management/management/commands/import.py b/rdmo/management/management/commands/import.py
index 32eae1d5ab..db86203747 100644
--- a/rdmo/management/management/commands/import.py
+++ b/rdmo/management/management/commands/import.py
@@ -1,9 +1,8 @@
 import logging
 
 from django.core.management.base import BaseCommand, CommandError
-from django.utils.translation import gettext_lazy as _
 
-from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
+from rdmo.core.xml import XmlParser
 from rdmo.management.imports import import_elements
 
 logger = logging.getLogger(__name__)
@@ -15,16 +14,16 @@ def add_arguments(self, parser):
         parser.add_argument('xmlfile', action='store', default=False, help='RDMO XML export file')
 
     def handle(self, *args, **options):
-        root = read_xml_file(options['xmlfile'])
-        if root is None:
-            raise CommandError(_('The content of the xml file does not consist of well formed data or markup.'))
-        elif root.tag != 'rdmo':
-            raise CommandError(_('This XML does not contain RDMO content.'))
-        else:
-            version = root.attrib.get('version')
-            elements = flat_xml_to_elements(root)
-            elements = convert_elements(elements, version)
-            elements = order_elements(elements)
-            parsed_elements = list(elements.values())
-
-            import_elements(parsed_elements)
+
+        try:
+            xml_parser = XmlParser(file_name=options['xmlfile'])
+        except CommandError as e:
+            logger.info('Import failed with XML parsing errors.')
+            raise CommandError(str(e)) from e
+
+        # step 7: check if valid
+        if not xml_parser.is_valid():
+            logger.info('Import failed with XML validation errors.')
+            raise CommandError(" ".join(map(str, xml_parser.errors)))
+
+        import_elements(xml_parser.parsed_elements)
diff --git a/rdmo/management/tests/__init__.py b/rdmo/management/tests/__init__.py
index 697d6309a9..58aae3e159 100644
--- a/rdmo/management/tests/__init__.py
+++ b/rdmo/management/tests/__init__.py
@@ -1,16 +1,26 @@
-from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
 
+from collections import OrderedDict
+from typing import Dict, List, Tuple
+
+from rdmo.core.xml import XmlParser
+
+xml_error_files = [
+    ('file-does-not-exist.xml', 'may not be blank'),
+    ('xml/error.xml', 'syntax error'),
+    ('xml/error-version.xml', 'RDMO XML Version: 99'),
+    ('xml/elements/legacy/catalog-error-key.xml', 'Missing legacy elements'),
+]
 
 def read_xml_and_parse_to_elements(xml_file):
-    root = read_xml_file(xml_file)
-    version = root.attrib.get('version')
-    elements = flat_xml_to_elements(root)
-    elements = convert_elements(elements, version)
-    elements = order_elements(elements)
-    parsed_elements = list(elements.values())
-    return parsed_elements, root
 
-def change_fields_elements(elements, update_dict=None, n=3):
+    xml_parser = XmlParser(file_name=xml_file)
+    if xml_parser.errors:
+        _msg = "\n".join(xml_parser.errors)
+        raise ValueError(f"This test function should NOT raise any Exceptions. {_msg!s}")
+    return xml_parser.parsed_elements, xml_parser.root
+
+def _test_helper_change_fields_elements(elements, update_dict=None, n=3) -> Tuple[Dict, List]:
+    """ xml test preparation function """
 
     update_dict = update_dict if update_dict is not None else {}
     _default_update_dict = {'comment':  "this is a test comment {}"}
@@ -18,9 +28,9 @@ def change_fields_elements(elements, update_dict=None, n=3):
 
     if len(elements) < n:
         raise ValueError("Length of elements should not be smaller than n.")
-    _new_elements = []
-    _changed_elements = []
-    for _n,_element in enumerate(elements):
+    _new_elements = OrderedDict()
+    _changed_elements = OrderedDict()
+    for _n,(_uri, _element) in enumerate(elements.items()):
         if _n <= n-1:
             updated_and_changed = {}
             changed_element = _element
@@ -31,6 +41,10 @@ def change_fields_elements(elements, update_dict=None, n=3):
                 _element[k] = val
             if updated_and_changed:
                 changed_element['updated_and_changed'] = updated_and_changed
-            _changed_elements.append(changed_element)
-        _new_elements.append(_element)
-    return _new_elements, _changed_elements
+            _changed_elements[_uri] = changed_element
+        _new_elements[_uri] = _element
+    return _new_elements, list(_changed_elements.values())
+
+def _test_helper_filter_updated_and_changed(elements: List[Dict]) -> List[Dict]:
+    filtered_elements = filter(lambda x: x.get('updated_and_changed', False), elements)
+    return list(filtered_elements)
diff --git a/rdmo/management/tests/test_commands.py b/rdmo/management/tests/test_commands.py
index b9b252d776..f410a3da08 100644
--- a/rdmo/management/tests/test_commands.py
+++ b/rdmo/management/tests/test_commands.py
@@ -24,7 +24,7 @@ def test_import_error(db, settings):
     with pytest.raises(CommandError) as e:
         call_command('import', xml_file, stdout=stdout, stderr=stderr)
 
-    assert str(e.value) == 'The content of the xml file does not consist of well formed data or markup.'
+    assert str(e.value).startswith('The content of the xml file does not consist of well formed data or markup.')
 
 
 def test_import_error2(db, settings):
diff --git a/rdmo/management/tests/test_import_conditions.py b/rdmo/management/tests/test_import_conditions.py
index c001754c83..6884f230b2 100644
--- a/rdmo/management/tests/test_import_conditions.py
+++ b/rdmo/management/tests/test_import_conditions.py
@@ -5,7 +5,11 @@
 from rdmo.conditions.models import Condition
 from rdmo.management.imports import import_elements
 
-from . import change_fields_elements, read_xml_and_parse_to_elements
+from . import (
+    _test_helper_change_fields_elements,
+    _test_helper_filter_updated_and_changed,
+    read_xml_and_parse_to_elements,
+)
 
 imported_update_changes = [
     None,
@@ -44,9 +48,9 @@ def test_update_conditions_with_changed_fields(db, settings, update_dict):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
     elements, root = read_xml_and_parse_to_elements(xml_file)
-    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=7)
+    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=7)
     imported_elements = import_elements(elements)
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert len(root) == len(imported_elements) == 15
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_domain.py b/rdmo/management/tests/test_import_domain.py
index eda3e222dc..34c6de1a70 100644
--- a/rdmo/management/tests/test_import_domain.py
+++ b/rdmo/management/tests/test_import_domain.py
@@ -3,7 +3,11 @@
 from rdmo.domain.models import Attribute
 from rdmo.management.imports import import_elements
 
-from . import change_fields_elements, read_xml_and_parse_to_elements
+from . import (
+    _test_helper_change_fields_elements,
+    _test_helper_filter_updated_and_changed,
+    read_xml_and_parse_to_elements,
+)
 
 
 def test_create_domain(db, settings):
@@ -34,9 +38,10 @@ def test_update_attributes_with_changed_fields(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
 
     elements, root = read_xml_and_parse_to_elements(xml_file)
-    elements, changed_elements = change_fields_elements(elements, n=50)
+    _change_count = Attribute.objects.count() / 2
+    elements, changed_elements = _test_helper_change_fields_elements(elements, n=_change_count)
     imported_elements = import_elements(elements)
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert len(root) == len(imported_elements) == 86
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index e6d93c4cec..215cd2d785 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -5,7 +5,11 @@
 from rdmo.management.imports import import_elements
 from rdmo.options.models import Option, OptionSet
 
-from . import change_fields_elements, read_xml_and_parse_to_elements
+from . import (
+    _test_helper_change_fields_elements,
+    _test_helper_filter_updated_and_changed,
+    read_xml_and_parse_to_elements,
+)
 
 imported_update_changes = [None]
 
@@ -45,10 +49,11 @@ def test_update_optionsets_with_changed_fields(db, settings, update_dict):
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 13
     # start test with fresh options in db
-    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=7)
+    _n_change = int(Option.objects.count() / 2)
+    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=7)
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 13
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
     assert len(imported_and_changed) == len(changed_elements)
@@ -91,9 +96,9 @@ def test_update_options_with_changed_fields(db, settings, update_dict):
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 9
     # start test with fresh options in db
-    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=4)
+    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=4)
     imported_elements = import_elements(elements)
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert len(root) == len(imported_elements) == 9
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py
index 1bfe24aacf..3306b60f00 100644
--- a/rdmo/management/tests/test_import_questions.py
+++ b/rdmo/management/tests/test_import_questions.py
@@ -5,7 +5,11 @@
 from rdmo.management.imports import import_elements
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
-from . import change_fields_elements, read_xml_and_parse_to_elements
+from . import (
+    _test_helper_change_fields_elements,
+    _test_helper_filter_updated_and_changed,
+    read_xml_and_parse_to_elements,
+)
 
 imported_update_changes = [None]
 
@@ -57,9 +61,9 @@ def test_update_catalogs_with_changed_fields(db, settings, update_dict):
     assert len(root) == len(imported_elements) == 148
     # start test with fresh elements in db
 
-    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=75)
+    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=75)
     imported_elements = import_elements(elements)
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert len(imported_elements) == 148
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
@@ -114,9 +118,9 @@ def test_update_sections_with_changed_fields(db, settings, update_dict):
     assert len(root) == len(imported_elements) == 146
     # start test with fresh elements in db
 
-    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=75)
+    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=75)
     imported_elements = import_elements(elements)
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
     assert len(imported_and_changed) == len(changed_elements)
@@ -167,9 +171,9 @@ def test_update_pages_with_changed_fields(db, settings, update_dict):
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 140
     # start test with fresh elements in db
-    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=75)
+    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=75)
     imported_elements = import_elements(elements)
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert len(imported_elements) == 140
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
@@ -223,9 +227,9 @@ def test_update_questionsets_with_changed_fields(db, settings, update_dict):
     assert len(root) == 10  # two questionsets appear twice in the export file
     assert len(imported_elements) == 8
     # start test with fresh elements in db
-    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=5)
+    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=5)
     imported_elements = import_elements(elements)
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert len(imported_elements) == 8
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
@@ -275,9 +279,9 @@ def test_update_questions_with_changed_fields(db, settings, update_dict):
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 89
     # start test with fresh elements in db
-    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=45)
+    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=45)
     imported_elements = import_elements(elements)
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert len(imported_elements) == 89
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
@@ -311,7 +315,7 @@ def test_create_legacy_questions(db, settings):
     # check that all elements ended up in the catalog
     catalog = Catalog.objects.prefetch_elements().first()
     descendant_uris = {element.uri for element in catalog.descendants}
-    element_uris = {element['uri'] for element in elements if element['uri'] != catalog.uri}
+    element_uris = {element['uri'] for _uri, element in elements.items() if element['uri'] != catalog.uri}
     assert descendant_uris == element_uris
 
 
@@ -328,5 +332,5 @@ def test_update_legacy_questions(db, settings):
     # check that all elements ended up in the catalog
     catalog = Catalog.objects.prefetch_elements().first()
     descendant_uris = {element.uri for element in catalog.descendants}
-    element_uris = {element['uri'] for element in elements if element['uri'] != catalog.uri}
+    element_uris = {element['uri'] for _uri, element in elements.items() if element['uri'] != catalog.uri}
     assert descendant_uris == element_uris
diff --git a/rdmo/management/tests/test_import_tasks.py b/rdmo/management/tests/test_import_tasks.py
index 5e0c5c426c..8a031ed99c 100644
--- a/rdmo/management/tests/test_import_tasks.py
+++ b/rdmo/management/tests/test_import_tasks.py
@@ -5,7 +5,11 @@
 from rdmo.management.imports import import_elements
 from rdmo.tasks.models import Task
 
-from . import change_fields_elements, read_xml_and_parse_to_elements
+from . import (
+    _test_helper_change_fields_elements,
+    _test_helper_filter_updated_and_changed,
+    read_xml_and_parse_to_elements,
+)
 
 imported_update_changes = [None]
 
@@ -39,9 +43,9 @@ def test_update_tasks_with_changed_fields(db, settings, update_dict):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'tasks.xml'
 
     elements, root = read_xml_and_parse_to_elements(xml_file)
-    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=1)
+    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=1)
     imported_elements = import_elements(elements)
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert len(root) == len(imported_elements) == 2
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_views.py b/rdmo/management/tests/test_import_views.py
index 72a1873cce..bdf1161d01 100644
--- a/rdmo/management/tests/test_import_views.py
+++ b/rdmo/management/tests/test_import_views.py
@@ -5,7 +5,11 @@
 from rdmo.management.imports import import_elements
 from rdmo.views.models import View
 
-from . import change_fields_elements, read_xml_and_parse_to_elements
+from . import (
+    _test_helper_change_fields_elements,
+    _test_helper_filter_updated_and_changed,
+    read_xml_and_parse_to_elements,
+)
 
 imported_update_changes = [None]
 
@@ -39,9 +43,9 @@ def test_update_views_with_changed_fields(db, settings, update_dict):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'views.xml'
 
     elements, root = read_xml_and_parse_to_elements(xml_file)
-    elements, changed_elements = change_fields_elements(elements, update_dict=update_dict, n=2)
+    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=2)
     imported_elements = import_elements(elements)
-    imported_and_changed = [i for i in elements if i['updated_and_changed']]
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
     assert len(root) == len(imported_elements) == 3
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_viewset_upload.py b/rdmo/management/tests/test_viewset_upload.py
index ae731b3445..b517512162 100644
--- a/rdmo/management/tests/test_viewset_upload.py
+++ b/rdmo/management/tests/test_viewset_upload.py
@@ -7,6 +7,8 @@
 
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
+from . import xml_error_files
+
 users = (
     ('editor', 'editor'),
     ('reviewer', 'reviewer'),
@@ -31,11 +33,7 @@
     'list': 'v1-management:upload-list'
 }
 
-xml_error_files = [
-    ('file-does-not-exist.xml', 'may not be blank'),
-    ('xml/error.xml', 'syntax error'),
-    ('xml/error-version.xml', 'RDMO XML Version: 99'),
-]
+
 
 @pytest.mark.parametrize('username,password', users)
 def test_list(db, client, username, password):
@@ -125,9 +123,10 @@ def test_create_error(db, client, username, password, xml_file_path, error_messa
         with open(xml_file, encoding='utf8') as f:
             response = client.post(url, {'file': f})
     except FileNotFoundError:
+        # one test case is for a non-existent file
         response = client.post(url)
 
     assert response.status_code == status_map['create_error'][username], response.json()
     if response.status_code == 400:
-        response_msg = ",".join(response.json()['file'])
+        response_msg = " ".join(response.json()['file'])
         assert error_message in response_msg
diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index 25d2df0480..5beab57b2e 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -9,13 +9,10 @@
 from rest_framework.response import Response
 from rest_framework.serializers import ValidationError
 
-from packaging.version import parse
-
-from rdmo import __version__
 from rdmo.core.imports import handle_uploaded_file
 from rdmo.core.permissions import CanToggleElementCurrentSite
 from rdmo.core.utils import get_model_field_meta, is_truthy
-from rdmo.core.xml import convert_elements, flat_xml_to_elements, order_elements, read_xml_file
+from rdmo.core.xml import XmlParser
 
 from .constants import RDMO_MODEL_PATH_MAPPER
 from .imports import import_elements
@@ -44,49 +41,23 @@ def create(self, request, *args, **kwargs):
             raise ValidationError({'file': [_('This field may not be blank.')]}) from e
         else:
             import_tmpfile_name = handle_uploaded_file(uploaded_file)
-
-        # step 2: parse xml
         try:
-            root = read_xml_file(import_tmpfile_name, raise_exception=True)
-        except Exception as e:
-            logger.info('XML parsing error. Import failed.')
-            raise ValidationError({'file': [
-                _('The content of the xml file does not consist of well formed data or markup.'),
-                _('Error') + f': {e!s}'
-            ]}) from e
-
-        # step 2.1: validate parsed xml
-        root_version = root.attrib.get('version') or '1.11.0'
-        parsed_version, rdmo_version = parse(root_version), parse(__version__)
-        if parsed_version > rdmo_version:
-            logger.info(f'Import failed version validation ({parsed_version} > {rdmo_version})')
-            raise ValidationError({'file': [_('This RDMO XML file does not have a valid version number.'),
-                                            f'RDMO XML Version: {root_version}']})
-
-        # step 3: create element dicts from xml
-        try:
-            elements = flat_xml_to_elements(root)
-        except KeyError as e:
-            logger.info('Import failed with KeyError (%s)' % e)
-            raise ValidationError({'file': [_('This is not a valid RDMO XML file.')]}) from e
-        except TypeError as e:
-            logger.info('Import failed with TypeError (%s)' % e)
-            raise ValidationError({'file': [_('This is not a valid RDMO XML file.')]}) from e
-        except AttributeError as e:
-            logger.info('Import failed with AttributeError (%s)' % e)
-            raise ValidationError({'file': [_('This is not a valid RDMO XML file.')]}) from e
-
-        # step 4: convert elements from previous versions
-        elements = convert_elements(elements, root_version)
-
-        # step 5: order the elements and return
-        elements = order_elements(elements)
-
-        # step 6: convert elements to a list
-        _elements = list(elements.values())
+            # step 1.1: initialize XmlParser
+            # step 2-6: parse xml, validate and convert to
+            xml_parser = XmlParser(import_tmpfile_name)
+        except ValidationError as e:
+            logger.info('Import failed with XML parsing errors.')
+            raise ValidationError({'file': e}) from e
+
+        # step 7: check if valid
+        if not xml_parser.is_valid():
+            logger.info('Import failed with XML validation errors.')
+            raise ValidationError({'file': xml_parser.errors})
 
         # step 8: import the elements if save=True is set
-        imported_elements = import_elements(_elements, save=is_truthy(request.POST.get('import')), request=request)
+        imported_elements = import_elements(xml_parser.parsed_elements,
+                                            save=is_truthy(request.POST.get('import')),
+                                            request=request)
 
         # step 9: return the list of, json-serializable, elements
         return Response(imported_elements)
@@ -99,7 +70,9 @@ class ImportViewSet(viewsets.ViewSet):
     def create(self, request, *args, **kwargs):
         # step 1: store xml file as tmp file
         try:
-            elements = request.data['elements']
+            elements_data = request.data['elements']
+            _elements = filter(lambda x: 'uri' in x, elements_data)
+            elements = {i['uri']: i for i in _elements}
         except KeyError as e:
             raise ValidationError({'elements': [_('This field may not be blank.')]}) from e
         except TypeError as e:
diff --git a/testing/xml/elements/legacy/catalog-error-key.xml b/testing/xml/elements/legacy/catalog-error-key.xml
new file mode 100644
index 0000000000..de962af714
--- /dev/null
+++ b/testing/xml/elements/legacy/catalog-error-key.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rdmo xmlns:dc="http://purl.org/dc/elements/1.1/" created="2023-04-20T09:38:39.297292+02:00">
+	<catalog dc:uri="http://example.com/terms/questions/catalog">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>catalog-legacy-error-title</uri_path>
+		<dc:comment/>
+		<order>0</order>
+		<sections>
+			<section dc:uri="http://example.com/terms/questions/catalog/individual" order="1"/>
+		</sections>
+	</catalog>
+</rdmo>

From 798fca2f4f2652a0794d76b17ff5f3eb701b8317 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 16 Feb 2024 11:49:44 +0100
Subject: [PATCH 082/205] js refactor Import and ImportInfo, fix ImportFilters

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../components/import/common/ImportFilters.js | 79 +++++++------------
 .../js/components/import/common/ImportInfo.js | 44 ++++++-----
 .../assets/js/components/main/Import.js       | 55 +++++++++++--
 3 files changed, 99 insertions(+), 79 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/ImportFilters.js b/rdmo/management/assets/js/components/import/common/ImportFilters.js
index 4bbcdde7ee..47afdd0d8b 100644
--- a/rdmo/management/assets/js/components/import/common/ImportFilters.js
+++ b/rdmo/management/assets/js/components/import/common/ImportFilters.js
@@ -5,82 +5,59 @@ import get from 'lodash/get'
 import {getUriPrefixes} from '../../../utils/filter'
 import {Checkbox} from '../../common/Checkboxes'
 
-const ImportFilters = ({ config, elements, updatedAndChanged, configActions }) => {
-
+const ImportFilters = ({ config, elements, updatedAndChanged, filteredElements, configActions }) => {
+  console.log('importFilter', updatedAndChanged, elements)
   const updateFilterString = (value) => configActions.updateConfig('filter.import.elements.search', value)
+  const getValueFilterString = () => get(config, 'filter.import.elements.search', '')
   const updateFilterUriPrefix = (value) => configActions.updateConfig('filter.import.elements.uri_prefix', value)
+  const getValueFilterUirPrefix = () => get(config, 'filter.import.elements.uri_prefix', '')
   const updateFilterChanged = (value) => configActions.updateConfig('filter.import.elements.changed', value)
-
-  const searchString = get(config, 'filter.import.elements.search', '')
-  const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
-  const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
-
-  const filterByChanged = (elements, selectFilterChanged) => {
-    if (selectFilterChanged === true && updatedAndChanged.length > 0) {
-    return updatedAndChanged
-  } else {
-    return elements
-  }}
-  const filterByUriSearch = (elements, searchString) => {
-    if (searchString) {
-      const lowercaseSearch = searchString.toLowerCase()
-      return elements.filter((element) =>
-        element.uri.toLowerCase().includes(lowercaseSearch)
-          // || element.title.toLowerCase().includes(lowercaseSearch)
-      )
-    } else {
-      return elements
-    }
-  }
-    const filterByUriPrefix = (elements, searchUriPrefix) => {
-    if (searchUriPrefix) {
-      return elements.filter((element) =>
-        element.uri_prefix.toLowerCase().includes(searchUriPrefix)
-          // || element.title.toLowerCase().includes(lowercaseSearch)
-      )
-    } else {
-      return elements
-    }
-  }
-  const filteredElements = filterByUriSearch(
-                                  filterByUriPrefix(
-                                        filterByChanged(elements, selectFilterChanged),
-                                  selectedUriPrefix),
-                          searchString)
-
+  const getValueFilterChanged = () => get(config, 'filter.import.elements.changed', false)
 
   return (
+    <div className="panel-body">
     <div className="row">
       <div className="row">
         <div className={'col-sm-8'}>
-          <FilterString value={get(config, 'filter.import.elements.search', '')} onChange={updateFilterString}
+          <FilterString value={getValueFilterString()} onChange={updateFilterString}
                         placeholder={gettext('Filter uri')}/>
         </div>
         <div className="col-sm-4">
-          <FilterUriPrefix value={get(config, 'filter.import.elements.uri_prefix', '')}
+          <FilterUriPrefix value={getValueFilterUirPrefix()}
                            onChange={updateFilterUriPrefix}
                            options={getUriPrefixes(elements)}/>
         </div>
       </div>
       <div className="row-checkbox">
         {
-          updatedAndChanged.length > 0 && <div className="checkboxes">
-            <span className="mr-10">{gettext('Changed:')}</span>
-            <Checkbox label={<code className="code-questions">{gettext('Filter changed')}</code>}
-                      value={get(config, 'filter.import.elements.changed', false)} onChange={updateFilterChanged}/>
+          updatedAndChanged.length > 0 &&
+          <div className="pull-left">
+            <div className="checkboxes">
+              <span className="mr-10">{gettext('Changed:')}</span>
+              <Checkbox label={<code
+                className="code-questions">{gettext('Filter changed')}{' ('}{updatedAndChanged.length}{') '}</code>}
+                        value={getValueFilterChanged()} onChange={updateFilterChanged}/>
+            </div>
+          </div>
+        }
+        { filteredElements.length > 0 &&
+          <div className="pull-right">
+            <span>{gettext('Shown')}: {filteredElements.length} / {elements.length} </span>
           </div>
         }
-        {filteredElements.length}
+
+
       </div>
     </div>
-
-)
+    </div>
+  )
 }
 
 ImportFilters.propTypes = {
   config: PropTypes.object.isRequired,
-  elements: PropTypes.object.isRequired,
-  updatedAndChanged: PropTypes.array,
+  elements: PropTypes.array.isRequired,
+  updatedAndChanged: PropTypes.array.isRequired,
+  filteredElements: PropTypes.array.isRequired,
   configActions: PropTypes.object.isRequired,
 }
 
diff --git a/rdmo/management/assets/js/components/import/common/ImportInfo.js b/rdmo/management/assets/js/components/import/common/ImportInfo.js
index 6f74d8bc70..59d156778d 100644
--- a/rdmo/management/assets/js/components/import/common/ImportInfo.js
+++ b/rdmo/management/assets/js/components/import/common/ImportInfo.js
@@ -1,28 +1,30 @@
 import React from 'react'
 import PropTypes from 'prop-types'
-import { isUndefined } from 'lodash'
+import {isUndefined} from 'lodash'
 
-const ImportInfo = ({ elementsLength, updatedLength, createdLength, changedLength, warningsLength, errorsLength }) => {
-  return ( !isUndefined(elementsLength) && elementsLength > 0 &&
+const renderElementLengthInfo = (label, length) => length > 0
+  && <span>{gettext(label)}: {length} </span>
+
+const ImportInfo = ({
+                      elementsLength,
+                      updatedLength,
+                      createdLength,
+                      changedLength,
+                      warningsLength,
+                      errorsLength
+                    }) => {
+  if (isUndefined(elementsLength) || elementsLength === 0) {
+    return null
+  }
+
+  return (
     <div className="pull-right">
-      {
-        elementsLength > 0 && <span>{gettext('Total')}: {elementsLength} </span>
-      }
-      {
-        updatedLength > 0 && <span>{gettext('Updated')}: {updatedLength} </span>
-      }
-      { updatedLength > 0 &&
-        <span>{' ('}{gettext('Changed')}: {changedLength}{') '}</span>
-      }
-      {
-        createdLength > 0 && <span>{gettext('Created')}: {createdLength} </span>
-      }
-       {
-        warningsLength > 0 && <span>{gettext('Warnings')}: {warningsLength} </span>
-      }
-      {
-        errorsLength > 0 && <span>{gettext('Errors')}: {errorsLength} </span>
-      }
+      {renderElementLengthInfo('Total', elementsLength)}
+      {renderElementLengthInfo('Updated', updatedLength)}
+      {updatedLength > 0 && <span>{' ('}{gettext('Changed')}{': '}{changedLength}{') '}</span>}
+      {renderElementLengthInfo('Created', createdLength)}
+      {renderElementLengthInfo('Warnings', warningsLength)}
+      {renderElementLengthInfo('Errors', errorsLength)}
     </div>
   )
 }
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index d2a7411c62..b8fdd3d11f 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -7,8 +7,8 @@ import ImportSuccessElement from '../import/ImportSuccessElement'
 import ImportWarningsPanel from '../import/ImportWarningsPanel'
 import ImportErrorsPanel from '../import/ImportErrorsPanel'
 import ImportInfo from '../import/common/ImportInfo'
-import ImportFilters from '../import/common/ImportInfo'
-
+import ImportFilters from '../import/common/ImportFilters'
+import get from 'lodash/get'
 
 const Import = ({ config, imports, configActions, importActions }) => {
   const { file, elements, success } = imports
@@ -21,7 +21,44 @@ const Import = ({ config, imports, configActions, importActions }) => {
   const importWarnings = elements.filter(element => !isEmpty(element.warnings))
   const importErrors = elements.filter(element => !isEmpty(element.errors))
 
-  const filteredElements = elements
+  const searchString = get(config, 'filter.import.elements.search', '')
+  const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
+  const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
+
+  // filter func callbacks
+  const filterByChanged = (elements, selectFilterChanged, updatedAndChangedElements) => {
+    if (selectFilterChanged === true && updatedAndChangedElements.length > 0) {
+    return updatedAndChangedElements
+  } else {
+    return elements
+  }}
+  const filterByUriSearch = (elements, searchString) => {
+    if (searchString) {
+      const lowercaseSearch = searchString.toLowerCase()
+      return elements.filter((element) =>
+        element.uri.toLowerCase().includes(lowercaseSearch)
+          // || element.title.toLowerCase().includes(lowercaseSearch)
+      )
+    } else {
+      return elements
+    }
+  }
+    const filterByUriPrefix = (elements, searchUriPrefix) => {
+    if (searchUriPrefix) {
+      return elements.filter((element) =>
+        element.uri_prefix.toLowerCase().includes(searchUriPrefix)
+          // || element.title.toLowerCase().includes(lowercaseSearch)
+      )
+    } else {
+      return elements
+    }
+  }
+
+  const filteredElements = filterByUriSearch(
+                                  filterByUriPrefix(
+                                        filterByChanged(elements, selectFilterChanged, updatedAndChangedElements),
+                                  selectedUriPrefix),
+                          searchString)
 
   return (
     <div className="panel panel-default panel-import">
@@ -32,9 +69,14 @@ const Import = ({ config, imports, configActions, importActions }) => {
                         warningsLength={importWarnings.length} errorsLength={importErrors.length}/>
 
       </div>
-      <div className="panel-body">
-        <ImportFilters config={config} elements={elements} updatedAndChanged={updatedAndChangedElements}
-                         importActions={importActions} configActions={configActions}/>
+        {
+          updatedAndChangedElements.length > 0 &&
+          <ImportFilters config={config} elements={elements}
+                         updatedAndChanged={updatedAndChangedElements}
+                         filteredElements={filteredElements}
+                         configActions={configActions}
+          />
+        }
 
         {
         importWarnings.length > 0 &&
@@ -46,7 +88,6 @@ const Import = ({ config, imports, configActions, importActions }) => {
           <ImportErrorsPanel config={config} elements={importErrors} importActions={importActions}
                                configActions={configActions}/>
         }
-      </div>
         <ul className="list-group">
           {
             filteredElements.map((element, index) => {

From 8d2fe8ae51d89212f9802c34a67251d14d177a6b Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 19 Feb 2024 17:25:51 +0100
Subject: [PATCH 083/205] fix set_lang_field at import and refactor
 ElementImportHelper

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 2f0f58d3a0..2edbdbe95c 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -61,11 +61,12 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
         return f"{verbose_name} created with {uri}"
     return f"{verbose_name} {uri} updated"
 
-@dataclass
+@dataclass(kw_only=True, frozen=True)
 class ElementImportHelper:
-    model: str
-    validators: Iterable[Callable]
-    serializer: Callable
+    model: models.Model | None = field(default=None)
+    model_path: str | None = field(default=None)
+    validators: Iterable[Callable] = field(default_factory=list)
+    serializer: Callable | None = field(default=None)
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
     lang_fields: Sequence[str] = field(default_factory=list)
     foreign_fields: Sequence[str] = field(default_factory=list)
@@ -73,32 +74,39 @@ class ElementImportHelper:
     m2m_instance_fields: Sequence[str] = field(default_factory=list)
     m2m_through_instance_fields: Sequence[Dict[str, str]] = field(default_factory=list)
     reverse_m2m_through_instance_fields: Sequence[Dict[str, str]] = field(default_factory=list)
-    add_current_site_editors: bool = True
-    add_current_site_sites: bool = False
+    add_current_site_editors: bool = field(default=True)
+    add_current_site_sites: bool = field(default=False)
 
 
 def get_lang_field_values(field_name: str,
                         element: Optional[dict] = None,
                         instance: Optional[models.Model] = None,
-                        by_field: bool = True):
-    if (element and instance):
+                        get_by_lang_field_key: bool = True):
+    if (element is not None and instance is not None):
         raise ValueError("Please choose one of each")
 
-    ret = {}
+    ret = []
     for lang_code, lang_verbose_name, lang_field in get_languages():
         name_code = f'{field_name}_{lang_code}'
         name_field = f'{field_name}_{lang_field}'
-        get_key = name_field if by_field else name_code
-        set_key = name_code if by_field else name_field
+        # get_key = name_field if get_by_lang_field_key else name_code
+        # set_key = name_code if get_by_lang_field_key else name_field
+        row = {}
+        row['element_key'] = name_code
+        row['instance_field'] = name_field
         if element:
-            ret[set_key] = element.get(get_key, '')
+            row['value'] = element.get(name_code, '') or ''
         if instance:
-            ret[set_key] = getattr(instance, get_key, '')
+            row['value'] = getattr(instance, name_field, '') or ''
+        ret.append(row)
     return ret
 
 def set_lang_field(instance, field_name, element):
-    lang_fields = get_lang_field_values(field_name, element=element)
-    for field_lang_name, field_value in lang_fields.items():
+    languages_field_values = get_lang_field_values(field_name, element=element)
+    for lang_fields_value in languages_field_values:
+        field_lang_name = lang_fields_value['instance_field']
+        field_value = lang_fields_value['value']
+
         setattr(instance, field_lang_name, field_value)
 
 

From d8bbd880477bce00cabc82b89106cf7521fde2b6 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 21 Feb 2024 09:30:01 +0100
Subject: [PATCH 084/205] js add set initial state at import/uploadFileInit

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/js/reducers/importsReducer.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index cfb4273251..ad41517117 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -19,7 +19,7 @@ export default function importsReducer(state = initialState, action) {
   switch(action.type) {
     // upload file
     case 'import/uploadFileInit':
-      return {...state, file: action.file}
+      return {...state, ...initialState,  file: action.file}
     case 'elements/fetchElementsInit':
     case 'elements/fetchElementInit':
       return {...state, elements: [], errors: [], success: false}

From 12d4c3267e7e33b3bec18e7d95cfdf6887aa56b0 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 21 Feb 2024 11:15:21 +0100
Subject: [PATCH 085/205] refactor set_foreign_field import func and minor
 change in set_extra_field

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 59 +++++++++++++++++++++++++++-----------------
 1 file changed, 37 insertions(+), 22 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 2edbdbe95c..2142f42cbf 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -119,33 +119,48 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None) -> None
     if not foreign_element:
         setattr(instance, field_name, None)
         return
-    if not isinstance(foreign_element, dict):
-        raise TypeError("foreign_field must be a dictionary")
 
-    foreign_uri = foreign_element.get('uri')
+    if 'uri' not in foreign_element:
+        message = 'Foreign model can not be assigned on {instance_model}.{field_name} {instance_uri} due to missing uri.'.format( # noqa: E501
+            instance_model=instance._meta.object_name,
+            instance_uri=element.get('uri'),
+            field_name=field_name
+        )
+        logger.info(message)
+        element['errors'][element.get('uri')].append(message)
+        return
+
+    foreign_uri = foreign_element['uri']
 
     model_info = model_meta.get_field_info(instance)
     foreign_model = model_info.forward_relations[field_name].related_model
-
+    foreign_instance = None
     try:
         foreign_instance = foreign_model.objects.get(uri=foreign_uri)
-        setattr(instance, field_name, foreign_instance)
-        return
     except foreign_model.DoesNotExist:
-        # check for existence of foreign_uri in currently uploaded uris
-        uploaded_uris = uploaded_uris if uploaded_uris is not None else []
-        if foreign_uri in uploaded_uris and foreign_uri is not None:
-            setattr(instance, field_name, foreign_uri)
-            return
-
-    message = '{foreign_model} {foreign_uri} for {instance_model} {instance_uri} does not exist.'.format(
-        foreign_model=foreign_model._meta.object_name,
-        foreign_uri=foreign_uri,
-        instance_model=instance._meta.object_name,
-        instance_uri=element.get('uri')
-    )
-    logger.info(message)
-    element['warnings'][foreign_uri].append(message)
+        message = '{foreign_model} {foreign_uri} for {instance_model} {instance_uri} does not exist.'.format(
+            foreign_model=foreign_model._meta.object_name,
+            foreign_uri=foreign_uri,
+            instance_model=instance._meta.object_name,
+            instance_uri=element.get('uri')
+        )
+        logger.info(message)
+        element['warnings'][foreign_uri].append(message)
+    try:
+        if foreign_instance is not None:
+            setattr(instance, field_name, foreign_instance)
+    except ValueError:
+        message = '{foreign_model} {foreign_uri} can not be assigned on {instance_model}.{field_name} {instance_uri} .'.format( # noqa: E501
+            foreign_model=foreign_model._meta.object_name,
+            foreign_uri=foreign_uri,
+            instance_model=instance._meta.object_name,
+            instance_uri=element.get('uri'),
+            field_name=field_name,
+            )
+        logger.info(message)
+        element['errors'][foreign_uri].append(message)
+
+
 
 def set_extra_field(instance, field_name, element, questions_widget_types=None) -> None:
 
@@ -157,8 +172,8 @@ def set_extra_field(instance, field_name, element, questions_widget_types=None)
             extra_value = element_value
         else:
             extra_value = default_value
-    if field_name == "path" and hasattr(instance, "build_path"):
-        if instance.key:
+    if field_name == "path":
+        if instance.key and hasattr(instance, "build_path"):
             extra_value = instance.build_path(instance.key, instance.parent)
         else:
             exception_message = _('This field may not be blank.')

From 198f3748a85ee9d196947077ae0bc4a4c16b17bc Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 21 Feb 2024 11:18:14 +0100
Subject: [PATCH 086/205] refactor: add model fields to ElementImportHelpers

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/conditions/imports.py |  4 +++-
 rdmo/domain/imports.py     |  4 +++-
 rdmo/options/imports.py    |  7 +++++--
 rdmo/questions/imports.py  | 16 +++++++++++-----
 rdmo/tasks/imports.py      |  4 +++-
 rdmo/views/imports.py      |  4 +++-
 6 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index 8229b8d13e..838a91fcaf 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -1,10 +1,12 @@
 from rdmo.core.imports import ElementImportHelper
 
+from .models import Condition
 from .serializers.v1 import ConditionSerializer
 from .validators import ConditionLockedValidator, ConditionUniqueURIValidator
 
 import_helper_condition = ElementImportHelper(
-    model="conditions.condition",
+    model= Condition,
+    model_path = "conditions.condition",
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
     foreign_fields=('source', 'target_option'),
     serializer=ConditionSerializer,
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 9846193073..e43d719874 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -4,13 +4,15 @@
     ElementImportHelper,
 )
 
+from .models import Attribute
 from .serializers.v1 import BaseAttributeSerializer
 from .validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
 
 logger = logging.getLogger(__name__)
 
 import_helper_attribute = ElementImportHelper(
-    model="domain.attribute",
+    model=Attribute,
+    model_path="domain.attribute",
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     foreign_fields=('parent',),
     extra_fields=('path',),
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index dd4ad6cea3..2ade3d3811 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -2,6 +2,7 @@
     ElementImportHelper,
 )
 
+from .models import Option, OptionSet
 from .serializers.v1 import OptionSerializer, OptionSetSerializer
 from .validators import (
     OptionLockedValidator,
@@ -11,7 +12,8 @@
 )
 
 import_helper_option = ElementImportHelper(
-    model="options.option",
+    model = Option,
+    model_path="options.option",
     validators=(OptionLockedValidator, OptionUniqueURIValidator),
     lang_fields=('text',),
     serializer = OptionSerializer,
@@ -24,7 +26,8 @@
 )
 
 import_helper_optionset = ElementImportHelper(
-    model="options.optionset",
+    model = OptionSet,
+    model_path="options.optionset",
     validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
     serializer = OptionSetSerializer,
     extra_fields=('additional_input',),
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 3275c532a3..a2dc97509a 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -2,6 +2,7 @@
     ElementImportHelper,
 )
 
+from .models import Catalog, Page, Question, QuestionSet, Section
 from .serializers.v1 import (
     CatalogSerializer,
     PageSerializer,
@@ -23,7 +24,8 @@
 )
 
 import_helper_catalog = ElementImportHelper(
-    model="questions.catalog",
+    model = Catalog,
+    model_path="questions.catalog",
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
     lang_fields=('help', 'title'),
     serializer = CatalogSerializer,
@@ -36,7 +38,8 @@
 )
 
 import_helper_section = ElementImportHelper(
-    model="questions.section",
+    model = Section,
+    model_path="questions.section",
     validators=(SectionLockedValidator, SectionUniqueURIValidator),
     lang_fields=('title',),
     serializer = SectionSerializer,
@@ -52,7 +55,8 @@
 
 
 import_helper_page = ElementImportHelper(
-    model="questions.page",
+    model = Page,
+    model_path="questions.page",
     validators=(PageLockedValidator, PageUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
@@ -73,7 +77,8 @@
 
 
 import_helper_questionset = ElementImportHelper(
-    model="questions.questionset",
+    model = QuestionSet,
+    model_path = "questions.questionset",
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
@@ -96,7 +101,8 @@
 
 
 import_helper_question = ElementImportHelper(
-    model="questions.question",
+    model = Question,
+    model_path = "questions.question",
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
     lang_fields=('text', 'help', 'default_text', 'verbose_name'),
     foreign_fields=('attribute','default_option'),
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index c6d8986e33..6508a2b15e 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -1,10 +1,12 @@
 from rdmo.core.imports import ElementImportHelper
 
+from .models import Task
 from .serializers.v1 import TaskSerializer
 from .validators import TaskLockedValidator, TaskUniqueURIValidator
 
 import_helper_task = ElementImportHelper(
-    model="tasks.task",
+    model=Task,
+    model_path="tasks.task",
     validators=(TaskLockedValidator, TaskUniqueURIValidator),
     lang_fields=('title', 'text'),
     foreign_fields=('start_attribute', 'end_attribute'),
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 2ac455e31c..fa46b92270 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -1,10 +1,12 @@
 from rdmo.core.imports import ElementImportHelper
 
+from .models import View
 from .serializers.v1 import ViewSerializer
 from .validators import ViewLockedValidator, ViewUniqueURIValidator
 
 import_helper_view = ElementImportHelper(
-    model="views.view",
+    model=View,
+    model_path="views.view",
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
     lang_fields=('help', 'title'),
     serializer=ViewSerializer,

From 4f7748f6a9f1756327fb9b2ec8f2cb922e224113 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 21 Feb 2024 11:20:57 +0100
Subject: [PATCH 087/205] tests: refactor import tests to use
 delete_all_objects func

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/__init__.py             |  4 ++
 rdmo/management/tests/test_import_options.py  | 13 ++--
 .../management/tests/test_import_questions.py | 60 ++++---------------
 rdmo/management/tests/test_viewset_import.py  |  8 +--
 .../tests/test_viewset_import_multisite.py    |  8 +--
 rdmo/management/tests/test_viewset_upload.py  |  8 +--
 6 files changed, 29 insertions(+), 72 deletions(-)

diff --git a/rdmo/management/tests/__init__.py b/rdmo/management/tests/__init__.py
index 58aae3e159..89aa249e79 100644
--- a/rdmo/management/tests/__init__.py
+++ b/rdmo/management/tests/__init__.py
@@ -11,6 +11,10 @@
     ('xml/elements/legacy/catalog-error-key.xml', 'Missing legacy elements'),
 ]
 
+def delete_all_objects(db_models: List):
+    for db_model in db_models:
+        db_model.objects.all().delete()
+
 def read_xml_and_parse_to_elements(xml_file):
 
     xml_parser = XmlParser(file_name=xml_file)
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 215cd2d785..7e0cce0e2f 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -8,14 +8,14 @@
 from . import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
+    delete_all_objects,
     read_xml_and_parse_to_elements,
 )
 
 imported_update_changes = [None]
 
 def test_create_optionsets(db, settings):
-    OptionSet.objects.all().delete()
-    Option.objects.all().delete()
+    delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
     elements, root = read_xml_and_parse_to_elements(xml_file)
@@ -41,8 +41,7 @@ def test_update_optionsets(db, settings):
 
 @pytest.mark.parametrize('update_dict', imported_update_changes)
 def test_update_optionsets_with_changed_fields(db, settings, update_dict):
-    OptionSet.objects.all().delete()
-    Option.objects.all().delete()
+    delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
     elements, root = read_xml_and_parse_to_elements(xml_file)
@@ -88,8 +87,7 @@ def test_update_options(db, settings):
 
 @pytest.mark.parametrize('update_dict', imported_update_changes)
 def test_update_options_with_changed_fields(db, settings, update_dict):
-    OptionSet.objects.all().delete()
-    Option.objects.all().delete()
+    delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
     elements, root = read_xml_and_parse_to_elements(xml_file)
@@ -109,8 +107,7 @@ def test_update_options_with_changed_fields(db, settings, update_dict):
 
 
 def test_create_legacy_options(db, settings):
-    OptionSet.objects.all().delete()
-    Option.objects.all().delete()
+    delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'options.xml'
 
diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py
index 3306b60f00..c8d86c5c5e 100644
--- a/rdmo/management/tests/test_import_questions.py
+++ b/rdmo/management/tests/test_import_questions.py
@@ -8,6 +8,7 @@
 from . import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
+    delete_all_objects,
     read_xml_and_parse_to_elements,
 )
 
@@ -15,11 +16,7 @@
 
 
 def test_create_catalogs(db, settings):
-    Catalog.objects.all().delete()
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
 
@@ -49,11 +46,7 @@ def test_update_catalogs(db, settings):
 
 @pytest.mark.parametrize('update_dict', imported_update_changes)
 def test_update_catalogs_with_changed_fields(db, settings, update_dict):
-    Catalog.objects.all().delete()
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
     elements, root = read_xml_and_parse_to_elements(xml_file)
@@ -74,10 +67,7 @@ def test_update_catalogs_with_changed_fields(db, settings, update_dict):
 
 
 def test_create_sections(db, settings):
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
 
@@ -106,11 +96,7 @@ def test_update_sections(db, settings):
 
 @pytest.mark.parametrize('update_dict', imported_update_changes)
 def test_update_sections_with_changed_fields(db, settings, update_dict):
-    Catalog.objects.all().delete()
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
     elements, root = read_xml_and_parse_to_elements(xml_file)
@@ -130,9 +116,7 @@ def test_update_sections_with_changed_fields(db, settings, update_dict):
 
 
 def test_create_pages(db, settings):
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
 
@@ -160,11 +144,7 @@ def test_update_pages(db, settings):
 
 @pytest.mark.parametrize('update_dict', imported_update_changes)
 def test_update_pages_with_changed_fields(db, settings, update_dict):
-    Catalog.objects.all().delete()
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
     elements, root = read_xml_and_parse_to_elements(xml_file)
@@ -184,9 +164,7 @@ def test_update_pages_with_changed_fields(db, settings, update_dict):
 
 
 def test_create_questionsets(db, settings):
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
 
@@ -215,11 +193,7 @@ def test_update_questionsets(db, settings):
 
 @pytest.mark.parametrize('update_dict', imported_update_changes)
 def test_update_questionsets_with_changed_fields(db, settings, update_dict):
-    Catalog.objects.all().delete()
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
     elements, root = read_xml_and_parse_to_elements(xml_file)
@@ -240,9 +214,7 @@ def test_update_questionsets_with_changed_fields(db, settings, update_dict):
 
 
 def test_create_questions(db, settings):
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
 
@@ -268,11 +240,7 @@ def test_update_questions(db, settings):
 
 @pytest.mark.parametrize('update_dict', imported_update_changes)
 def test_update_questions_with_changed_fields(db, settings, update_dict):
-    Catalog.objects.all().delete()
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
     elements, root = read_xml_and_parse_to_elements(xml_file)
@@ -292,11 +260,7 @@ def test_update_questions_with_changed_fields(db, settings, update_dict):
 
 
 def test_create_legacy_questions(db, settings):
-    Catalog.objects.all().delete()
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'questions.xml'
 
diff --git a/rdmo/management/tests/test_viewset_import.py b/rdmo/management/tests/test_viewset_import.py
index 23cb6742d5..3791d14200 100644
--- a/rdmo/management/tests/test_viewset_import.py
+++ b/rdmo/management/tests/test_viewset_import.py
@@ -4,6 +4,8 @@
 
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
+from . import delete_all_objects
+
 users = (
     ('editor', 'editor'),
     ('reviewer', 'reviewer'),
@@ -40,11 +42,7 @@ def test_list(db, client, username, password):
 
 @pytest.mark.parametrize('username,password', users)
 def test_create_create(db, client, username, password, json_data):
-    Catalog.objects.all().delete()
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     client.login(username=username, password=password)
 
diff --git a/rdmo/management/tests/test_viewset_import_multisite.py b/rdmo/management/tests/test_viewset_import_multisite.py
index bfc8dc1d0d..17234f0b99 100644
--- a/rdmo/management/tests/test_viewset_import_multisite.py
+++ b/rdmo/management/tests/test_viewset_import_multisite.py
@@ -6,6 +6,8 @@
 from rdmo.core.tests.utils import get_obj_perms_status_code
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
+from . import delete_all_objects
+
 status_map = {
     'list': {
         'default': 405, 'anonymous': 401
@@ -41,11 +43,7 @@ def test_list(db, client, username, password):
 
 @pytest.mark.parametrize('username,password', users)
 def test_create_create(db, client, username, password, json_data):
-    Catalog.objects.all().delete()
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     client.login(username=username, password=password)
 
diff --git a/rdmo/management/tests/test_viewset_upload.py b/rdmo/management/tests/test_viewset_upload.py
index b517512162..97c2481b89 100644
--- a/rdmo/management/tests/test_viewset_upload.py
+++ b/rdmo/management/tests/test_viewset_upload.py
@@ -7,7 +7,7 @@
 
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
-from . import xml_error_files
+from . import delete_all_objects, xml_error_files
 
 users = (
     ('editor', 'editor'),
@@ -65,11 +65,7 @@ def test_create(db, client, username, password):
 
 @pytest.mark.parametrize('username,password', users)
 def test_create_import_create(db, client, username, password):
-    Catalog.objects.all().delete()
-    Section.objects.all().delete()
-    Page.objects.all().delete()
-    QuestionSet.objects.all().delete()
-    Question.objects.all().delete()
+    delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     client.login(username=username, password=password)
 

From e4afb689a22728f69fa8b441df5f9b197241345f Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 21 Feb 2024 11:22:43 +0100
Subject: [PATCH 088/205] refactor: check for type parsed version

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/xml.py | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index c07ae75e19..353b7ddeff 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -7,7 +7,7 @@
 from django.utils.translation import gettext_lazy as _
 
 import defusedxml.ElementTree as ET
-from packaging.version import parse
+from packaging.version import Version, parse
 
 from rdmo import __version__ as VERSION
 
@@ -74,15 +74,16 @@ def parse_xml_to_elements(self, xml_file: Path, raise_exception:bool=False) -> N
         self.root = root
 
         # step 2.1: validate parsed xml
-        root_version = root.attrib.get('version') or DEFAULT_RDMO_XML_VERSION
-        parsed_version, rdmo_version = parse(root_version), parse(VERSION)
-        if parsed_version > rdmo_version:
-            logger.info(f'Import failed version validation ({parsed_version} > {rdmo_version})')
+        unparsed_root_version = root.attrib.get('version') or DEFAULT_RDMO_XML_VERSION
+        root_version, rdmo_version = parse(unparsed_root_version), parse(VERSION)
+        if root_version > rdmo_version:
+            logger.info(f'Import failed version validation ({root_version} > {rdmo_version})')
             self.errors.append(_('This RDMO XML file does not have a valid version number.'))
             self.errors.append(f'RDMO XML Version: {root_version}')
             return
 
         # step 3: create element dicts from xml
+        elements = OrderedDict()
         try:
             elements = flat_xml_to_elements(root)
         except KeyError as e:
@@ -99,11 +100,11 @@ def parse_xml_to_elements(self, xml_file: Path, raise_exception:bool=False) -> N
 
         # step 3.1: validate elements for legacy versions
         try:
-            pre_conversion_validate_legacy_elements(elements, parsed_version)
+            pre_conversion_validate_legacy_elements(elements, root_version)
         except ValueError as e:
             logger.info('Import failed with ValueError (%s)' % e)
-            self.errors.append(_('This is not a valid RDMO XML file.'))
             self.errors.append(_('XML Parsing Error') + f': {e!s}')
+            self.errors.append(_('This is not a valid RDMO XML file.'))
         if self.errors:
             return
         # step 4: convert elements from previous versions
@@ -214,23 +215,24 @@ def strip_ns(tag, ns_map):
     return tag
 
 
-def convert_elements(elements, version):
-    parsed_version = parse('1.11.0') if version is None else parse(version)
-    pre_conversion_validate_legacy_elements(elements, parsed_version)
-    if parsed_version < parse('2.0.0'):
+def convert_elements(elements, version: Version):
+    if not isinstance(version, Version):
+        raise TypeError('Version should be a parsed version type. (parse(version))')
+    pre_conversion_validate_legacy_elements(elements, version)
+    if version < parse('2.0.0'):
         elements = convert_legacy_elements(elements)
 
-    if parsed_version < parse('2.1.0'):
+    if version < parse('2.1.0'):
         elements = convert_additional_input(elements)
 
     return elements
 
 
-def pre_conversion_validate_legacy_elements(elements, parsed_version):
-    if parsed_version < parse('2.0.0'):
+def pre_conversion_validate_legacy_elements(elements, version: Version) -> None:
+    if version < parse('2.0.0'):
         _keys_in_elements = list(filter(lambda x: 'key' in x, elements.values()))
         if not _keys_in_elements:
-            raise ValueError(f"Missing legacy elements, elements containing 'key' were expected for this XML with version {parsed_version}.")   # noqa: E501
+            raise ValueError(f"Missing legacy elements, elements containing 'key' were expected for this XML with version {version}.")   # noqa: E501
 
 
 def convert_legacy_elements(elements):

From 32ad357177a4e40ef5de7908d6c0960cae80432a Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 22 Feb 2024 18:28:45 +0100
Subject: [PATCH 089/205] tests: refactor frontend tests and add
 frontend_import test

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/fixtures_frontend.py    |  50 ++++++++
 rdmo/management/tests/test_frontend.py        | 116 +++++-------------
 rdmo/management/tests/test_frontend_import.py |  67 ++++++++++
 3 files changed, 146 insertions(+), 87 deletions(-)
 create mode 100644 rdmo/management/tests/fixtures_frontend.py
 create mode 100644 rdmo/management/tests/test_frontend_import.py

diff --git a/rdmo/management/tests/fixtures_frontend.py b/rdmo/management/tests/fixtures_frontend.py
new file mode 100644
index 0000000000..5808c00714
--- /dev/null
+++ b/rdmo/management/tests/fixtures_frontend.py
@@ -0,0 +1,50 @@
+import pytest
+
+from django.core.management import call_command
+
+from playwright.sync_api import BrowserType, Page, expect
+from pytest_django.live_server_helper import LiveServer
+
+from rdmo.accounts.utils import set_group_permissions
+
+
+@pytest.fixture(scope="function")
+def e2e_tests_django_db_setup(django_db_setup, django_db_blocker, fixtures):
+    """Set up database and populate with fixtures, that get restored for every test case."""
+    with django_db_blocker.unblock():
+        call_command("loaddata", *fixtures)
+        set_group_permissions()
+
+
+
+@pytest.fixture
+def base_url_page(live_server: LiveServer, browser: BrowserType) -> Page:
+    """Enable playwright to address URLs with base URL automatically prefixed."""
+    context = browser.new_context(base_url=live_server.url)
+    page = context.new_page()
+    yield page
+    context.close()
+
+
+# helper function for logging in the user
+def login_user(page: Page, username: str, password: str) -> Page:
+    page.goto("/account/login")
+    page.get_by_label("Username").fill(username, timeout=5000)
+    page.get_by_label("Password").fill(password)
+    page.get_by_role("button", name="Login").click()
+    page.goto("/management")
+    return page
+
+def logout_user(page: Page):
+    page.goto("/account/logout")
+    page.get_by_role("button", name="Logout").click()
+    expect(page).to_have_url('/')
+    return page
+
+@pytest.fixture(scope="function")
+def logged_in_user(e2e_tests_django_db_setup, base_url_page, username:str, password: str) -> Page:
+    """Log in as admin user through Django login UI, returns logged in page for e2e tests."""
+    # breakpoint()
+    page = login_user(base_url_page, username, password)
+    yield page
+    logout_user(page)
diff --git a/rdmo/management/tests/test_frontend.py b/rdmo/management/tests/test_frontend.py
index 3bf05f75d1..2143ae777c 100644
--- a/rdmo/management/tests/test_frontend.py
+++ b/rdmo/management/tests/test_frontend.py
@@ -1,101 +1,38 @@
+# ruff: noqa: F811
 import os
 import re
-from dataclasses import dataclass
 from urllib.parse import urlparse
 
 import pytest
 
-from django.core.management import call_command
-
 from playwright.sync_api import Page, expect
-from pytest_django.live_server_helper import LiveServer
 
-from rdmo.accounts.utils import set_group_permissions
 from rdmo.conditions.models import Condition
-from rdmo.core.models import Model
 from rdmo.domain.models import Attribute
-from rdmo.options.models import Option, OptionSet
-from rdmo.questions.models import Catalog, Question, Section
-from rdmo.questions.models import Page as PageModel
-from rdmo.questions.models.questionset import QuestionSet
-from rdmo.tasks.models import Task
-from rdmo.views.models import View
+from rdmo.questions.models import Catalog
+
+from .fixtures_frontend import (
+    base_url_page,  # noqa: F401
+    e2e_tests_django_db_setup,  # noqa: F401
+    logged_in_user,  # noqa: F401
+)
+
+# logged_in_user,  # ruff: noqa: F811
+from .helpers_models import ModelHelper, model_helpers
 
 pytestmark = pytest.mark.e2e
 
 # needed for playwright to run
 os.environ.setdefault("DJANGO_ALLOW_ASYNC_UNSAFE", "true")
 
+test_users = [('editor', 'editor')]
 
-@dataclass
-class ModelHelper:
-    """Helper class to bundle information about models for test cases."""
-
-    model: Model
-    form_field: str = "URI Path"
-    db_field: str = "uri_path"
-    has_nested: bool = False
-
-    @property
-    def url(self) -> str:
-        return f"{self.model._meta.model_name}s"
-
-    @property
-    def verbose_name(self) -> str:
-        """Return the verbose_name for the model."""
-        return self.model._meta.verbose_name
-
-    @property
-    def verbose_name_plural(self) -> str:
-        """Return the verbose_name_plural for the model."""
-        return self.model._meta.verbose_name_plural
-
-
-@pytest.fixture(scope="function")
-def e2e_tests_django_db_setup(django_db_setup, django_db_blocker, fixtures):
-    """Set up database and populate with fixtures, that get restored for every test case."""
-    with django_db_blocker.unblock():
-        call_command("loaddata", *fixtures)
-        set_group_permissions()
-
-
-@pytest.fixture(scope="session")
-def base_url(live_server: LiveServer) -> str:
-    """Enable playwright to address URLs with base URL automatically prefixed."""
-    return live_server.url
-
-
-@pytest.fixture
-def logged_in_admin_user(e2e_tests_django_db_setup, page: Page) -> Page:
-    """Log in as admin user through django login UI, returns logged in page for e2e tests."""
-    page.goto("/account/login")
-    page.get_by_label("Username").fill("admin", timeout=5000)
-    page.get_by_label("Password").fill("admin")
-    page.get_by_role("button", name="Login").click()
-    page.goto("/management")
-    yield page
-
-
-model_helpers = (
-    ModelHelper(Catalog, has_nested=True),
-    ModelHelper(Section, has_nested=True),
-    ModelHelper(PageModel, has_nested=True),
-    ModelHelper(QuestionSet, has_nested=True),
-    ModelHelper(Question),
-    ModelHelper(
-        Attribute, has_nested=True, form_field="Key", db_field="key"
-    ),
-    ModelHelper(OptionSet, has_nested=True),
-    ModelHelper(Option),
-    ModelHelper(Condition),
-    ModelHelper(Task),
-    ModelHelper(View),
-)
-
+@pytest.mark.parametrize("username, password", test_users)
 @pytest.mark.parametrize("helper", model_helpers)
-def test_management_navigation(logged_in_admin_user: Page, helper: ModelHelper) -> None:
+def test_management_navigation(logged_in_user: Page, helper: ModelHelper, username: str, password: str) -> None:
     """Test that each content type is available through the navigation."""
-    page = logged_in_admin_user
+    # breakpoint()
+    page = logged_in_user
     expect(page.get_by_role("heading", name="Management")).to_be_visible()
 
     # click a link in the navigation
@@ -114,22 +51,25 @@ def test_management_navigation(logged_in_admin_user: Page, helper: ModelHelper)
         page.screenshot(path="screenshots/management-navigation-catalog.png", full_page=True)
 
 
+
+@pytest.mark.parametrize("username, password", test_users)
 @pytest.mark.parametrize("helper", model_helpers)
-def test_management_has_items(logged_in_admin_user: Page, helper: ModelHelper) -> None:
+def test_management_has_items(logged_in_user: Page, helper: ModelHelper) -> None:
     """Test all items in database are visible in management UI."""
-    page = logged_in_admin_user
+    page = logged_in_user
     num_items_in_database = helper.model.objects.count()
     page.goto(f"/management/{helper.url}")
     items_in_ui = page.locator(".list-group > .list-group-item")
     expect(items_in_ui).to_have_count(num_items_in_database)
 
 
+@pytest.mark.parametrize("username, password", test_users)
 @pytest.mark.parametrize("helper", model_helpers)
 def test_management_nested_view(
-    logged_in_admin_user: Page, helper: ModelHelper
+    logged_in_user: Page, helper: ModelHelper
 ) -> None:
     """For each element type, that has a nested view, click the first example."""
-    page = logged_in_admin_user
+    page = logged_in_user
     page.goto(f"/management/{helper.url}")
     # Open nested view for element type
     if helper.has_nested:
@@ -138,12 +78,13 @@ def test_management_nested_view(
         expect(page.locator(".panel-default > .panel-body").first).to_be_visible()
 
 
+@pytest.mark.parametrize("username, password", test_users)
 @pytest.mark.parametrize("helper", model_helpers)
 def test_management_create_model(
-    logged_in_admin_user: Page, helper: ModelHelper
+    logged_in_user: Page, helper: ModelHelper
 ) -> None:
     """Test management UI can create objects in the database."""
-    page = logged_in_admin_user
+    page = logged_in_user
     num_objects_at_start = helper.model.objects.count()
     page.goto(f"/management/{helper.url}")
     # click "New" button
@@ -174,9 +115,10 @@ def test_management_create_model(
     assert helper.model.objects.get(**query)
 
 
+@pytest.mark.parametrize("username, password", test_users)
 @pytest.mark.parametrize("helper", model_helpers)
-def test_management_edit_model(logged_in_admin_user: Page, helper: ModelHelper) -> None:
-    page = logged_in_admin_user
+def test_management_edit_model(logged_in_user: Page, helper: ModelHelper) -> None:
+    page = logged_in_user
     page.goto(f"/management/{helper.url}")
     # click edit
     edit_button_title = f"Edit {helper.verbose_name}"
diff --git a/rdmo/management/tests/test_frontend_import.py b/rdmo/management/tests/test_frontend_import.py
new file mode 100644
index 0000000000..3c46a84ca2
--- /dev/null
+++ b/rdmo/management/tests/test_frontend_import.py
@@ -0,0 +1,67 @@
+# ruff: noqa: F811
+import os
+
+import pytest
+
+from playwright.sync_api import Page, expect
+
+from rdmo.questions.models import Catalog, Question, Section
+from rdmo.questions.models import Page as PageModel
+from rdmo.questions.models.questionset import QuestionSet
+
+from .fixtures_frontend import (
+    base_url_page,  # noqa: F401
+    e2e_tests_django_db_setup,  # noqa: F401
+    logged_in_user,  # noqa: F401
+)
+from .helpers_models import delete_all_objects
+
+pytestmark = pytest.mark.e2e
+
+# needed for playwright to run
+os.environ.setdefault("DJANGO_ALLOW_ASYNC_UNSAFE", "true")
+
+test_users = [('editor', 'editor')]
+
+@pytest.mark.parametrize("username, password", test_users)  # consumed by fixture
+def test_import_in_management(logged_in_user: Page) -> None:
+    """Test that each content type is available through the navigation."""
+    delete_all_objects([Catalog, Section, PageModel, QuestionSet, Question])
+
+    page = logged_in_user
+    expect(page.get_by_role("heading", name="Management")).to_be_visible()
+    page.screenshot(path="screenshots/management-import-before-initial.png", full_page=True)
+    # choose the file to be imported
+    page.locator("input[name=\"uploaded_file\"]").set_input_files("./testing/xml/elements/catalogs.xml")
+    # click the import form submit button, this will take some time
+    page.locator('#sidebar div.elements-sidebar form.upload-form.sidebar-form div.sidebar-form-button button.btn.btn-primary').click()  # noqa: E501
+    # wait for import to be finished with timeout 30s
+    expect(page.get_by_text("Import from: catalogs.xml")).to_be_visible(timeout=30_000)
+    ## TODO test if ImportInfo numbers are correct
+    # test the components of the import-before-import staging page
+    page.locator(".element-link").first.click()
+    page.get_by_role("link", name="Unselect all").click()
+    page.get_by_role("link", name="Select all", exact=True).click()
+    page.get_by_role("link", name="Show all").click()
+    rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")
+    # there are 2 rows per object displayed
+    expect(rows_displayed_in_ui).to_have_count(148*2)
+    page.get_by_role("link", name="Hide all").click()
+    page.screenshot(path="screenshots/management-import-pre-hide-all.png", full_page=True)
+    ## TODO test show changed elements
+    ## TODO test select changed elements
+    ## TODO test for warnings, errors
+
+    # click the import button to start saving the instances to the db
+    page.get_by_role("button", name="Import 148 elements").click()
+    page.screenshot(path="screenshots/management-import-post-import.png", full_page=True)
+    page.get_by_text("Created:").click()
+    # go back to management page
+    page.get_by_role("button", name="Back").click()
+    expect(page.get_by_role("heading", name="Management")).to_be_visible()
+    # assert all Model objects in db
+    assert Catalog.objects.count() == 2
+    assert Section.objects.count() == 6
+    assert PageModel.objects.count() == 48
+    assert QuestionSet.objects.count() == 3
+    assert Question.objects.count() == 89

From 9d4a2a1f6afdb7f96a22bd345b77928780b6360b Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 22 Feb 2024 18:29:53 +0100
Subject: [PATCH 090/205] tests: refactor import tests

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/__init__.py             | 54 -----------------
 .../tests/helpers_import_elements.py          | 33 +++++++++++
 rdmo/management/tests/helpers_models.py       | 59 +++++++++++++++++++
 rdmo/management/tests/helpers_xml.py          | 18 ++++++
 .../tests/test_import_conditions.py           |  4 +-
 rdmo/management/tests/test_import_domain.py   | 18 +++---
 rdmo/management/tests/test_import_options.py  |  6 +-
 .../management/tests/test_import_questions.py |  6 +-
 rdmo/management/tests/test_import_tasks.py    |  4 +-
 rdmo/management/tests/test_import_views.py    |  4 +-
 rdmo/management/tests/test_viewset_import.py  |  2 +-
 .../tests/test_viewset_import_multisite.py    |  2 +-
 rdmo/management/tests/test_viewset_upload.py  |  3 +-
 13 files changed, 136 insertions(+), 77 deletions(-)
 create mode 100644 rdmo/management/tests/helpers_import_elements.py
 create mode 100644 rdmo/management/tests/helpers_models.py
 create mode 100644 rdmo/management/tests/helpers_xml.py

diff --git a/rdmo/management/tests/__init__.py b/rdmo/management/tests/__init__.py
index 89aa249e79..e69de29bb2 100644
--- a/rdmo/management/tests/__init__.py
+++ b/rdmo/management/tests/__init__.py
@@ -1,54 +0,0 @@
-
-from collections import OrderedDict
-from typing import Dict, List, Tuple
-
-from rdmo.core.xml import XmlParser
-
-xml_error_files = [
-    ('file-does-not-exist.xml', 'may not be blank'),
-    ('xml/error.xml', 'syntax error'),
-    ('xml/error-version.xml', 'RDMO XML Version: 99'),
-    ('xml/elements/legacy/catalog-error-key.xml', 'Missing legacy elements'),
-]
-
-def delete_all_objects(db_models: List):
-    for db_model in db_models:
-        db_model.objects.all().delete()
-
-def read_xml_and_parse_to_elements(xml_file):
-
-    xml_parser = XmlParser(file_name=xml_file)
-    if xml_parser.errors:
-        _msg = "\n".join(xml_parser.errors)
-        raise ValueError(f"This test function should NOT raise any Exceptions. {_msg!s}")
-    return xml_parser.parsed_elements, xml_parser.root
-
-def _test_helper_change_fields_elements(elements, update_dict=None, n=3) -> Tuple[Dict, List]:
-    """ xml test preparation function """
-
-    update_dict = update_dict if update_dict is not None else {}
-    _default_update_dict = {'comment':  "this is a test comment {}"}
-    update_dict.update(**_default_update_dict)
-
-    if len(elements) < n:
-        raise ValueError("Length of elements should not be smaller than n.")
-    _new_elements = OrderedDict()
-    _changed_elements = OrderedDict()
-    for _n,(_uri, _element) in enumerate(elements.items()):
-        if _n <= n-1:
-            updated_and_changed = {}
-            changed_element = _element
-            for k,val in update_dict.items():
-                if isinstance(val, str):
-                    val = val.format(_n)
-                updated_and_changed[k]= {'current': _element[k], 'uploaded': val}
-                _element[k] = val
-            if updated_and_changed:
-                changed_element['updated_and_changed'] = updated_and_changed
-            _changed_elements[_uri] = changed_element
-        _new_elements[_uri] = _element
-    return _new_elements, list(_changed_elements.values())
-
-def _test_helper_filter_updated_and_changed(elements: List[Dict]) -> List[Dict]:
-    filtered_elements = filter(lambda x: x.get('updated_and_changed', False), elements)
-    return list(filtered_elements)
diff --git a/rdmo/management/tests/helpers_import_elements.py b/rdmo/management/tests/helpers_import_elements.py
new file mode 100644
index 0000000000..fd380d5f3d
--- /dev/null
+++ b/rdmo/management/tests/helpers_import_elements.py
@@ -0,0 +1,33 @@
+from collections import OrderedDict
+from typing import Dict, List, Tuple
+
+
+def _test_helper_change_fields_elements(elements, update_dict=None, n=3) -> Tuple[Dict, List]:
+    """ xml test preparation function """
+
+    update_dict = update_dict if update_dict is not None else {}
+    _default_update_dict = {'comment':  "this is a test comment {}"}
+    update_dict.update(**_default_update_dict)
+
+    if len(elements) < n:
+        raise ValueError("Length of elements should not be smaller than n.")
+    _new_elements = OrderedDict()
+    _changed_elements = OrderedDict()
+    for _n,(_uri, _element) in enumerate(elements.items()):
+        if _n <= n-1:
+            updated_and_changed = {}
+            changed_element = _element
+            for k,val in update_dict.items():
+                if isinstance(val, str):
+                    val = val.format(_n)
+                updated_and_changed[k]= {'current': _element[k], 'uploaded': val}
+                _element[k] = val
+            if updated_and_changed:
+                changed_element['updated_and_changed'] = updated_and_changed
+            _changed_elements[_uri] = changed_element
+        _new_elements[_uri] = _element
+    return _new_elements, list(_changed_elements.values())
+
+def _test_helper_filter_updated_and_changed(elements: List[Dict]) -> List[Dict]:
+    filtered_elements = filter(lambda x: x.get('updated_and_changed', False), elements)
+    return list(filtered_elements)
diff --git a/rdmo/management/tests/helpers_models.py b/rdmo/management/tests/helpers_models.py
new file mode 100644
index 0000000000..bc0042662f
--- /dev/null
+++ b/rdmo/management/tests/helpers_models.py
@@ -0,0 +1,59 @@
+from dataclasses import dataclass
+from typing import List
+
+from rdmo.conditions.models import Condition
+from rdmo.core.models import Model
+from rdmo.domain.models import Attribute
+from rdmo.options.models import Option, OptionSet
+from rdmo.questions.models import Catalog, Question, Section
+from rdmo.questions.models import Page as PageModel
+from rdmo.questions.models.questionset import QuestionSet
+from rdmo.tasks.models import Task
+from rdmo.views.models import View
+
+
+@dataclass
+class ModelHelper:
+    """Helper class to bundle information about models for test cases."""
+
+    model: Model
+    form_field: str = "URI Path"
+    db_field: str = "uri_path"
+    has_nested: bool = False
+
+    @property
+    def url(self) -> str:
+        return f"{self.model._meta.model_name}s"
+
+    @property
+    def verbose_name(self) -> str:
+        """Return the verbose_name for the model."""
+        return self.model._meta.verbose_name
+
+    @property
+    def verbose_name_plural(self) -> str:
+        """Return the verbose_name_plural for the model."""
+        return self.model._meta.verbose_name_plural
+
+
+
+model_helpers = (
+    ModelHelper(Catalog, has_nested=True),
+    ModelHelper(Section, has_nested=True),
+    ModelHelper(PageModel, has_nested=True),
+    ModelHelper(QuestionSet, has_nested=True),
+    ModelHelper(Question),
+    ModelHelper(
+        Attribute, has_nested=True, form_field="Key", db_field="key"
+    ),
+    ModelHelper(OptionSet, has_nested=True),
+    ModelHelper(Option),
+    ModelHelper(Condition),
+    ModelHelper(Task),
+    ModelHelper(View),
+)
+
+
+def delete_all_objects(db_models: List):
+    for db_model in db_models:
+        db_model.objects.all().delete()
diff --git a/rdmo/management/tests/helpers_xml.py b/rdmo/management/tests/helpers_xml.py
new file mode 100644
index 0000000000..8768ba456a
--- /dev/null
+++ b/rdmo/management/tests/helpers_xml.py
@@ -0,0 +1,18 @@
+
+from rdmo.core.xml import XmlParser
+
+xml_error_files = [
+    ('file-does-not-exist.xml', 'may not be blank'),
+    ('xml/error.xml', 'syntax error'),
+    ('xml/error-version.xml', 'RDMO XML Version: 99'),
+    ('xml/elements/legacy/catalog-error-key.xml', 'Missing legacy elements'),
+]
+
+
+def read_xml_and_parse_to_elements(xml_file):
+
+    xml_parser = XmlParser(file_name=xml_file)
+    if xml_parser.errors:
+        _msg = "\n".join(xml_parser.errors)
+        raise ValueError(f"This test function should NOT raise any Exceptions. {_msg!s}")
+    return xml_parser.parsed_elements, xml_parser.root
diff --git a/rdmo/management/tests/test_import_conditions.py b/rdmo/management/tests/test_import_conditions.py
index 6884f230b2..374297cee0 100644
--- a/rdmo/management/tests/test_import_conditions.py
+++ b/rdmo/management/tests/test_import_conditions.py
@@ -5,11 +5,11 @@
 from rdmo.conditions.models import Condition
 from rdmo.management.imports import import_elements
 
-from . import (
+from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
-    read_xml_and_parse_to_elements,
 )
+from .helpers_xml import read_xml_and_parse_to_elements
 
 imported_update_changes = [
     None,
diff --git a/rdmo/management/tests/test_import_domain.py b/rdmo/management/tests/test_import_domain.py
index 34c6de1a70..882f7c85c5 100644
--- a/rdmo/management/tests/test_import_domain.py
+++ b/rdmo/management/tests/test_import_domain.py
@@ -3,11 +3,10 @@
 from rdmo.domain.models import Attribute
 from rdmo.management.imports import import_elements
 
-from . import (
+from .helpers_import_elements import (
     _test_helper_change_fields_elements,
-    _test_helper_filter_updated_and_changed,
-    read_xml_and_parse_to_elements,
 )
+from .helpers_xml import read_xml_and_parse_to_elements
 
 
 def test_create_domain(db, settings):
@@ -35,13 +34,16 @@ def test_update_domain(db, settings):
 
 
 def test_update_attributes_with_changed_fields(db, settings):
+    _change_count = Attribute.objects.count() / 2
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
-
     elements, root = read_xml_and_parse_to_elements(xml_file)
-    _change_count = Attribute.objects.count() / 2
-    elements, changed_elements = _test_helper_change_fields_elements(elements, n=_change_count)
-    imported_elements = import_elements(elements)
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    # import initial elements from xml
+    _el = import_elements(elements, save=True)
+    # update the elements and call import again
+    updated_elements, changed_elements = _test_helper_change_fields_elements(elements, n=_change_count)
+    imported_elements = import_elements(updated_elements)
+    imported_and_changed = list(filter(lambda x: x.get('updated_and_changed'), imported_elements))
+
     assert len(root) == len(imported_elements) == 86
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 7e0cce0e2f..19bbd48b3b 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -5,12 +5,12 @@
 from rdmo.management.imports import import_elements
 from rdmo.options.models import Option, OptionSet
 
-from . import (
+from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
-    delete_all_objects,
-    read_xml_and_parse_to_elements,
 )
+from .helpers_models import delete_all_objects
+from .helpers_xml import read_xml_and_parse_to_elements
 
 imported_update_changes = [None]
 
diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py
index c8d86c5c5e..e20db83776 100644
--- a/rdmo/management/tests/test_import_questions.py
+++ b/rdmo/management/tests/test_import_questions.py
@@ -5,12 +5,12 @@
 from rdmo.management.imports import import_elements
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
-from . import (
+from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
-    delete_all_objects,
-    read_xml_and_parse_to_elements,
 )
+from .helpers_models import delete_all_objects
+from .helpers_xml import read_xml_and_parse_to_elements
 
 imported_update_changes = [None]
 
diff --git a/rdmo/management/tests/test_import_tasks.py b/rdmo/management/tests/test_import_tasks.py
index 8a031ed99c..4d026cbabe 100644
--- a/rdmo/management/tests/test_import_tasks.py
+++ b/rdmo/management/tests/test_import_tasks.py
@@ -5,11 +5,11 @@
 from rdmo.management.imports import import_elements
 from rdmo.tasks.models import Task
 
-from . import (
+from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
-    read_xml_and_parse_to_elements,
 )
+from .helpers_xml import read_xml_and_parse_to_elements
 
 imported_update_changes = [None]
 
diff --git a/rdmo/management/tests/test_import_views.py b/rdmo/management/tests/test_import_views.py
index bdf1161d01..c0ebd35e4c 100644
--- a/rdmo/management/tests/test_import_views.py
+++ b/rdmo/management/tests/test_import_views.py
@@ -5,11 +5,11 @@
 from rdmo.management.imports import import_elements
 from rdmo.views.models import View
 
-from . import (
+from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
-    read_xml_and_parse_to_elements,
 )
+from .helpers_xml import read_xml_and_parse_to_elements
 
 imported_update_changes = [None]
 
diff --git a/rdmo/management/tests/test_viewset_import.py b/rdmo/management/tests/test_viewset_import.py
index 3791d14200..7bf246b932 100644
--- a/rdmo/management/tests/test_viewset_import.py
+++ b/rdmo/management/tests/test_viewset_import.py
@@ -4,7 +4,7 @@
 
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
-from . import delete_all_objects
+from .helpers_models import delete_all_objects
 
 users = (
     ('editor', 'editor'),
diff --git a/rdmo/management/tests/test_viewset_import_multisite.py b/rdmo/management/tests/test_viewset_import_multisite.py
index 17234f0b99..afb48bd5d7 100644
--- a/rdmo/management/tests/test_viewset_import_multisite.py
+++ b/rdmo/management/tests/test_viewset_import_multisite.py
@@ -6,7 +6,7 @@
 from rdmo.core.tests.utils import get_obj_perms_status_code
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
-from . import delete_all_objects
+from .helpers_models import delete_all_objects
 
 status_map = {
     'list': {
diff --git a/rdmo/management/tests/test_viewset_upload.py b/rdmo/management/tests/test_viewset_upload.py
index 97c2481b89..72bd1fe9b9 100644
--- a/rdmo/management/tests/test_viewset_upload.py
+++ b/rdmo/management/tests/test_viewset_upload.py
@@ -7,7 +7,8 @@
 
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
-from . import delete_all_objects, xml_error_files
+from .helpers_models import delete_all_objects
+from .helpers_xml import xml_error_files
 
 users = (
     ('editor', 'editor'),

From 0aac03bda694c9bc067badf5f931ab80034e3f22 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 22 Feb 2024 18:31:39 +0100
Subject: [PATCH 091/205] chore: clean up import_element func

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/imports.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 1acaad2eeb..984769a4c4 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -32,11 +32,10 @@
 from rdmo.tasks.imports import import_helper_task
 from rdmo.views.imports import import_helper_view
 
-from .constants import RDMO_MODEL_PATH_MAPPER
-
 logger = logging.getLogger(__name__)
 
 
+# mapping is redundant, since each ImportHelper has a .model_path attribute
 ELEMENT_IMPORT_HELPERS = {
     "conditions.condition": import_helper_condition,
     "domain.attribute": import_helper_attribute,
@@ -95,9 +94,11 @@ def import_element(
     for _k,_val in IMPORT_ELEMENT_INIT_DICT.items():
         element[_k] = _val()
 
-    model = RDMO_MODEL_PATH_MAPPER[model_path]
     user = request.user if request is not None else None
     import_helper = ELEMENT_IMPORT_HELPERS[model_path]
+    if import_helper.model_path != model_path:
+        raise ValueError(f'Invalid import helper model path: {import_helper.model_path}. Expected {model_path}.')
+    model = import_helper.model
     validators = import_helper.validators
     common_fields = import_helper.common_fields
     lang_field_names = import_helper.lang_fields
@@ -130,13 +131,11 @@ def import_element(
     # set common field values from element on instance
     for common_field in common_fields:
         common_value = element.get(common_field) or ''
-        # handle URI Prefix ending with slash
-        if common_field == 'uri_prefix' and common_value.endswith('/'):
-            common_value = common_value.rstrip('/')
-            element[common_field] = common_value
-            if original_instance:
-                original_instance.uri_prefix = original_instance.uri_prefix.rstrip('/')
         setattr(instance, common_field, common_value)
+    strip_uri_prefix_endswith_slash(element)
+    ## strip uri_prefix slash for comparison diff
+    if original_instance:
+        original_instance.uri_prefix = original_instance.uri_prefix.rstrip('/')
     # set language fields
     for lang_field_name in lang_field_names:
         set_lang_field(instance, lang_field_name, element)
@@ -146,7 +145,6 @@ def import_element(
     # set extra fields
     for extra_field in extra_field_names:
         set_extra_field(instance, extra_field, element, questions_widget_types=questions_widget_types)
-
     # call the validators on the instance
     validate_instance(instance, element, *validators)
 
@@ -180,6 +178,14 @@ def import_element(
 
     return element
 
+def strip_uri_prefix_endswith_slash(element: dict) -> dict:
+    # handle URI Prefix ending with slash
+    if 'uri_prefix' not in element:
+        return element
+    if element['uri_prefix'].endswith('/'):
+        element['uri_prefix'] = element['uri_prefix'].rstrip('/')
+    return element
+
 
 def get_updated_changes(element, new_instance,
                         original_instance, serializer, request=None) -> Dict[str, str]:

From b42321ddba691f1fac4ffd4cdae1447fad069470 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 22 Feb 2024 18:32:36 +0100
Subject: [PATCH 092/205] chore: add custom common_fields to Attribute import
 helper

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/domain/imports.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index e43d719874..2ff10106db 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -13,6 +13,7 @@
 import_helper_attribute = ElementImportHelper(
     model=Attribute,
     model_path="domain.attribute",
+    common_fields=('uri_prefix', 'key', 'comment'),
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     foreign_fields=('parent',),
     extra_fields=('path',),

From 996e04d660687333461fe36c08bf4b25a0db5cc3 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 23 Feb 2024 08:58:08 +0100
Subject: [PATCH 093/205] fix: py3.8 compatibility, remove dataclass kw_only
 kwarg

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 2142f42cbf..f7ac89afcc 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -61,7 +61,7 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
         return f"{verbose_name} created with {uri}"
     return f"{verbose_name} {uri} updated"
 
-@dataclass(kw_only=True, frozen=True)
+@dataclass(frozen=True)
 class ElementImportHelper:
     model: models.Model | None = field(default=None)
     model_path: str | None = field(default=None)

From a4722e255c60b38622f3e3e5b6fc0bc7dad7e815 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 23 Feb 2024 09:56:12 +0100
Subject: [PATCH 094/205] tests: refactor frontend tests and add
 frontend_import test

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index f7ac89afcc..46755f3ce6 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -63,10 +63,10 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
 
 @dataclass(frozen=True)
 class ElementImportHelper:
-    model: models.Model | None = field(default=None)
-    model_path: str | None = field(default=None)
+    model: Optional[models.Model] = field(default=None)
+    model_path: Optional[str] = field(default=None)
     validators: Iterable[Callable] = field(default_factory=list)
-    serializer: Callable | None = field(default=None)
+    serializer: Optional[Callable] = field(default=None)
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
     lang_fields: Sequence[str] = field(default_factory=list)
     foreign_fields: Sequence[str] = field(default_factory=list)

From 0a5abbd8cd2773176704696e8c404a640f0e56cc Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 23 Feb 2024 12:57:12 +0100
Subject: [PATCH 095/205] tests: refactor frontend tests and add
 frontend_import test

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py       | 13 ++++--
 rdmo/management/imports.py |  5 +-
 rdmo/options/imports.py    | 20 +++++---
 rdmo/questions/imports.py  | 95 +++++++++++++++++++++++---------------
 4 files changed, 84 insertions(+), 49 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 46755f3ce6..ff9022bf62 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -5,7 +5,7 @@
 from os.path import join as pj
 from pathlib import Path
 from random import randint
-from typing import Callable, Dict, Iterable, Optional, Sequence, Tuple
+from typing import Callable, Iterable, Optional, Sequence, Tuple
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
@@ -61,6 +61,13 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
         return f"{verbose_name} created with {uri}"
     return f"{verbose_name} {uri} updated"
 
+@dataclass(frozen=True)
+class ThroughInstanceMapper:
+    field_name: str
+    source_name: str
+    target_name: str
+    through_name: str
+
 @dataclass(frozen=True)
 class ElementImportHelper:
     model: Optional[models.Model] = field(default=None)
@@ -72,8 +79,8 @@ class ElementImportHelper:
     foreign_fields: Sequence[str] = field(default_factory=list)
     extra_fields: Sequence[str] = field(default_factory=list)
     m2m_instance_fields: Sequence[str] = field(default_factory=list)
-    m2m_through_instance_fields: Sequence[Dict[str, str]] = field(default_factory=list)
-    reverse_m2m_through_instance_fields: Sequence[Dict[str, str]] = field(default_factory=list)
+    m2m_through_instance_fields: Sequence[ThroughInstanceMapper] = field(default_factory=list)
+    reverse_m2m_through_instance_fields: Sequence[ThroughInstanceMapper] = field(default_factory=list)
     add_current_site_editors: bool = field(default=True)
     add_current_site_sites: bool = field(default=False)
 
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 984769a4c4..e2839bc2bc 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -1,6 +1,7 @@
 import copy
 import logging
 from collections import defaultdict
+from dataclasses import asdict
 from typing import AbstractSet, Dict, List, Optional
 
 from django.contrib.sites.shortcuts import get_current_site
@@ -167,9 +168,9 @@ def import_element(
         for m2m_field in import_helper.m2m_instance_fields:
             set_m2m_instances(instance, element, m2m_field)
         for m2m_through_fields in import_helper.m2m_through_instance_fields:
-            set_m2m_through_instances(instance, element, **m2m_through_fields)
+            set_m2m_through_instances(instance, element, **asdict(m2m_through_fields))
         for reverse_m2m_fields in import_helper.reverse_m2m_through_instance_fields:
-            set_reverse_m2m_through_instance(instance, element, **reverse_m2m_fields)
+            set_reverse_m2m_through_instance(instance, element, **asdict(reverse_m2m_fields))
 
         if import_helper.add_current_site_editors:
             instance.editors.add(current_site)
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 2ade3d3811..1bc1c91029 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -1,6 +1,4 @@
-from rdmo.core.imports import (
-    ElementImportHelper,
-)
+from rdmo.core.imports import ElementImportHelper, ThroughInstanceMapper
 
 from .models import Option, OptionSet
 from .serializers.v1 import OptionSerializer, OptionSetSerializer
@@ -20,8 +18,12 @@
     extra_fields = ('order', 'provider_key', 'additional_input'),
     m2m_instance_fields = ('conditions', ),
     m2m_through_instance_fields = [
-        {'field_name': 'options', 'source_name': 'optionset',
-         'target_name': 'option', 'through_name': 'optionset_options'}
+        ThroughInstanceMapper(
+            field_name='options',
+            source_name='optionset',
+            target_name='option',
+            through_name='optionset_options'
+        )
     ]
 )
 
@@ -32,7 +34,11 @@
     serializer = OptionSetSerializer,
     extra_fields=('additional_input',),
     reverse_m2m_through_instance_fields=[
-        {'field_name': 'optionset', 'source_name': 'option',
-         'target_name': 'optionset', 'through_name': 'option_optionsets'}
+        ThroughInstanceMapper(
+            field_name='optionset',
+            source_name='option',
+            target_name='optionset',
+            through_name='option_optionsets'
+        )
     ]
 )
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index a2dc97509a..8022a4f023 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -1,6 +1,4 @@
-from rdmo.core.imports import (
-    ElementImportHelper,
-)
+from rdmo.core.imports import ElementImportHelper, ThroughInstanceMapper
 
 from .models import Catalog, Page, Question, QuestionSet, Section
 from .serializers.v1 import (
@@ -31,8 +29,10 @@
     serializer = CatalogSerializer,
     extra_fields = ('order', 'available'),
     m2m_through_instance_fields=[
-        {'field_name': 'sections', 'source_name': 'catalog',
-         'target_name': 'section', 'through_name': 'catalog_sections'}
+        ThroughInstanceMapper(
+            field_name='sections', source_name='catalog',
+            target_name='section', through_name='catalog_sections'
+        )
     ],
     add_current_site_sites = True,
 )
@@ -44,16 +44,19 @@
     lang_fields=('title',),
     serializer = SectionSerializer,
     m2m_through_instance_fields=[
-        {'field_name': 'pages', 'source_name': 'section',
-         'target_name': 'page', 'through_name': 'section_pages'}
+        ThroughInstanceMapper(
+            field_name='pages', source_name='section',
+            target_name='page', through_name='section_pages'
+        )
     ],
     reverse_m2m_through_instance_fields=[
-        {'field_name': 'catalog', 'source_name': 'section',
-         'target_name': 'catalog', 'through_name': 'section_catalogs'}
+        ThroughInstanceMapper(
+            field_name='catalog', source_name='section',
+            target_name='catalog', through_name='section_catalogs'
+        )
     ]
 )
 
-
 import_helper_page = ElementImportHelper(
     model = Page,
     model_path="questions.page",
@@ -64,56 +67,74 @@
     extra_fields = ('is_collection',),
     m2m_instance_fields = ('conditions', ),
     m2m_through_instance_fields=[
-        {'field_name': 'questionsets', 'source_name': 'page',
-         'target_name': 'questionset', 'through_name': 'page_questionsets'},
-        {'field_name': 'questions', 'source_name': 'page',
-         'target_name': 'question', 'through_name': 'page_questions'}
+        ThroughInstanceMapper(
+            field_name='questionsets', source_name='page',
+            target_name='questionset', through_name='page_questionsets'
+        ),
+        ThroughInstanceMapper(
+            field_name='questions', source_name='page',
+            target_name='question', through_name='page_questions'
+        )
     ],
     reverse_m2m_through_instance_fields=[
-        {'field_name': 'section', 'source_name': 'page',
-         'target_name': 'section', 'through_name': 'page_sections'}
+        ThroughInstanceMapper(
+            field_name='section', source_name='page',
+            target_name='section', through_name='page_sections'
+        )
     ]
 )
 
-
 import_helper_questionset = ElementImportHelper(
     model = QuestionSet,
-    model_path = "questions.questionset",
+    model_path="questions.questionset",
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
     serializer = QuestionSetSerializer,
     extra_fields = ('is_collection',),
-    m2m_instance_fields = ('conditions', ),
+    m2m_instance_fields=('conditions', ),
+
     m2m_through_instance_fields=[
-        {'field_name': 'questionsets', 'source_name': 'parent',
-         'target_name': 'questionset', 'through_name': 'questionset_questionsets'},
-        {'field_name': 'questions', 'source_name': 'questionset',
-         'target_name': 'question', 'through_name': 'questionset_questions'}
+        ThroughInstanceMapper(
+            field_name='questionsets', source_name='parent',
+            target_name='questionset', through_name='questionset_questionsets'
+        ),
+        ThroughInstanceMapper(
+            field_name='questions', source_name='questionset',
+            target_name='question', through_name='questionset_questions'
+        )
     ],
     reverse_m2m_through_instance_fields=[
-        {'field_name': 'page', 'source_name': 'questionset',
-         'target_name': 'page', 'through_name': 'questionset_pages'},
-        {'field_name': 'questionset', 'source_name': 'questionset',
-         'target_name': 'parent', 'through_name': 'questionset_parents'}
+        ThroughInstanceMapper(
+            field_name='page', source_name='questionset',
+            target_name='page', through_name='questionset_pages'
+        ),
+        ThroughInstanceMapper(
+            field_name='questionset', source_name='questionset',
+            target_name='parent', through_name='questionset_parents'
+        )
     ]
 )
 
-
 import_helper_question = ElementImportHelper(
-    model = Question,
-    model_path = "questions.question",
+    model=Question,
+    model_path="questions.question",
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
     lang_fields=('text', 'help', 'default_text', 'verbose_name'),
-    foreign_fields=('attribute','default_option'),
+    foreign_fields=('attribute', 'default_option'),
     serializer=QuestionSerializer,
-    extra_fields=('is_collection','is_optional', 'default_external_id', 'widget_type',
-                    'value_type', 'maximum', 'minimum', 'step', 'unit','width'),
+    extra_fields=('is_collection', 'is_optional', 'default_external_id',
+                  'widget_type', 'value_type', 'maximum', 'minimum', 'step',
+                  'unit', 'width'),
     m2m_instance_fields=('conditions', 'optionsets'),
     reverse_m2m_through_instance_fields=[
-        {'field_name': 'page', 'source_name': 'question',
-         'target_name': 'page', 'through_name': 'question_pages'},
-        {'field_name': 'questionset', 'source_name': 'question',
-         'target_name': 'questionset', 'through_name': 'question_questionsets'}
+        ThroughInstanceMapper(
+            field_name='page', source_name='question',
+            target_name='page', through_name='question_pages'
+        ),
+        ThroughInstanceMapper(
+            field_name='questionset', source_name='question',
+            target_name='questionset', through_name='question_questionsets'
+        )
     ]
 )

From 25a0b731b4f3e3a75a4809cef7de823185b8fd1a Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 26 Feb 2024 17:41:52 +0100
Subject: [PATCH 096/205] js: fix typo in FieldsDiffs.js

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/import/common/FieldsDiffs.js           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index 1e2ee9bd34..ec6155f259 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -5,7 +5,7 @@ import ReactDiffViewer from 'react-diff-viewer-continued'
 import { isUndefined } from 'lodash'
 
 const FieldsDiffs = ({ element, field }) => {
-  const newVal = element.updated_and_changed[field].uploaded ?? ''
+  const newVal = element.updated_and_changed[field].updated ?? ''
   const oldVal = element.updated_and_changed[field].current ?? ''
   return (!isUndefined(element) &&
           !isEmpty(element.updated_and_changed) &&

From e95ea375ccbed46c227b02f8c0d447f4a19eb049 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 26 Feb 2024 17:42:29 +0100
Subject: [PATCH 097/205] chore: remove key from default common import fields

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/constants.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/rdmo/core/constants.py b/rdmo/core/constants.py
index 0453c4ca65..11451a838a 100644
--- a/rdmo/core/constants.py
+++ b/rdmo/core/constants.py
@@ -81,7 +81,6 @@
 ELEMENT_COMMON_FIELDS = (
     'uri_prefix',
     'uri_path',
-    'key',
     'comment',
 )
 

From c89979ee2c2ca3102ec108266905b09a9f0c1918 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 26 Feb 2024 18:02:25 +0100
Subject: [PATCH 098/205] chore: fix ElementImportHelper definitions for
 Options

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/options/imports.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 1bc1c91029..5dcb6cd462 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -9,13 +9,12 @@
     OptionUniqueURIValidator,
 )
 
-import_helper_option = ElementImportHelper(
-    model = Option,
-    model_path="options.option",
-    validators=(OptionLockedValidator, OptionUniqueURIValidator),
-    lang_fields=('text',),
-    serializer = OptionSerializer,
-    extra_fields = ('order', 'provider_key', 'additional_input'),
+import_helper_optionset = ElementImportHelper(
+    model = OptionSet,
+    model_path = "options.optionset",
+    validators = (OptionSetLockedValidator, OptionSetUniqueURIValidator),
+    serializer = OptionSetSerializer,
+    extra_fields = ('order', 'provider_key'),
     m2m_instance_fields = ('conditions', ),
     m2m_through_instance_fields = [
         ThroughInstanceMapper(
@@ -23,22 +22,23 @@
             source_name='optionset',
             target_name='option',
             through_name='optionset_options'
-        )
+        ),
     ]
 )
 
-import_helper_optionset = ElementImportHelper(
-    model = OptionSet,
-    model_path="options.optionset",
-    validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
-    serializer = OptionSetSerializer,
-    extra_fields=('additional_input',),
-    reverse_m2m_through_instance_fields=[
+import_helper_option = ElementImportHelper(
+    model = Option,
+    model_path = "options.option",
+    validators = (OptionLockedValidator, OptionUniqueURIValidator),
+    lang_fields = ('text','help','view_text'),
+    serializer = OptionSerializer,
+    extra_fields = ('additional_input',),
+    reverse_m2m_through_instance_fields = [
         ThroughInstanceMapper(
             field_name='optionset',
             source_name='option',
             target_name='optionset',
             through_name='option_optionsets'
-        )
+        ),
     ]
 )

From 256c459093875817928c3991e901ed76f9ef19b8 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 26 Feb 2024 18:05:43 +0100
Subject: [PATCH 099/205] feat: add track changes for each field during import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py       | 221 ++++++++++++++++++++++++++++---------
 rdmo/management/imports.py |  47 ++++----
 2 files changed, 192 insertions(+), 76 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index ff9022bf62..16782f12c6 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -5,10 +5,11 @@
 from os.path import join as pj
 from pathlib import Path
 from random import randint
-from typing import Callable, Iterable, Optional, Sequence, Tuple
+from typing import Callable, Iterable, List, Optional, Sequence, Tuple, Union
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
+from django.utils.encoding import force_str
 from django.utils.translation import gettext_lazy as _
 
 from rest_framework.utils import model_meta
@@ -61,6 +62,35 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
         return f"{verbose_name} created with {uri}"
     return f"{verbose_name} {uri} updated"
 
+
+def track_changes_on_element(element: dict,
+                             element_field: str,
+                             new_value: Union[str,List],
+                             instance_field: Optional[str]=None,
+                             original=None,
+                             original_value: Optional[Union[str,List]]=None):
+    if original is None and original_value is None:
+        return
+
+    _get_field = element_field if instance_field is None else instance_field
+    if original_value is None:
+        original_value = force_str(getattr(original, _get_field, ''))
+
+    if isinstance(new_value, list) and isinstance(original_value, list):
+        # cast a list of elements with uris to a string with newlines
+        new_value = "\n".join(i['uri'] for i in  new_value)
+        original_value = "\n".join(i['uri'] for i in  original_value)
+
+    if new_value == original_value:
+        return
+    #  TODO maybe rename updated to new
+    changes =  {'current': original_value, 'updated': new_value}
+    if element['updated_and_changed'].get(element_field) is None:
+        element['updated_and_changed'][element_field] = changes
+    else:
+        element['updated_and_changed'][element_field].update(changes)
+
+
 @dataclass(frozen=True)
 class ThroughInstanceMapper:
     field_name: str
@@ -84,7 +114,6 @@ class ElementImportHelper:
     add_current_site_editors: bool = field(default=True)
     add_current_site_sites: bool = field(default=False)
 
-
 def get_lang_field_values(field_name: str,
                         element: Optional[dict] = None,
                         instance: Optional[models.Model] = None,
@@ -108,16 +137,30 @@ def get_lang_field_values(field_name: str,
         ret.append(row)
     return ret
 
-def set_lang_field(instance, field_name, element):
+def set_lang_field(instance, field_name, element, original=None):
     languages_field_values = get_lang_field_values(field_name, element=element)
     for lang_fields_value in languages_field_values:
         field_lang_name = lang_fields_value['instance_field']
         field_value = lang_fields_value['value']
+        element_field = lang_fields_value['element_key']
 
+        track_changes_on_element(element, element_field,
+                                 field_value,
+                                 instance_field=field_lang_name,
+                                 original=original)
         setattr(instance, field_lang_name, field_value)
 
-
-def set_foreign_field(instance, field_name, element, uploaded_uris=None) -> None:
+def track_changes_on_uri_of_foreign_field(element, field_name, foreign_uri, original=None):
+    if original is None:
+        return
+    # get foreign uri of original
+    original_foreign_instance = getattr(original, field_name, '')
+    original_foreign_uri = ''
+    if original_foreign_instance:
+        original_foreign_uri = getattr(original_foreign_instance, 'uri', '')
+    track_changes_on_element(element, field_name, foreign_uri, original_value=original_foreign_uri)
+
+def set_foreign_field(instance, field_name, element, uploaded_uris=None, original=None) -> None:
     if field_name not in element:
         return
 
@@ -138,7 +181,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None) -> None
         return
 
     foreign_uri = foreign_element['uri']
-
+    # breakpoint()
     model_info = model_meta.get_field_info(instance)
     foreign_model = model_info.forward_relations[field_name].related_model
     foreign_instance = None
@@ -156,6 +199,11 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None) -> None
     try:
         if foreign_instance is not None:
             setattr(instance, field_name, foreign_instance)
+        _foreign_uri = foreign_uri if foreign_instance is not None else ""
+        track_changes_on_uri_of_foreign_field(element,
+                                    field_name,
+                                    _foreign_uri,
+                                    original=original)
     except ValueError:
         message = '{foreign_model} {foreign_uri} can not be assigned on {instance_model}.{field_name} {instance_uri} .'.format( # noqa: E501
             foreign_model=foreign_model._meta.object_name,
@@ -168,8 +216,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None) -> None
         element['errors'][foreign_uri].append(message)
 
 
-
-def set_extra_field(instance, field_name, element, questions_widget_types=None) -> None:
+def set_extra_field(instance, field_name, element, questions_widget_types=None, original=None) -> None:
 
     element_value = element.get(field_name)
     default_value = ELEMENT_IMPORT_EXTRA_FIELDS_DEFAULTS.get(field_name)
@@ -193,9 +240,24 @@ def set_extra_field(instance, field_name, element, questions_widget_types=None)
             element['errors'].append(message)
 
     setattr(instance, field_name, extra_value)
+    # track changes
+    track_changes_on_element(element, field_name,
+                                 extra_value, original=original)
+
+def track_changes_m2m_instances(element, field_name,
+                                foreign_instances, original=None):
+    if original is None:
+        return
+    original_m2m_instance = getattr(original, field_name)
+    if original_m2m_instance is None:
+        return
+    original_m2m_uris = original_m2m_instance.values_list('uri', flat=True)
+    foreign_uris = [i.uri for i in foreign_instances]
+    track_changes_on_element(element, field_name, foreign_uris,
+                             original_value=original_m2m_uris)
 
 
-def set_m2m_instances(instance, element, field_name):
+def set_m2m_instances(instance, element, field_name, original=None, save=None):
     if field_name not in element:
         return
 
@@ -225,24 +287,43 @@ def set_m2m_instances(instance, element, field_name):
             )
             logger.info(message)
             element['warnings'][foreign_uri].append(message)
-
-    getattr(instance, field_name).set(foreign_instances)
+    if save:
+        getattr(instance, field_name).set(foreign_instances)
+    track_changes_m2m_instances(element, field_name,
+                                foreign_instances, original=original)
 
 
 def set_m2m_through_instances(instance, element, field_name=None, source_name=None,
-                              target_name=None, through_name=None) -> None:
+                              target_name=None, through_name=None,
+                              original=None, save=None) -> None:
     if field_name not in element:
         return
     if not all([source_name, target_name, through_name]):
         return
 
     target_elements = element.get(field_name) or []
+    if isinstance(target_elements, str):
+        target_elements = [target_elements]
 
     model_info = model_meta.get_field_info(instance)
     through_model = model_info.reverse_relations[through_name].related_model
     target_model = model_info.forward_relations[field_name].related_model
     through_instances = list(getattr(instance, through_name).all())
 
+    _track_changes = {}
+    _track_changes['new_data'] = []
+    _track_changes['current_data'] = []
+    if original is not None:
+        try:
+            for _order, _through_instance in enumerate(getattr(original, field_name).all()):
+                _track_changes['current_data'].append({
+                    'uri': _through_instance.uri,
+                    'order': _order,
+                    'model': target_name
+                })
+        except AttributeError:
+            pass  # legacy elements miss the field_name
+
     for target_element in target_elements:
         target_uri = target_element.get('uri')
         order = target_element.get('order')
@@ -256,19 +337,24 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
                                                through_instances))
 
                 # update order if the item if it changed
-                if through_instance.order != order:
+                if through_instance.order != order and save:
                     through_instance.order = order
                     through_instance.save()
-
-                # remove the through_instance from the through_instances list so that it won't get removed
-                through_instances.remove(through_instance)
+                if save:
+                    # remove the through_instance from the through_instances list so that it won't get removed
+                    through_instances.remove(through_instance)
+                if original is not None:
+                    _track_changes['new_data'].append(target_element)
             except StopIteration:
                 # create a new item
-                through_model(**{
-                    source_name: instance,
-                    target_name: target_instance,
-                    'order': order
-                }).save()
+                if save:
+                    through_model(**{
+                        source_name: instance,
+                        target_name: target_instance,
+                        'order': order
+                    }).save()
+                if original is not None:
+                    _track_changes['new_data'].append(target_element)
 
         except target_model.DoesNotExist:
             message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} does not exist.'.format(
@@ -279,58 +365,89 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
             )
             logger.info(message)
             element['warnings'][target_uri].append(message)
-
-    # remove the remainders of the items list
-    for through_instance in through_instances:
-        if getattr(through_instance, target_name).uri_prefix == instance.uri_prefix:
-            through_instance.delete()
+    if save:
+        # remove the remainders of the items list
+        for through_instance in through_instances:
+            if getattr(through_instance, target_name).uri_prefix == instance.uri_prefix:
+                through_instance.delete()
+    # sort the tracked changes by order in-place
+    _track_changes['new'] = sorted(_track_changes['new_data'], key=lambda k: k['order'])
+    _track_changes['current'] = sorted(_track_changes['current_data'], key=lambda k: k['order'])
+    _store = {'new_data' : _track_changes['new_data'],  'current_data' : _track_changes['current_data']}
+    element['updated_and_changed'][field_name] = _store
+    track_changes_on_element(element, field_name, _track_changes['new'],
+                             original_value=_track_changes['current'])
 
 
 def set_reverse_m2m_through_instance(instance, element, field_name=None, source_name=None,
-                                     target_name=None, through_name=None) -> None:
+                                     target_name=None, through_name=None,
+                                     original=None, save=None) -> None:
     if field_name not in element:
         return
     if not all([source_name, target_name, through_name]):
         return
 
-    target_element = element.get(field_name)
+    target_elements = element.get(field_name) or []
+    if isinstance(target_elements, str):
+        target_elements = [target_elements]
 
     model_info = model_meta.get_field_info(instance)
     through_model = model_info.reverse_relations[through_name].related_model
     through_info = model_meta.get_field_info(through_model)
     target_model = through_info.forward_relations[target_name].related_model
+    _track_changes = {}
+    _track_changes['new_data'] = []
+    _track_changes['current_data'] = []
+    if original is not None:
+        try:
+            for _order, _through_instance in enumerate(getattr(original, field_name).all()):
+                _track_changes['current_data'].append({
+                    'uri': _through_instance.uri,
+                    'order': _order,
+                    'model': target_name
+                })
+        except AttributeError:
+            pass  # legacy elements miss the field_name
 
-    target_uri = target_element.get('uri')
-    order = target_element.get('order')
+    for target_element in target_elements:
+        target_uri = target_element.get('uri')
+        order = target_element.get('order')
+        try:
+            target_instance = target_model.objects.get(uri=target_uri)
+            if target_instance.is_locked:
+                message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} is locked.'.format(
+                    target_model=target_model._meta.object_name,
+                    target_uri=target_uri,
+                    instance_model=instance._meta.object_name,
+                    instance_uri=element.get('uri')
+                )
+                logger.info(message)
+                element['errors'].append(message)
+                continue
+            if save:
+                through_instance, created = through_model.objects.get_or_create(**{
+                    source_name: instance,
+                    target_name: target_instance
+                })
+                through_instance.order = order
+                through_instance.save()
+            if original is not None:
+                _track_changes['new_data'].append(target_element)
 
-    try:
-        target_instance = target_model.objects.get(uri=target_uri)
-        if target_instance.is_locked:
-            message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} is locked.'.format(
+        except target_model.DoesNotExist:
+            message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} does not exist.'.format(
                 target_model=target_model._meta.object_name,
                 target_uri=target_uri,
                 instance_model=instance._meta.object_name,
                 instance_uri=element.get('uri')
             )
             logger.info(message)
-            element['errors'].append(message)
-        else:
-            through_instance, created = through_model.objects.get_or_create(**{
-                source_name: instance,
-                target_name: target_instance
-            })
-            through_instance.order = order
-            through_instance.save()
-
-    except target_model.DoesNotExist:
-        message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} does not exist.'.format(
-            target_model=target_model._meta.object_name,
-            target_uri=target_uri,
-            instance_model=instance._meta.object_name,
-            instance_uri=element.get('uri')
-        )
-        logger.info(message)
-        element['warnings'][target_uri].append(message)
+            element['warnings'][target_uri].append(message)
+    # sort the tracked changes by order in-place
+    _track_changes['new'] = sorted(_track_changes['new_data'], key=lambda k: k['order'])
+    _track_changes['current'] = sorted(_track_changes['current_data'], key=lambda k: k['order'])
+    track_changes_on_element(element, field_name, _track_changes['new'],
+                             original_value=_track_changes['current'])
 
 
 def validate_instance(instance, element, *validators):
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index e2839bc2bc..e394cdc124 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -4,6 +4,7 @@
 from dataclasses import asdict
 from typing import AbstractSet, Dict, List, Optional
 
+from django.conf import settings
 from django.contrib.sites.shortcuts import get_current_site
 from django.http import HttpRequest
 
@@ -18,6 +19,7 @@
     set_m2m_instances,
     set_m2m_through_instances,
     set_reverse_m2m_through_instance,
+    track_changes_on_element,
     validate_instance,
 )
 from rdmo.domain.imports import import_helper_attribute
@@ -113,7 +115,7 @@ def import_element(
     # keep a copy of the original
     # when the element is updated
     # needs to be created here, else the changes will be overwritten
-    original_instance = copy.deepcopy(instance) if not _created else None
+    original = copy.deepcopy(instance) if not _created else None
 
     # prepare a log message
     _msg = make_import_info_msg(model._meta.verbose_name, _created, uri=uri)
@@ -125,21 +127,23 @@ def import_element(
         element["errors"].append(_perms_error_msg)
         return element
 
-    # prepare original element when updated (maybe rename into lookup)
     _updated = not _created
+    element['created'] = _created
+    element['updated'] = _updated
+    # dict element['updated_and_changed'] is filled by tracking changes
 
+    element = strip_uri_prefix_endswith_slash(element)
     # start to set values on the instance
     # set common field values from element on instance
     for common_field in common_fields:
         common_value = element.get(common_field) or ''
         setattr(instance, common_field, common_value)
-    strip_uri_prefix_endswith_slash(element)
-    ## strip uri_prefix slash for comparison diff
-    if original_instance:
-        original_instance.uri_prefix = original_instance.uri_prefix.rstrip('/')
+        if _updated and original:
+            # track changes for common fields
+            track_changes_on_element(element, common_field, common_value, original=original)
     # set language fields
     for lang_field_name in lang_field_names:
-        set_lang_field(instance, lang_field_name, element)
+        set_lang_field(instance, lang_field_name, element, original=original)
     # set foreign fields
     for foreign_field in foreign_field_names:
         set_foreign_field(instance, foreign_field, element, uploaded_uris=uploaded_uris)
@@ -151,27 +155,22 @@ def import_element(
 
     if element.get('errors'):
         return element
-
-    if _updated and not _created:
-        element['updated'] = _updated
-        # and instance is not original_instance
-        # keep only strings, make json serializable
-        serializer = import_helper.serializer
-        changes = get_updated_changes(element, instance, original_instance, serializer, request=request)
-        element['updated_and_changed'] = changes
-
     if save:
         logger.info(_msg)
-        element['created'] = _created
-        element['updated'] = _updated
         instance.save()
+    # breakpoint()
+    if save or _updated:
         for m2m_field in import_helper.m2m_instance_fields:
-            set_m2m_instances(instance, element, m2m_field)
+            set_m2m_instances(instance, element, m2m_field, original=original, save=save)
         for m2m_through_fields in import_helper.m2m_through_instance_fields:
-            set_m2m_through_instances(instance, element, **asdict(m2m_through_fields))
+            set_m2m_through_instances(instance, element, **asdict(m2m_through_fields),
+                                      original=original, save=save)
         for reverse_m2m_fields in import_helper.reverse_m2m_through_instance_fields:
-            set_reverse_m2m_through_instance(instance, element, **asdict(reverse_m2m_fields))
+            set_reverse_m2m_through_instance(instance, element, **asdict(reverse_m2m_fields),
+                                             original=original, save=save)
 
+    if save and settings.MULTISITE:
+        # could be optimized with a bulk_create of through model later
         if import_helper.add_current_site_editors:
             instance.editors.add(current_site)
         if import_helper.add_current_site_sites:
@@ -187,10 +186,10 @@ def strip_uri_prefix_endswith_slash(element: dict) -> dict:
         element['uri_prefix'] = element['uri_prefix'].rstrip('/')
     return element
 
-
+#  TODO remove get_updated_changes
 def get_updated_changes(element, new_instance,
-                        original_instance, serializer, request=None) -> Dict[str, str]:
-    original_serializer = serializer(original_instance, context={'request': request})
+                        original, serializer, request=None) -> Dict[str, str]:
+    original_serializer = serializer(original, context={'request': request})
     original_data = original_serializer.data
     original_element = {k: val for k,val in original_data.items() if k in element}
     uploaded_serializer = serializer(new_instance, context={'request': request})

From 2c3000509c7359cb4ef1cc58f2ddd52982b92b9c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 26 Feb 2024 18:06:37 +0100
Subject: [PATCH 100/205] tests: add tests file with m2m changes for optionsets
 import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../updated-and-changed/optionsets-1.xml      | 140 ++++++++++++++++++
 1 file changed, 140 insertions(+)
 create mode 100644 testing/xml/elements/updated-and-changed/optionsets-1.xml

diff --git a/testing/xml/elements/updated-and-changed/optionsets-1.xml b/testing/xml/elements/updated-and-changed/optionsets-1.xml
new file mode 100644
index 0000000000..e2349f919a
--- /dev/null
+++ b/testing/xml/elements/updated-and-changed/optionsets-1.xml
@@ -0,0 +1,140 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rdmo xmlns:dc="http://purl.org/dc/elements/1.1/" created="2023-10-04T11:01:10.645345+02:00" version="2.0.0">
+	<optionset dc:uri="http://example.com/terms/options/condition">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>condition</uri_path>
+		<dc:comment/>
+		<provider_key/>
+		<options>
+			<option dc:uri="http://example.com/terms/options/condition/other" order="1"/>
+		</options>
+		<conditions>
+			<condition dc:uri="http://example.com/terms/conditions/optionset_bool_is_false"/>
+		</conditions>
+	</optionset>
+	<option dc:uri="http://example.com/terms/options/condition/other">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>condition/other</uri_path>
+		<dc:comment/>
+		<text lang="en">Other</text>
+		<help lang="en"/>
+		<text lang="de">Sonstige</text>
+		<help lang="de"/>
+		<additional_input>text</additional_input>
+	</option>
+	<optionset dc:uri="http://example.com/terms/options/one_two_three">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>one_two_three</uri_path>
+		<dc:comment/>
+		<provider_key/>
+		<options>
+			<option dc:uri="http://example.com/terms/options/one_two_three/three" order="1"/>
+			<option dc:uri="http://example.com/terms/options/one_two_three/two" order="2"/>
+			<option dc:uri="http://example.com/terms/options/one_two_three/one" order="3"/>
+            <option dc:uri="http://example.com/terms/options/one_two_three/four" order="4"/>
+		</options>
+		<conditions/>
+	</optionset>
+	<option dc:uri="http://example.com/terms/options/one_two_three/one">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>one_two_three/one</uri_path>
+		<dc:comment/>
+		<text lang="en">One</text>
+		<help lang="en"/>
+		<text lang="de">Eins</text>
+		<help lang="de"/>
+		<additional_input/>
+	</option>
+	<option dc:uri="http://example.com/terms/options/one_two_three/two">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>one_two_three/two</uri_path>
+		<dc:comment/>
+		<text lang="en">Two</text>
+		<help lang="en"/>
+		<text lang="de">Zwei</text>
+		<help lang="de"/>
+		<additional_input/>
+	</option>
+	<option dc:uri="http://example.com/terms/options/one_two_three/three">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>one_two_three/three</uri_path>
+		<dc:comment/>
+		<text lang="en">Three</text>
+		<help lang="en"/>
+		<text lang="de">Drei</text>
+		<help lang="de"/>
+		<additional_input/>
+	</option>
+	<optionset dc:uri="http://example.com/terms/options/one_two_three_other">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>one_two_three_other</uri_path>
+		<dc:comment/>
+		<provider_key/>
+		<options>
+			<option dc:uri="http://example.com/terms/options/one_two_three_other/textarea" order="1"/>
+			<option dc:uri="http://example.com/terms/options/one_two_three_other/text" order="2"/>
+			<option dc:uri="http://example.com/terms/options/one_two_three_other/three" order="3"/>
+			<option dc:uri="http://example.com/terms/options/one_two_three_other/two" order="4"/>
+			<option dc:uri="http://example.com/terms/options/one_two_three_other/one" order="5"/>
+		</options>
+		<conditions/>
+	</optionset>
+	<option dc:uri="http://example.com/terms/options/one_two_three_other/one">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>one_two_three_other/one</uri_path>
+		<dc:comment/>
+		<text lang="en">One - 3</text>
+		<help lang="en"/>
+		<text lang="de">Eins - 3</text>
+		<help lang="de"/>
+		<additional_input/>
+	</option>
+	<option dc:uri="http://example.com/terms/options/one_two_three_other/two">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>one_two_three_other/two</uri_path>
+		<dc:comment/>
+		<text lang="en">Two</text>
+		<help lang="en"/>
+		<text lang="de">Zwei</text>
+		<help lang="de"/>
+		<additional_input/>
+	</option>
+	<option dc:uri="http://example.com/terms/options/one_two_three_other/three">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>one_two_three_other/three</uri_path>
+		<dc:comment/>
+		<text lang="en">Three - 1</text>
+		<help lang="en"/>
+		<text lang="de">Drei - 1</text>
+		<help lang="de"/>
+		<additional_input/>
+	</option>
+	<option dc:uri="http://example.com/terms/options/one_two_three_other/text">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>one_two_three_other/text</uri_path>
+		<dc:comment/>
+		<text lang="en">Text</text>
+		<help lang="en"/>
+		<text lang="de">Text</text>
+		<help lang="de"/>
+		<additional_input>text</additional_input>
+	</option>
+	<option dc:uri="http://example.com/terms/options/one_two_three_other/textarea">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>one_two_three_other/textarea</uri_path>
+		<dc:comment/>
+		<text lang="en">Text</text>
+		<help lang="en"/>
+		<text lang="de">Text</text>
+		<help lang="de"/>
+		<additional_input>textarea</additional_input>
+	</option>
+	<optionset dc:uri="http://example.com/terms/options/plugin">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>plugin</uri_path>
+		<dc:comment/>
+		<provider_key>simple</provider_key>
+		<options/>
+		<conditions/>
+	</optionset>
+</rdmo>

From 740f7e0046ef6f6928cf8f6909a5d8d5cc13dd07 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 29 Feb 2024 16:31:03 +0100
Subject: [PATCH 101/205] chore: add diff_match_patch to
 track_changes_on_element

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 16782f12c6..9e035502e9 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -14,6 +14,8 @@
 
 from rest_framework.utils import model_meta
 
+from diff_match_patch import diff_match_patch
+
 from rdmo.core.constants import ELEMENT_COMMON_FIELDS, ELEMENT_IMPORT_EXTRA_FIELDS_DEFAULTS
 from rdmo.core.utils import get_languages
 
@@ -80,11 +82,14 @@ def track_changes_on_element(element: dict,
         # cast a list of elements with uris to a string with newlines
         new_value = "\n".join(i['uri'] for i in  new_value)
         original_value = "\n".join(i['uri'] for i in  original_value)
-
-    if new_value == original_value:
-        return
+    new_value = force_str(new_value)
+    original_value = force_str(original_value)
+    dmp = diff_match_patch()
+    diff = dmp.diff_main(original_value, new_value)
+    dmp.diff_cleanupSemantic(diff)
+    changed: bool = any(i[0] != dmp.DIFF_EQUAL for i in diff)
     #  TODO maybe rename updated to new
-    changes =  {'current': original_value, 'updated': new_value}
+    changes = {'current': original_value, 'updated': new_value, 'changed': changed}
     if element['updated_and_changed'].get(element_field) is None:
         element['updated_and_changed'][element_field] = changes
     else:
@@ -386,10 +391,11 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
         return
     if not all([source_name, target_name, through_name]):
         return
-
     target_elements = element.get(field_name) or []
     if isinstance(target_elements, str):
         target_elements = [target_elements]
+    elif isinstance(target_elements, dict):
+        target_elements = [target_elements]
 
     model_info = model_meta.get_field_info(instance)
     through_model = model_info.reverse_relations[through_name].related_model

From 85c9460246a9e4624b2e1bd3f525f6b7a467d690 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 29 Feb 2024 16:38:40 +0100
Subject: [PATCH 102/205] tests: fix and refactor tests for changed fields

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../tests/helpers_import_elements.py          | 55 +++++++++++--------
 rdmo/management/tests/helpers_xml.py          |  4 +-
 .../tests/test_import_conditions.py           | 18 +++---
 rdmo/management/tests/test_import_domain.py   | 15 +++--
 rdmo/management/tests/test_import_options.py  | 20 ++++---
 .../management/tests/test_import_questions.py | 49 +++++++++--------
 rdmo/management/tests/test_import_tasks.py    | 11 ++--
 rdmo/management/tests/test_import_views.py    | 11 ++--
 8 files changed, 101 insertions(+), 82 deletions(-)

diff --git a/rdmo/management/tests/helpers_import_elements.py b/rdmo/management/tests/helpers_import_elements.py
index fd380d5f3d..cf12633eeb 100644
--- a/rdmo/management/tests/helpers_import_elements.py
+++ b/rdmo/management/tests/helpers_import_elements.py
@@ -1,33 +1,44 @@
 from collections import OrderedDict
-from typing import Dict, List, Tuple
+from functools import partial
+from typing import Dict, List, Optional, Tuple
 
+from rdmo.core.imports import track_changes_on_element
+from rdmo.management.imports import set_updated_and_changed_meta_info
 
-def _test_helper_change_fields_elements(elements, update_dict=None, n=3) -> Tuple[Dict, List]:
-    """ xml test preparation function """
+UPDATE_FIELD_FUNCS = {
+    'comment': lambda text: f"this is a test comment {text}",
+    'target_text': lambda text: f"test target_text {text}",
+    'relation': lambda text: "notempty".format(),
+}
 
-    update_dict = update_dict if update_dict is not None else {}
-    _default_update_dict = {'comment':  "this is a test comment {}"}
-    update_dict.update(**_default_update_dict)
+
+def filter_changed_fields(element, updated_fields=None) -> bool:
+    if updated_fields is None:
+        return element.get('changed', False)
+    return element.get('changed', False) and any(i in updated_fields for i in element.get('changed_fields', []))
+
+
+def _test_helper_filter_updated_and_changed(elements: List[Dict], updated_fields: Optional[Tuple]) -> List[Dict]:
+    filter_func = partial(filter_changed_fields, updated_fields=updated_fields)
+    changed_elements = filter(filter_func, elements)
+    return list(changed_elements)
+
+def _test_helper_change_fields_elements(elements, fields_to_update: Optional[Tuple]=None, n=3) -> Tuple[Dict, List]:
+    """ elements test preparation function """
 
     if len(elements) < n:
         raise ValueError("Length of elements should not be smaller than n.")
     _new_elements = OrderedDict()
-    _changed_elements = OrderedDict()
     for _n,(_uri, _element) in enumerate(elements.items()):
         if _n <= n-1:
-            updated_and_changed = {}
-            changed_element = _element
-            for k,val in update_dict.items():
-                if isinstance(val, str):
-                    val = val.format(_n)
-                updated_and_changed[k]= {'current': _element[k], 'uploaded': val}
-                _element[k] = val
-            if updated_and_changed:
-                changed_element['updated_and_changed'] = updated_and_changed
-            _changed_elements[_uri] = changed_element
+            _element['updated_and_changed'] = {}
+            _element['changed'] = False
+            _element['changed_fields'] = []
+            for field in fields_to_update:
+                original_value = _element[field] or ''
+                new_val = UPDATE_FIELD_FUNCS[field](_n)
+                track_changes_on_element(_element, field, new_val, original_value=original_value)
+                _element[field] = new_val
+            set_updated_and_changed_meta_info(_element)
         _new_elements[_uri] = _element
-    return _new_elements, list(_changed_elements.values())
-
-def _test_helper_filter_updated_and_changed(elements: List[Dict]) -> List[Dict]:
-    filtered_elements = filter(lambda x: x.get('updated_and_changed', False), elements)
-    return list(filtered_elements)
+    return _new_elements
diff --git a/rdmo/management/tests/helpers_xml.py b/rdmo/management/tests/helpers_xml.py
index 8768ba456a..e594549550 100644
--- a/rdmo/management/tests/helpers_xml.py
+++ b/rdmo/management/tests/helpers_xml.py
@@ -1,5 +1,5 @@
 
-from rdmo.core.xml import XmlParser
+from rdmo.core.xml import XmlToElementsParser
 
 xml_error_files = [
     ('file-does-not-exist.xml', 'may not be blank'),
@@ -11,7 +11,7 @@
 
 def read_xml_and_parse_to_elements(xml_file):
 
-    xml_parser = XmlParser(file_name=xml_file)
+    xml_parser = XmlToElementsParser(file_name=xml_file)
     if xml_parser.errors:
         _msg = "\n".join(xml_parser.errors)
         raise ValueError(f"This test function should NOT raise any Exceptions. {_msg!s}")
diff --git a/rdmo/management/tests/test_import_conditions.py b/rdmo/management/tests/test_import_conditions.py
index 374297cee0..6784039078 100644
--- a/rdmo/management/tests/test_import_conditions.py
+++ b/rdmo/management/tests/test_import_conditions.py
@@ -11,13 +11,7 @@
 )
 from .helpers_xml import read_xml_and_parse_to_elements
 
-imported_update_changes = [
-    None,
-    {
-    'target_text' : 'test target_text {}',
-    'relation': 'notempty'
-    }
-]
+fields_to_be_changed = (('comment',),)
 
 def test_create_conditions(db, settings):
     Condition.objects.all().delete()
@@ -43,14 +37,16 @@ def test_update_conditions(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-@pytest.mark.parametrize('update_dict', imported_update_changes)
-def test_update_conditions_with_changed_fields(db, settings, update_dict):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_conditions_with_changed_fields(db, settings, updated_fields):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
     elements, root = read_xml_and_parse_to_elements(xml_file)
-    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=7)
+    # breakpoint()
+    elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=7)
+    changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert len(root) == len(imported_elements) == 15
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_domain.py b/rdmo/management/tests/test_import_domain.py
index 882f7c85c5..5700662b4e 100644
--- a/rdmo/management/tests/test_import_domain.py
+++ b/rdmo/management/tests/test_import_domain.py
@@ -1,13 +1,14 @@
 from pathlib import Path
 
+import pytest
+
 from rdmo.domain.models import Attribute
 from rdmo.management.imports import import_elements
 
-from .helpers_import_elements import (
-    _test_helper_change_fields_elements,
-)
+from .helpers_import_elements import _test_helper_change_fields_elements, _test_helper_filter_updated_and_changed
 from .helpers_xml import read_xml_and_parse_to_elements
 
+fields_to_be_changed = (('comment',),)
 
 def test_create_domain(db, settings):
     Attribute.objects.all().delete()
@@ -33,16 +34,18 @@ def test_update_domain(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-def test_update_attributes_with_changed_fields(db, settings):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_attributes_with_changed_fields(db, settings, updated_fields):
     _change_count = Attribute.objects.count() / 2
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
     elements, root = read_xml_and_parse_to_elements(xml_file)
     # import initial elements from xml
     _el = import_elements(elements, save=True)
     # update the elements and call import again
-    updated_elements, changed_elements = _test_helper_change_fields_elements(elements, n=_change_count)
+    updated_elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=_change_count)
+    changed_elements = _test_helper_filter_updated_and_changed(updated_elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(updated_elements)
-    imported_and_changed = list(filter(lambda x: x.get('updated_and_changed'), imported_elements))
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
 
     assert len(root) == len(imported_elements) == 86
     assert all(element['created'] is False for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 19bbd48b3b..8ca1d0bfdd 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -12,7 +12,7 @@
 from .helpers_models import delete_all_objects
 from .helpers_xml import read_xml_and_parse_to_elements
 
-imported_update_changes = [None]
+fields_to_be_changed = (('comment',),)
 
 def test_create_optionsets(db, settings):
     delete_all_objects([OptionSet, Option])
@@ -39,8 +39,8 @@ def test_update_optionsets(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-@pytest.mark.parametrize('update_dict', imported_update_changes)
-def test_update_optionsets_with_changed_fields(db, settings, update_dict):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_optionsets_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
@@ -49,10 +49,11 @@ def test_update_optionsets_with_changed_fields(db, settings, update_dict):
     assert len(root) == len(imported_elements) == 13
     # start test with fresh options in db
     _n_change = int(Option.objects.count() / 2)
-    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=7)
+    elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=7)
+    changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 13
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
     assert len(imported_and_changed) == len(changed_elements)
@@ -85,8 +86,8 @@ def test_update_options(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-@pytest.mark.parametrize('update_dict', imported_update_changes)
-def test_update_options_with_changed_fields(db, settings, update_dict):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_options_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
@@ -94,9 +95,10 @@ def test_update_options_with_changed_fields(db, settings, update_dict):
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 9
     # start test with fresh options in db
-    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=4)
+    elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=4)
+    changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert len(root) == len(imported_elements) == 9
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py
index e20db83776..db2ac94a3b 100644
--- a/rdmo/management/tests/test_import_questions.py
+++ b/rdmo/management/tests/test_import_questions.py
@@ -12,7 +12,7 @@
 from .helpers_models import delete_all_objects
 from .helpers_xml import read_xml_and_parse_to_elements
 
-imported_update_changes = [None]
+fields_to_be_changed = (('comment',),)
 
 
 def test_create_catalogs(db, settings):
@@ -44,8 +44,8 @@ def test_update_catalogs(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-@pytest.mark.parametrize('update_dict', imported_update_changes)
-def test_update_catalogs_with_changed_fields(db, settings, update_dict):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_catalogs_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
@@ -53,10 +53,11 @@ def test_update_catalogs_with_changed_fields(db, settings, update_dict):
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 148
     # start test with fresh elements in db
-
-    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=75)
+    # breakpoint()
+    elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=75)
+    changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert len(imported_elements) == 148
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
@@ -94,8 +95,8 @@ def test_update_sections(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-@pytest.mark.parametrize('update_dict', imported_update_changes)
-def test_update_sections_with_changed_fields(db, settings, update_dict):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_sections_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
@@ -104,9 +105,10 @@ def test_update_sections_with_changed_fields(db, settings, update_dict):
     assert len(root) == len(imported_elements) == 146
     # start test with fresh elements in db
 
-    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=75)
+    elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=75)
+    changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
     assert len(imported_and_changed) == len(changed_elements)
@@ -142,8 +144,8 @@ def test_update_pages(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-@pytest.mark.parametrize('update_dict', imported_update_changes)
-def test_update_pages_with_changed_fields(db, settings, update_dict):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_pages_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
@@ -151,9 +153,10 @@ def test_update_pages_with_changed_fields(db, settings, update_dict):
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 140
     # start test with fresh elements in db
-    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=75)
+    elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=75)
+    changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert len(imported_elements) == 140
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
@@ -191,8 +194,8 @@ def test_update_questionsets(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-@pytest.mark.parametrize('update_dict', imported_update_changes)
-def test_update_questionsets_with_changed_fields(db, settings, update_dict):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_questionsets_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
@@ -201,9 +204,10 @@ def test_update_questionsets_with_changed_fields(db, settings, update_dict):
     assert len(root) == 10  # two questionsets appear twice in the export file
     assert len(imported_elements) == 8
     # start test with fresh elements in db
-    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=5)
+    elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=5)
+    changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert len(imported_elements) == 8
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
@@ -238,8 +242,8 @@ def test_update_questions(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-@pytest.mark.parametrize('update_dict', imported_update_changes)
-def test_update_questions_with_changed_fields(db, settings, update_dict):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_questions_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
@@ -247,9 +251,10 @@ def test_update_questions_with_changed_fields(db, settings, update_dict):
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 89
     # start test with fresh elements in db
-    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=45)
+    elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=45)
+    changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert len(imported_elements) == 89
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_tasks.py b/rdmo/management/tests/test_import_tasks.py
index 4d026cbabe..ff169fb2dc 100644
--- a/rdmo/management/tests/test_import_tasks.py
+++ b/rdmo/management/tests/test_import_tasks.py
@@ -11,7 +11,7 @@
 )
 from .helpers_xml import read_xml_and_parse_to_elements
 
-imported_update_changes = [None]
+fields_to_be_changed = (('comment',),)
 
 
 def test_create_tasks(db, settings):
@@ -38,14 +38,15 @@ def test_update_tasks(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-@pytest.mark.parametrize('update_dict', imported_update_changes)
-def test_update_tasks_with_changed_fields(db, settings, update_dict):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_tasks_with_changed_fields(db, settings, updated_fields):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'tasks.xml'
 
     elements, root = read_xml_and_parse_to_elements(xml_file)
-    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=1)
+    elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=1)
+    changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert len(root) == len(imported_elements) == 2
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_views.py b/rdmo/management/tests/test_import_views.py
index c0ebd35e4c..f3cf0017ed 100644
--- a/rdmo/management/tests/test_import_views.py
+++ b/rdmo/management/tests/test_import_views.py
@@ -11,7 +11,7 @@
 )
 from .helpers_xml import read_xml_and_parse_to_elements
 
-imported_update_changes = [None]
+fields_to_be_changed = (('comment',),)
 
 
 def test_create_tasks(db, settings):
@@ -38,14 +38,15 @@ def test_update_tasks(db, settings):
     assert all(element['updated'] is True for element in imported_elements)
 
 
-@pytest.mark.parametrize('update_dict', imported_update_changes)
-def test_update_views_with_changed_fields(db, settings, update_dict):
+@pytest.mark.parametrize('updated_fields', fields_to_be_changed)
+def test_update_views_with_changed_fields(db, settings, updated_fields):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'views.xml'
 
     elements, root = read_xml_and_parse_to_elements(xml_file)
-    elements, changed_elements = _test_helper_change_fields_elements(elements, update_dict=update_dict, n=2)
+    elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=2)
+    changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
-    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements)
+    imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert len(root) == len(imported_elements) == 3
     assert all(element['created'] is False for element in imported_elements)
     assert all(element['updated'] is True for element in imported_elements)

From e142b48d48f181a0f29f6b87e82e6255d2772690 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 29 Feb 2024 16:39:57 +0100
Subject: [PATCH 103/205] chore: rename to XmlToElementsParser

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/xml.py                              | 7 ++++---
 rdmo/management/management/commands/import.py | 4 ++--
 rdmo/management/viewsets.py                   | 6 +++---
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index 353b7ddeff..c436368e16 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -3,6 +3,7 @@
 from collections import OrderedDict
 from dataclasses import dataclass, field
 from pathlib import Path
+from typing import Optional
 
 from django.utils.translation import gettext_lazy as _
 
@@ -31,7 +32,7 @@
 
 
 @dataclass
-class XmlParser:
+class XmlToElementsParser:
 
     file_name:str = None
     # post init attributes
@@ -56,7 +57,7 @@ def is_valid(self, raise_exception: bool = False) -> bool:
             raise ValueError(self.errors)
         return not bool(self.errors)
 
-    def parse_xml_to_elements(self, xml_file: Path, raise_exception:bool=False) -> None:
+    def parse_xml_to_elements(self, xml_file: Path, raise_exception:bool=False) -> Optional[OrderedDict]:
         root = None
         # step 2: parse xml
         try:
@@ -218,7 +219,7 @@ def strip_ns(tag, ns_map):
 def convert_elements(elements, version: Version):
     if not isinstance(version, Version):
         raise TypeError('Version should be a parsed version type. (parse(version))')
-    pre_conversion_validate_legacy_elements(elements, version)
+    # pre_conversion_validate_legacy_elements(elements, version)
     if version < parse('2.0.0'):
         elements = convert_legacy_elements(elements)
 
diff --git a/rdmo/management/management/commands/import.py b/rdmo/management/management/commands/import.py
index db86203747..73bec7cb5b 100644
--- a/rdmo/management/management/commands/import.py
+++ b/rdmo/management/management/commands/import.py
@@ -2,7 +2,7 @@
 
 from django.core.management.base import BaseCommand, CommandError
 
-from rdmo.core.xml import XmlParser
+from rdmo.core.xml import XmlToElementsParser
 from rdmo.management.imports import import_elements
 
 logger = logging.getLogger(__name__)
@@ -16,7 +16,7 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
 
         try:
-            xml_parser = XmlParser(file_name=options['xmlfile'])
+            xml_parser = XmlToElementsParser(file_name=options['xmlfile'])
         except CommandError as e:
             logger.info('Import failed with XML parsing errors.')
             raise CommandError(str(e)) from e
diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index 5beab57b2e..722c726c70 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -12,7 +12,7 @@
 from rdmo.core.imports import handle_uploaded_file
 from rdmo.core.permissions import CanToggleElementCurrentSite
 from rdmo.core.utils import get_model_field_meta, is_truthy
-from rdmo.core.xml import XmlParser
+from rdmo.core.xml import XmlToElementsParser
 
 from .constants import RDMO_MODEL_PATH_MAPPER
 from .imports import import_elements
@@ -42,9 +42,9 @@ def create(self, request, *args, **kwargs):
         else:
             import_tmpfile_name = handle_uploaded_file(uploaded_file)
         try:
-            # step 1.1: initialize XmlParser
+            # step 1.1: initialize XmlToElementsParser
             # step 2-6: parse xml, validate and convert to
-            xml_parser = XmlParser(import_tmpfile_name)
+            xml_parser = XmlToElementsParser(import_tmpfile_name)
         except ValidationError as e:
             logger.info('Import failed with XML parsing errors.')
             raise ValidationError({'file': e}) from e

From 515c567a772ac5bbbfa0e14b982e04b06055f558 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 29 Feb 2024 16:40:56 +0100
Subject: [PATCH 104/205] chore: update updated_and_changed meta info on
 element

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/imports.py | 39 +++++++++-----------------------------
 1 file changed, 9 insertions(+), 30 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index e394cdc124..e3bd9197e0 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -58,6 +58,8 @@
         'errors': list,
         'created': bool,
         'updated': bool,
+        'changed': bool,
+        'changed_fields': list,
         'updated_and_changed': dict,
     }
 
@@ -168,6 +170,9 @@ def import_element(
         for reverse_m2m_fields in import_helper.reverse_m2m_through_instance_fields:
             set_reverse_m2m_through_instance(instance, element, **asdict(reverse_m2m_fields),
                                              original=original, save=save)
+    # set aggregated changes potentially to True and a list of changed fields
+    if _updated and element['updated_and_changed']:
+        set_updated_and_changed_meta_info(element)
 
     if save and settings.MULTISITE:
         # could be optimized with a bulk_create of through model later
@@ -186,33 +191,7 @@ def strip_uri_prefix_endswith_slash(element: dict) -> dict:
         element['uri_prefix'] = element['uri_prefix'].rstrip('/')
     return element
 
-#  TODO remove get_updated_changes
-def get_updated_changes(element, new_instance,
-                        original, serializer, request=None) -> Dict[str, str]:
-    original_serializer = serializer(original, context={'request': request})
-    original_data = original_serializer.data
-    original_element = {k: val for k,val in original_data.items() if k in element}
-    uploaded_serializer = serializer(new_instance, context={'request': request})
-    uploaded_data = uploaded_serializer.data
-    uploaded_element = {k: val for k,val in uploaded_data.items() if k in element}
-
-    updated_and_changed = {}
-    for k, old_val in original_element.items():
-        new_val = uploaded_element[k]
-        if old_val != new_val and any([old_val,new_val]):
-            updated_and_changed[k] = {"current": old_val, "uploaded": new_val}
-    # overwrite the normal "element" name with the value from "element_uri"
-    uri_keys = {k for k in list(original_data.keys())+list(uploaded_data.keys())
-                if k.endswith('_uri') or k.endswith('_uris')}
-    for uri_key in uri_keys:
-        element_name, uri_field = uri_key.split('_')
-        if uri_key in updated_and_changed:
-            # e.g. set attribute as key instead of attribute_uri
-            uri_key_val = updated_and_changed[uri_key].pop()
-            updated_and_changed[element_name] = uri_key_val
-        if element_name in element and uri_key in original_data:
-            old_val = original_data[uri_key]
-            new_val = element[element_name].get('uri')
-            if old_val != new_val and any([old_val,new_val]):
-                updated_and_changed[element_name] = {"current": old_val, "uploaded": new_val}
-    return updated_and_changed
+def set_updated_and_changed_meta_info(element: dict) -> dict:
+    changed_fields = {k: val for k, val in element['updated_and_changed'].items() if val['changed']}
+    element['changed'] = bool(changed_fields)
+    element['changed_fields'] = list(changed_fields.keys())

From c282e80013b41f7c402d40d31b2f6832a6be7fb2 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 29 Feb 2024 16:41:35 +0100
Subject: [PATCH 105/205] js chore: resolve type casting

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/js/reducers/importsReducer.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index ad41517117..bcb910278b 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -43,7 +43,7 @@ export default function importsReducer(state = initialState, action) {
 
     // update element
     case 'import/updateElement':
-      index = state.elements.findIndex(element => element == action.element)
+      index = state.elements.findIndex(element => element === action.element)
       if (index > -1) {
         const elements = [...state.elements]
         elements[index] = {...elements[index], ...action.values}

From cca991698d017329930356616673ef4935c59d8a Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 29 Feb 2024 16:42:07 +0100
Subject: [PATCH 106/205] chore: update style

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/conditions/imports.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index 838a91fcaf..e83370efe1 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -5,8 +5,8 @@
 from .validators import ConditionLockedValidator, ConditionUniqueURIValidator
 
 import_helper_condition = ElementImportHelper(
-    model= Condition,
-    model_path = "conditions.condition",
+    model=Condition,
+    model_path="conditions.condition",
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
     foreign_fields=('source', 'target_option'),
     serializer=ConditionSerializer,

From f98bd2df100117330f65ddff8b0a35f174ce4216 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 29 Feb 2024 16:43:19 +0100
Subject: [PATCH 107/205] js: change prop on element object to .changed

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/js/components/import/ImportElement.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index 1bf5ea208a..29eb1d3ccb 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -21,7 +21,7 @@ const ImportElement = ({ config, element, importActions }) => {
       <div className="pull-right">
         <WarningLink show={!isEmpty(element.warnings)} onClick={updateShowField} />
         <ErrorLink show={!isEmpty(element.errors)} onClick={updateShowField} />
-        <ShowUpdatedLink show={(!isEmpty(element.updated_and_changed) && !element.created)} onClick={updateShowField} />
+        <ShowUpdatedLink show={(element.changed && !element.created)} onClick={updateShowField} />
         <ShowLink show={element.show} onClick={updateShowField} />
 
       </div>

From 142d13bd05f91effbfbeadaa9aa5a751ec10efa1 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 29 Feb 2024 17:15:36 +0100
Subject: [PATCH 108/205] build: add diff-match-patch==v20230430 to
 dependencies

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index 23e974baf2..4238b07566 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -62,6 +62,7 @@ dependencies = [
   "pypandoc~=1.11",
   "requests-toolbelt~=1.0",
   "rules~=3.3",
+  "diff-match-patch==v20230430",
 ]
 
 [project.optional-dependencies]

From 37ccc8e8f6aaecebf37bd710a2172234e113ad60 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 1 Mar 2024 03:31:08 +0100
Subject: [PATCH 109/205] js: rename to showTitle and clean up

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/xml.py                                              | 4 ++--
 rdmo/management/assets/js/components/import/ImportElement.js  | 2 +-
 .../assets/js/components/import/ImportSuccessElement.js       | 2 +-
 .../assets/js/components/import/ImportWarningsPanel.js        | 4 +---
 .../assets/js/components/import/common/ImportFilters.js       | 1 -
 rdmo/management/assets/js/components/main/Import.js           | 2 +-
 6 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index c436368e16..94577395ba 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -219,7 +219,7 @@ def strip_ns(tag, ns_map):
 def convert_elements(elements, version: Version):
     if not isinstance(version, Version):
         raise TypeError('Version should be a parsed version type. (parse(version))')
-    # pre_conversion_validate_legacy_elements(elements, version)
+    pre_conversion_validate_legacy_elements(elements, version)
     if version < parse('2.0.0'):
         elements = convert_legacy_elements(elements)
 
@@ -231,7 +231,7 @@ def convert_elements(elements, version: Version):
 
 def pre_conversion_validate_legacy_elements(elements, version: Version) -> None:
     if version < parse('2.0.0'):
-        _keys_in_elements = list(filter(lambda x: 'key' in x, elements.values()))
+        _keys_in_elements = list(filter(lambda x: 'key' in x and x['model'] != 'domain.attribute', elements.values()))
         if not _keys_in_elements:
             raise ValueError(f"Missing legacy elements, elements containing 'key' were expected for this XML with version {version}.")   # noqa: E501
 
diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index 29eb1d3ccb..1cd8764e33 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -36,7 +36,7 @@ const ImportElement = ({ config, element, importActions }) => {
         element.show && <>
           <Form config={config} element={element} updateElement={updateElement} />
           <Fields element={element} />
-          <Warnings element={element} success={false}/>
+          <Warnings element={element} showTitle={false}/>
           <Errors element={element} />
         </>
       }
diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index 263887fa27..d71be20d35 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -24,7 +24,7 @@ const ImportSuccessElement = ({ element }) => {
         }
         {'.'}
       </p>
-        <Warnings element={element} success={true}/>
+        <Warnings element={element} showTitle={true}/>
       {element.errors.map(message => <p key={uniqueId()} className="text-danger">{message}</p>)}
     </li>
   )
diff --git a/rdmo/management/assets/js/components/import/ImportWarningsPanel.js b/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
index d71bc668ff..a36c95d54c 100644
--- a/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
+++ b/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
@@ -15,10 +15,8 @@ const ImportWarningsPanel = ({ config, elements, configActions }) => {
   }
   const showWarnings = get(config, 'filter.import.warnings.show', false)
   const listWarnings = elements.map((element, index) => {
-              return (<Warnings key={index} element={element} showWarningTitle={true} success={false} />)
+              return (<Warnings key={index} element={element} showWarningTitle={true} showTitle={false} />)
               })
-  // const toggleImport = () => importActions.updateElement(element, {import: !element.import})
-  // const updateElement = (key, value) => importActions.updateElement(element, {[key]: value})
   return (
     <div className="panel panel-warning">
       <div className="panel-heading"><strong onClick={updateShowWarnings}>{gettext('Warnings')}{' '}({elements.length}){': '}</strong>
diff --git a/rdmo/management/assets/js/components/import/common/ImportFilters.js b/rdmo/management/assets/js/components/import/common/ImportFilters.js
index 47afdd0d8b..e0471b72b1 100644
--- a/rdmo/management/assets/js/components/import/common/ImportFilters.js
+++ b/rdmo/management/assets/js/components/import/common/ImportFilters.js
@@ -6,7 +6,6 @@ import {getUriPrefixes} from '../../../utils/filter'
 import {Checkbox} from '../../common/Checkboxes'
 
 const ImportFilters = ({ config, elements, updatedAndChanged, filteredElements, configActions }) => {
-  console.log('importFilter', updatedAndChanged, elements)
   const updateFilterString = (value) => configActions.updateConfig('filter.import.elements.search', value)
   const getValueFilterString = () => get(config, 'filter.import.elements.search', '')
   const updateFilterUriPrefix = (value) => configActions.updateConfig('filter.import.elements.uri_prefix', value)
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index b8fdd3d11f..aa8dc3d259 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -13,7 +13,7 @@ import get from 'lodash/get'
 const Import = ({ config, imports, configActions, importActions }) => {
   const { file, elements, success } = imports
 
-  const updatedAndChangedElements = elements.filter(element => element.updated && !isEmpty(element.updated_and_changed))
+  const updatedAndChangedElements = elements.filter(element => element.updated && element.changed)
 
   const updatedElements = elements.filter(element => element.updated)
   const createdElements = elements.filter(element => element.created)

From 6fee3c5ab03312c9667ae45cce18638958757090 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 1 Mar 2024 03:31:57 +0100
Subject: [PATCH 110/205] js: update FieldsDiffs.js, add messages

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/common/Fields.js     |  2 +
 .../components/import/common/FieldsDiffs.js   | 40 ++++++++++++++-----
 2 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index 3d543d395a..811af79842 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -26,6 +26,8 @@ const excludeKeys = [
   'valid',
   'warnings',
   'updated_and_changed',
+  'changed',
+  'changed_fields',
 ]
 
 const Fields = ({ element }) => {
diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index ec6155f259..555588f5c3 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -3,23 +3,45 @@ import PropTypes from 'prop-types'
 import isEmpty from 'lodash/isEmpty'
 import ReactDiffViewer from 'react-diff-viewer-continued'
 import { isUndefined } from 'lodash'
+import Warnings from './Warnings'
+import Errors from './Errors'
 
 const FieldsDiffs = ({ element, field }) => {
-  const newVal = element.updated_and_changed[field].updated ?? ''
-  const oldVal = element.updated_and_changed[field].current ?? ''
-  return (!isUndefined(element) &&
-          !isEmpty(element.updated_and_changed) &&
-          !isUndefined(newVal) &&
+  if (isEmpty(element.updated_and_changed[field])) {
+      return null
+  }
+  const fieldDiffData = element.updated_and_changed[field]
+  const newVal = fieldDiffData.updated ?? ''
+  const oldVal = fieldDiffData.current ?? ''
+  const changed = fieldDiffData.changed ?? false
+  const hideLineNumbers = fieldDiffData.hideLineNumbers ?? true
+  const splitView = fieldDiffData.splitView ?? true
+  const leftTitle = fieldDiffData.leftTitle ?? gettext('Current')
+  const rightTitle = fieldDiffData.rightTitle ?? gettext('Uploaded')
+  const warnings = fieldDiffData.warnings ?? {}
+  const errors = fieldDiffData.errors ?? []
+
+  return (changed && !isUndefined(newVal) && !isUndefined(oldVal) &&
      <div className="col-sm-12">
       <ReactDiffViewer
           oldValue={oldVal.toString()}
           newValue={newVal.toString()}
-          splitView={true}
-          hideLineNumbers={true}
-          leftTitle={gettext('Current')}
-          rightTitle={gettext('Uploaded')}
+          splitView={splitView}
+          hideLineNumbers={hideLineNumbers}
+          leftTitle={leftTitle}
+          rightTitle={rightTitle}
           >
           </ReactDiffViewer>
+      {
+        !isEmpty(warnings) && <>
+        <Warnings element={fieldDiffData} showTitle={true} shouldShowURI={false}/>
+        </>
+      }
+      {
+        !isEmpty(errors) && <>
+        <Errors element={fieldDiffData} />
+        </>
+      }
       </div>
   )
 }

From aedc594e0018ee58fd49ee0f46558ee8c0449c92 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 1 Mar 2024 03:37:07 +0100
Subject: [PATCH 111/205] js: add messages FieldsDiffs.js, update Warnings.js

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/common/Warnings.js   | 51 ++++++++++---------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/Warnings.js b/rdmo/management/assets/js/components/import/common/Warnings.js
index 52520a9066..d288087b5d 100644
--- a/rdmo/management/assets/js/components/import/common/Warnings.js
+++ b/rdmo/management/assets/js/components/import/common/Warnings.js
@@ -4,29 +4,30 @@ import PropTypes from 'prop-types'
 import uniqueId from 'lodash/uniqueId'
 import {codeClass} from '../../../constants/elements'
 
-const Warnings = ({element, success = false}) => {
-  const listWarningMessages = Object.entries(element.warnings).map(([uri, messages]) => {
-    return (
-      <li key={uniqueId('warning-uri-')}><code className={codeClass['domain.attribute']}>{uri}</code>
-        <ul className="list-unstyled">
-          {
-            messages.map(message => {
-              return (
-                <li className="text-warning" key={uniqueId('warning-uri-message')}>{message}</li>
-              )
-            })
-          }
-        </ul>
-      </li>
-    )
-  })
+
+const Warnings = ({element, showTitle = false, shouldShowURI = true}) => {
+  const generateWarningMessagesForUri = (messages, key) =>
+    messages.map(message => <li className="text-warning" key={key}>{message}</li>)
+
+  const prepareWarningsList = (warningsObj) =>
+    Object.entries(warningsObj).map(([uri, messages]) => (
+      <ul className="list-unstyled" key={uniqueId('warning-uri-list')}>
+        {shouldShowURI &&
+          <li key={uniqueId('warning-uri-')}>
+            <code className={codeClass['domain.attribute']}>{uri}</code>
+          </li>
+        }
+        {generateWarningMessagesForUri(messages, uniqueId('warning-uri-message'))}
+      </ul>
+    ))
+
+  const listWarningMessages = prepareWarningsList(element.warnings)
 
   return (
     <div className="row text-warning mt-10">
-      {
-        success === true && listWarningMessages.length > 0 &&
+      {showTitle === true && listWarningMessages.length > 0 &&
         <div className="col-sm-3 text-right">
-          {gettext('Warnings')}
+          {'Warnings'}
         </div>
       }
       <div className="col-sm-9">
@@ -36,9 +37,11 @@ const Warnings = ({element, success = false}) => {
   )
 }
 
-Warnings.propTypes = {
-  element: PropTypes.object.isRequired,
-  success: PropTypes.bool.isRequired
-}
 
-export default Warnings
+  Warnings.propTypes = {
+    element: PropTypes.object.isRequired,
+    showTitle: PropTypes.bool.isRequired,
+    shouldShowURI: PropTypes.bool,
+  }
+
+  export default Warnings

From 1933e6ff5ec90f79101e847fbecff5ed171a88c4 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 1 Mar 2024 03:43:52 +0100
Subject: [PATCH 112/205] chore: add track_messages_on_element

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 197 ++++++++++++++++++++++++++-----------------
 1 file changed, 120 insertions(+), 77 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 9e035502e9..4a74b3f743 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -1,6 +1,7 @@
 import logging
 import tempfile
 import time
+from collections import defaultdict
 from dataclasses import dataclass, field
 from os.path import join as pj
 from pathlib import Path
@@ -65,23 +66,45 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
     return f"{verbose_name} {uri} updated"
 
 
+def track_messages_on_element(element: dict, element_field: str,
+                              warning: Optional[str]=None, error: Optional[str]=None):
+    if element['updated_and_changed'].get(element_field) is None:
+        element['updated_and_changed'][element_field] = {}
+        element['updated_and_changed'][element_field]['errors'] = []
+        element['updated_and_changed'][element_field]['warnings'] = defaultdict(list)
+
+    if warning is not None:
+        if 'warning' not in element['updated_and_changed'][element_field]:
+            element['updated_and_changed'][element_field]['warnings'] = defaultdict(list)
+        element['updated_and_changed'][element_field]['warnings'][element['uri']].append(warning)
+    if error is not None:
+        if 'error' not in element['updated_and_changed'][element_field]:
+            element['updated_and_changed'][element_field]['errors'] = []
+        element['updated_and_changed'][element_field]['errors'].append(error)
+
+
 def track_changes_on_element(element: dict,
                              element_field: str,
-                             new_value: Union[str,List],
-                             instance_field: Optional[str]=None,
-                             original=None,
-                             original_value: Optional[Union[str,List]]=None):
+                             new_value: Union[str,List[str]],
+                             instance_field: Optional[str] = None,
+                             original = None,
+                             original_value: Optional[Union[str, List[str]]] = None):
     if original is None and original_value is None:
         return
 
+    js_diff_viewer_props = {}
+
     _get_field = element_field if instance_field is None else instance_field
     if original_value is None:
-        original_value = force_str(getattr(original, _get_field, ''))
+        original_value = getattr(original, _get_field, '')
 
     if isinstance(new_value, list) and isinstance(original_value, list):
-        # cast a list of elements with uris to a string with newlines
-        new_value = "\n".join(i['uri'] for i in  new_value)
-        original_value = "\n".join(i['uri'] for i in  original_value)
+        # cast a list of strings with uris to a string with newlines
+        new_value = "\n".join(new_value)
+        original_value = "\n".join(original_value)
+        js_diff_viewer_props['hideLineNumbers'] = False
+        js_diff_viewer_props['splitView'] = False
+
     new_value = force_str(new_value)
     original_value = force_str(original_value)
     dmp = diff_match_patch()
@@ -89,7 +112,11 @@ def track_changes_on_element(element: dict,
     dmp.diff_cleanupSemantic(diff)
     changed: bool = any(i[0] != dmp.DIFF_EQUAL for i in diff)
     #  TODO maybe rename updated to new
-    changes = {'current': original_value, 'updated': new_value, 'changed': changed}
+    changes = {'current': original_value,
+               'updated': new_value,
+               'changed': changed
+               }
+    changes.update(js_diff_viewer_props)
     if element['updated_and_changed'].get(element_field) is None:
         element['updated_and_changed'][element_field] = changes
     else:
@@ -183,6 +210,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
         )
         logger.info(message)
         element['errors'][element.get('uri')].append(message)
+        track_messages_on_element(element, field_name, error=message)
         return
 
     foreign_uri = foreign_element['uri']
@@ -201,6 +229,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
         )
         logger.info(message)
         element['warnings'][foreign_uri].append(message)
+        track_messages_on_element(element, field_name, warning=message)
     try:
         if foreign_instance is not None:
             setattr(instance, field_name, foreign_instance)
@@ -219,6 +248,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
             )
         logger.info(message)
         element['errors'][foreign_uri].append(message)
+        track_messages_on_element(element, field_name, error=message)
 
 
 def set_extra_field(instance, field_name, element, questions_widget_types=None, original=None) -> None:
@@ -243,11 +273,11 @@ def set_extra_field(instance, field_name, element, questions_widget_types=None,
             )
             logger.info(message)
             element['errors'].append(message)
+            track_messages_on_element(element, field_name, error=message)
 
     setattr(instance, field_name, extra_value)
     # track changes
-    track_changes_on_element(element, field_name,
-                                 extra_value, original=original)
+    track_changes_on_element(element, field_name, extra_value, original=original)
 
 def track_changes_m2m_instances(element, field_name,
                                 foreign_instances, original=None):
@@ -256,48 +286,12 @@ def track_changes_m2m_instances(element, field_name,
     original_m2m_instance = getattr(original, field_name)
     if original_m2m_instance is None:
         return
-    original_m2m_uris = original_m2m_instance.values_list('uri', flat=True)
+    original_m2m_uris = list(original_m2m_instance.values_list('uri', flat=True))
     foreign_uris = [i.uri for i in foreign_instances]
     track_changes_on_element(element, field_name, foreign_uris,
                              original_value=original_m2m_uris)
 
 
-def set_m2m_instances(instance, element, field_name, original=None, save=None):
-    if field_name not in element:
-        return
-
-    foreign_elements = element.get(field_name, [])
-
-    if not foreign_elements:
-        getattr(instance, field_name).clear()
-        return
-
-    foreign_instances = []
-
-    model_info = model_meta.get_field_info(instance)
-    foreign_model = model_info.forward_relations[field_name].related_model
-
-    for foreign_element in foreign_elements:
-        foreign_uri = foreign_element.get('uri')
-
-        try:
-            foreign_instance = foreign_model.objects.get(uri=foreign_uri)
-            foreign_instances.append(foreign_instance)
-        except foreign_model.DoesNotExist:
-            message = '{foreign_model} {foreign_uri} for {instance_model} {instance_uri} does not exist.'.format(
-                foreign_model=foreign_model._meta.object_name,
-                foreign_uri=foreign_uri,
-                instance_model=instance._meta.object_name,
-                instance_uri=element.get('uri')
-            )
-            logger.info(message)
-            element['warnings'][foreign_uri].append(message)
-    if save:
-        getattr(instance, field_name).set(foreign_instances)
-    track_changes_m2m_instances(element, field_name,
-                                foreign_instances, original=original)
-
-
 def set_m2m_through_instances(instance, element, field_name=None, source_name=None,
                               target_name=None, through_name=None,
                               original=None, save=None) -> None:
@@ -318,6 +312,8 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
     _track_changes = {}
     _track_changes['new_data'] = []
     _track_changes['current_data'] = []
+
+
     if original is not None:
         try:
             for _order, _through_instance in enumerate(getattr(original, field_name).all()):
@@ -370,18 +366,62 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
             )
             logger.info(message)
             element['warnings'][target_uri].append(message)
+            track_messages_on_element(element, field_name, warning=message)
     if save:
         # remove the remainders of the items list
         for through_instance in through_instances:
             if getattr(through_instance, target_name).uri_prefix == instance.uri_prefix:
                 through_instance.delete()
+    new_instance_data = sorted(_track_changes['new_data'], key=lambda k: k['order'])
+    original_instance_data = sorted(_track_changes['current_data'], key=lambda k: k['order'])
+    track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data)
+
+def track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data):
     # sort the tracked changes by order in-place
-    _track_changes['new'] = sorted(_track_changes['new_data'], key=lambda k: k['order'])
-    _track_changes['current'] = sorted(_track_changes['current_data'], key=lambda k: k['order'])
-    _store = {'new_data' : _track_changes['new_data'],  'current_data' : _track_changes['current_data']}
+    _store = {'new_data': new_instance_data,  'current_data': original_instance_data}
     element['updated_and_changed'][field_name] = _store
-    track_changes_on_element(element, field_name, _track_changes['new'],
-                             original_value=_track_changes['current'])
+    new_values = [i['uri'] for i in new_instance_data]
+    original_values = [i['uri'] for i in original_instance_data]
+    track_changes_on_element(element, field_name, new_values, original_value=original_values)
+
+
+def set_m2m_instances(instance, element, field_name, original=None, save=None):
+    if field_name not in element:
+        return
+
+    foreign_elements = element.get(field_name, [])
+
+    if not foreign_elements:
+        getattr(instance, field_name).clear()
+        return
+
+    foreign_instances = []
+
+    model_info = model_meta.get_field_info(instance)
+    foreign_model = model_info.forward_relations[field_name].related_model
+
+    for foreign_element in foreign_elements:
+        foreign_uri = foreign_element.get('uri')
+
+        try:
+            foreign_instance = foreign_model.objects.get(uri=foreign_uri)
+            foreign_instances.append(foreign_instance)
+        except foreign_model.DoesNotExist:
+            message = '{foreign_model} {foreign_uri} for {instance_model} {instance_uri} does not exist.'.format(
+                foreign_model=foreign_model._meta.object_name,
+                foreign_uri=foreign_uri,
+                instance_model=instance._meta.object_name,
+                instance_uri=element.get('uri')
+            )
+            logger.info(message)
+            element['warnings'][foreign_uri].append(message)
+            track_messages_on_element(element, field_name, warning=message)
+    if save:
+        getattr(instance, field_name).set(foreign_instances)
+    track_changes_m2m_instances(element, field_name,
+                                foreign_instances, original=original)
+
+
 
 
 def set_reverse_m2m_through_instance(instance, element, field_name=None, source_name=None,
@@ -429,6 +469,7 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
                 )
                 logger.info(message)
                 element['errors'].append(message)
+                track_messages_on_element(element, field_name, error=message)
                 continue
             if save:
                 through_instance, created = through_model.objects.get_or_create(**{
@@ -449,36 +490,38 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
             )
             logger.info(message)
             element['warnings'][target_uri].append(message)
+            track_messages_on_element(element, field_name, warning=message)
     # sort the tracked changes by order in-place
-    _track_changes['new'] = sorted(_track_changes['new_data'], key=lambda k: k['order'])
-    _track_changes['current'] = sorted(_track_changes['current_data'], key=lambda k: k['order'])
-    track_changes_on_element(element, field_name, _track_changes['new'],
-                             original_value=_track_changes['current'])
+    new_instance_data = sorted(_track_changes['new_data'], key=lambda k: k['order'])
+    original_instance_data = sorted(_track_changes['current_data'], key=lambda k: k['order'])
+    track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data)
 
 
 def validate_instance(instance, element, *validators):
     exception_message = None
-    try:
-        instance.full_clean()
-        for validator in validators:
-            validator(instance if instance.id else None)(vars(instance))
-    except ValidationError as e:
+    instance.full_clean()
+    for validator in validators:
         try:
-            exception_message = '; '.join(['{}: {}'.format(key, ', '.join(messages))
-                                           for key, messages in e.message_dict.items()])
-        except AttributeError:
-            exception_message = ''.join(e.messages)
-    except ObjectDoesNotExist as e:
-        exception_message = e
-    finally:
-        if exception_message is not None:
-            message = '{instance_model} {instance_uri} cannot be imported ({exception}).'.format(
-                instance_model=instance._meta.object_name,
-                instance_uri=element.get('uri'),
-                exception=exception_message
-            )
-            logger.info(message)
-            element['errors'].append(message)
+            validator(instance if instance.id else None)(vars(instance))
+        except ValidationError as e:
+            try:
+                exception_message = '; '.join(['{}: {}'.format(key, ', '.join(messages))
+                                               for key, messages in e.message_dict.items()])
+            except AttributeError:
+                exception_message = ''.join(e.messages)
+        except ObjectDoesNotExist as e:
+            exception_message = e
+        finally:
+            if exception_message is not None:
+                message = '{instance_model} {instance_uri} cannot be imported ({exception}).'.format(
+                    instance_model=instance._meta.object_name,
+                    instance_uri=element.get('uri'),
+                    exception=exception_message
+                )
+                logger.info(message)
+                _key = validator.__qualname__
+                element['errors'].append(message)
+                track_messages_on_element(element, _key, error=message)
 
 
 def check_permissions(instance: models.Model, element_uri: str, user: models.Model) -> Optional[str]:

From 6352bb3d8e55b9f7c2bcbe8f2bd433445b9dd6e2 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 4 Mar 2024 08:54:48 +0100
Subject: [PATCH 113/205] chore: fix xml legacy validation for domain

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/xml.py                     | 23 +++++++++++++++--------
 rdmo/management/tests/helpers_xml.py |  2 +-
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index 94577395ba..2b13cc736f 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -29,14 +29,15 @@
 }
 
 DEFAULT_RDMO_XML_VERSION = '1.11.0'
+ELEMENTS_USING_KEY = {'domain.attribute'}
 
 
 @dataclass
 class XmlToElementsParser:
 
-    file_name:str = None
+    file_name: str = None
     # post init attributes
-    file:Path = None  # will be set from file_name
+    file: Path = None  # will be set from file_name
     root = None
     errors: list = field(default_factory=list)
     parsed_elements: OrderedDict = field(default_factory=OrderedDict)
@@ -101,7 +102,7 @@ def parse_xml_to_elements(self, xml_file: Path, raise_exception:bool=False) -> O
 
         # step 3.1: validate elements for legacy versions
         try:
-            pre_conversion_validate_legacy_elements(elements, root_version)
+            pre_conversion_validate_missing_key_in_legacy_elements(elements, root_version)
         except ValueError as e:
             logger.info('Import failed with ValueError (%s)' % e)
             self.errors.append(_('XML Parsing Error') + f': {e!s}')
@@ -219,8 +220,8 @@ def strip_ns(tag, ns_map):
 def convert_elements(elements, version: Version):
     if not isinstance(version, Version):
         raise TypeError('Version should be a parsed version type. (parse(version))')
-    pre_conversion_validate_legacy_elements(elements, version)
     if version < parse('2.0.0'):
+        pre_conversion_validate_missing_key_in_legacy_elements(elements, version)
         elements = convert_legacy_elements(elements)
 
     if version < parse('2.1.0'):
@@ -229,11 +230,17 @@ def convert_elements(elements, version: Version):
     return elements
 
 
-def pre_conversion_validate_legacy_elements(elements, version: Version) -> None:
+def pre_conversion_validate_missing_key_in_legacy_elements(elements, version: Version) -> None:
     if version < parse('2.0.0'):
-        _keys_in_elements = list(filter(lambda x: 'key' in x and x['model'] != 'domain.attribute', elements.values()))
-        if not _keys_in_elements:
-            raise ValueError(f"Missing legacy elements, elements containing 'key' were expected for this XML with version {version}.")   # noqa: E501
+        models_in_elements = {i['model'] for i in elements.values()}
+        if models_in_elements <= ELEMENTS_USING_KEY:
+            # xml contains only domain.attribute or is empty
+            return
+        # inspect the elements for missing 'key' fields
+        elements_to_inspect = filter(lambda x: x['model'] not in ELEMENTS_USING_KEY, elements.values())
+        inspected_elements_containing_key = list(filter(lambda x: 'key' in x, elements_to_inspect))
+        if not inspected_elements_containing_key:
+            raise ValueError(f"Missing legacy elements, elements containing 'key' were expected for this XML with version {version} and elements {models_in_elements}.")   # noqa: E501
 
 
 def convert_legacy_elements(elements):
diff --git a/rdmo/management/tests/helpers_xml.py b/rdmo/management/tests/helpers_xml.py
index e594549550..dc2c9ec19d 100644
--- a/rdmo/management/tests/helpers_xml.py
+++ b/rdmo/management/tests/helpers_xml.py
@@ -13,6 +13,6 @@ def read_xml_and_parse_to_elements(xml_file):
 
     xml_parser = XmlToElementsParser(file_name=xml_file)
     if xml_parser.errors:
-        _msg = "\n".join(xml_parser.errors)
+        _msg = "\n".join(map(str, xml_parser.errors))
         raise ValueError(f"This test function should NOT raise any Exceptions. {_msg!s}")
     return xml_parser.parsed_elements, xml_parser.root

From a10cb49a1c65caad0644b578aa70b08f5a653fef Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 4 Mar 2024 15:10:06 +0100
Subject: [PATCH 114/205] refactor: rename constant models

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/xml.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index 2b13cc736f..c23c52841b 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -14,7 +14,7 @@
 
 logger = logging.getLogger(__name__)
 
-models = {
+RDMO_MODELS = {
   'catalog': 'questions.catalog',
   'section': 'questions.section',
   'page': 'questions.page',
@@ -29,7 +29,7 @@
 }
 
 DEFAULT_RDMO_XML_VERSION = '1.11.0'
-ELEMENTS_USING_KEY = {'domain.attribute'}
+ELEMENTS_USING_KEY = {RDMO_MODELS['attribute']}
 
 
 @dataclass
@@ -145,7 +145,7 @@ def flat_xml_to_elements(root):
 
         element = {
             'uri': get_uri(node, ns_map),
-            'model': models[node.tag]
+            'model': RDMO_MODELS[node.tag]
         }
 
         for sub_node in node:
@@ -156,8 +156,8 @@ def flat_xml_to_elements(root):
                 element[tag] = {
                     'uri': sub_node.attrib[uri_attrib]
                 }
-                if sub_node.tag in models:
-                    element[tag]['model'] = models[sub_node.tag]
+                if sub_node.tag in RDMO_MODELS:
+                    element[tag]['model'] = RDMO_MODELS[sub_node.tag]
             elif 'lang' in sub_node.attrib:
                 # this node has the lang attribute!
                 element['{}_{}'.format(tag, sub_node.attrib['lang'])] = sub_node.text
@@ -168,8 +168,8 @@ def flat_xml_to_elements(root):
                     sub_element = {
                         'uri': sub_sub_node.attrib[uri_attrib]
                     }
-                    if sub_sub_node.tag in models:
-                        sub_element['model'] = models[sub_sub_node.tag]
+                    if sub_sub_node.tag in RDMO_MODELS:
+                        sub_element['model'] = RDMO_MODELS[sub_sub_node.tag]
                     if 'order' in sub_sub_node.attrib:
                         sub_element['order'] = sub_sub_node.attrib['order']
 

From 68023487ce3e38e5e9bb9790b457552fa11ae1d3 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 4 Mar 2024 15:11:18 +0100
Subject: [PATCH 115/205] chore: remove up serializer arg

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/conditions/imports.py |  2 --
 rdmo/domain/imports.py     |  2 --
 rdmo/options/imports.py    |  5 +----
 rdmo/questions/imports.py  | 12 ------------
 rdmo/tasks/imports.py      |  2 --
 rdmo/views/imports.py      |  2 --
 6 files changed, 1 insertion(+), 24 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index e83370efe1..c0eccaa4a2 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -1,7 +1,6 @@
 from rdmo.core.imports import ElementImportHelper
 
 from .models import Condition
-from .serializers.v1 import ConditionSerializer
 from .validators import ConditionLockedValidator, ConditionUniqueURIValidator
 
 import_helper_condition = ElementImportHelper(
@@ -9,6 +8,5 @@
     model_path="conditions.condition",
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
     foreign_fields=('source', 'target_option'),
-    serializer=ConditionSerializer,
     extra_fields=('relation', 'target_text')
 )
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 2ff10106db..989a28607e 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -5,7 +5,6 @@
 )
 
 from .models import Attribute
-from .serializers.v1 import BaseAttributeSerializer
 from .validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
 
 logger = logging.getLogger(__name__)
@@ -17,5 +16,4 @@
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     foreign_fields=('parent',),
     extra_fields=('path',),
-    serializer=BaseAttributeSerializer,
 )
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 5dcb6cd462..a6826beedb 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -1,7 +1,6 @@
 from rdmo.core.imports import ElementImportHelper, ThroughInstanceMapper
 
 from .models import Option, OptionSet
-from .serializers.v1 import OptionSerializer, OptionSetSerializer
 from .validators import (
     OptionLockedValidator,
     OptionSetLockedValidator,
@@ -13,7 +12,6 @@
     model = OptionSet,
     model_path = "options.optionset",
     validators = (OptionSetLockedValidator, OptionSetUniqueURIValidator),
-    serializer = OptionSetSerializer,
     extra_fields = ('order', 'provider_key'),
     m2m_instance_fields = ('conditions', ),
     m2m_through_instance_fields = [
@@ -30,8 +28,7 @@
     model = Option,
     model_path = "options.option",
     validators = (OptionLockedValidator, OptionUniqueURIValidator),
-    lang_fields = ('text','help','view_text'),
-    serializer = OptionSerializer,
+    lang_fields = ('text', 'help', 'view_text'),
     extra_fields = ('additional_input',),
     reverse_m2m_through_instance_fields = [
         ThroughInstanceMapper(
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 8022a4f023..7ae24ee27e 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -1,13 +1,6 @@
 from rdmo.core.imports import ElementImportHelper, ThroughInstanceMapper
 
 from .models import Catalog, Page, Question, QuestionSet, Section
-from .serializers.v1 import (
-    CatalogSerializer,
-    PageSerializer,
-    QuestionSerializer,
-    QuestionSetSerializer,
-    SectionSerializer,
-)
 from .validators import (
     CatalogLockedValidator,
     CatalogUniqueURIValidator,
@@ -26,7 +19,6 @@
     model_path="questions.catalog",
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
     lang_fields=('help', 'title'),
-    serializer = CatalogSerializer,
     extra_fields = ('order', 'available'),
     m2m_through_instance_fields=[
         ThroughInstanceMapper(
@@ -42,7 +34,6 @@
     model_path="questions.section",
     validators=(SectionLockedValidator, SectionUniqueURIValidator),
     lang_fields=('title',),
-    serializer = SectionSerializer,
     m2m_through_instance_fields=[
         ThroughInstanceMapper(
             field_name='pages', source_name='section',
@@ -63,7 +54,6 @@
     validators=(PageLockedValidator, PageUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
-    serializer = PageSerializer,
     extra_fields = ('is_collection',),
     m2m_instance_fields = ('conditions', ),
     m2m_through_instance_fields=[
@@ -90,7 +80,6 @@
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
-    serializer = QuestionSetSerializer,
     extra_fields = ('is_collection',),
     m2m_instance_fields=('conditions', ),
 
@@ -122,7 +111,6 @@
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
     lang_fields=('text', 'help', 'default_text', 'verbose_name'),
     foreign_fields=('attribute', 'default_option'),
-    serializer=QuestionSerializer,
     extra_fields=('is_collection', 'is_optional', 'default_external_id',
                   'widget_type', 'value_type', 'maximum', 'minimum', 'step',
                   'unit', 'width'),
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 6508a2b15e..ec33a31c49 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -1,7 +1,6 @@
 from rdmo.core.imports import ElementImportHelper
 
 from .models import Task
-from .serializers.v1 import TaskSerializer
 from .validators import TaskLockedValidator, TaskUniqueURIValidator
 
 import_helper_task = ElementImportHelper(
@@ -10,7 +9,6 @@
     validators=(TaskLockedValidator, TaskUniqueURIValidator),
     lang_fields=('title', 'text'),
     foreign_fields=('start_attribute', 'end_attribute'),
-    serializer=TaskSerializer,
     extra_fields=('order', 'days_before', 'days_after', 'available'),
     m2m_instance_fields=('catalogs', 'conditions'),
 )
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index fa46b92270..084441f344 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -1,7 +1,6 @@
 from rdmo.core.imports import ElementImportHelper
 
 from .models import View
-from .serializers.v1 import ViewSerializer
 from .validators import ViewLockedValidator, ViewUniqueURIValidator
 
 import_helper_view = ElementImportHelper(
@@ -9,7 +8,6 @@
     model_path="views.view",
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
     lang_fields=('help', 'title'),
-    serializer=ViewSerializer,
     extra_fields=('order', 'template', 'available'),
     m2m_instance_fields=('catalogs',),
 )

From d1e38cac8d86e877e5d46a48dfaf65a96d5de9f9 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 5 Mar 2024 17:49:23 +0100
Subject: [PATCH 116/205] chore: refactor track_changes funcs and fix diff for
 m2m through order

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 122 +++++++++++++++++++++++++------------------
 1 file changed, 72 insertions(+), 50 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 4a74b3f743..9caee75926 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -22,6 +22,9 @@
 
 logger = logging.getLogger(__name__)
 
+ELEMENT_DIFF_FIELD_NAME = "updated_and_changed"
+NEW_DATA_FIELD = "new_data"
+CURRENT_DATA_FIELD = "current_data"
 
 def handle_uploaded_file(filedata):
     tempfilename = generate_tempfile_name()
@@ -66,32 +69,55 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=No
     return f"{verbose_name} {uri} updated"
 
 
-def track_messages_on_element(element: dict, element_field: str,
-                              warning: Optional[str]=None, error: Optional[str]=None):
-    if element['updated_and_changed'].get(element_field) is None:
-        element['updated_and_changed'][element_field] = {}
-        element['updated_and_changed'][element_field]['errors'] = []
-        element['updated_and_changed'][element_field]['warnings'] = defaultdict(list)
+def _initialize_tracking_field(element: dict, element_field: str):
+    if element[ELEMENT_DIFF_FIELD_NAME].get(element_field) is None:
+        element[ELEMENT_DIFF_FIELD_NAME][element_field] = {
+            'errors': [],
+            'warnings': defaultdict(list)
+        }
 
+
+def _append_warning(element: dict, element_field: str, warning: str):
+    element[ELEMENT_DIFF_FIELD_NAME][element_field]['warnings'][element['uri']].append(warning)
+
+
+def _append_error(element: dict, element_field: str, error: str):
+    element[ELEMENT_DIFF_FIELD_NAME][element_field]['errors'].append(error)
+
+
+def track_messages_on_element(element: dict, element_field: str, warning: Optional[str] = None,
+                              error: Optional[str] = None):
     if warning is not None:
-        if 'warning' not in element['updated_and_changed'][element_field]:
-            element['updated_and_changed'][element_field]['warnings'] = defaultdict(list)
-        element['updated_and_changed'][element_field]['warnings'][element['uri']].append(warning)
+        _initialize_tracking_field(element, element_field)
+        _append_warning(element, element_field, warning)
     if error is not None:
-        if 'error' not in element['updated_and_changed'][element_field]:
-            element['updated_and_changed'][element_field]['errors'] = []
-        element['updated_and_changed'][element_field]['errors'].append(error)
+        _initialize_tracking_field(element, element_field)
+        _append_error(element, element_field, error)
+
+def _initialize_track_changes_element_field(element: dict, element_field: str):
+    if ELEMENT_DIFF_FIELD_NAME not in element:
+        element[ELEMENT_DIFF_FIELD_NAME] = {}
+
+    if element_field and element_field not in element[ELEMENT_DIFF_FIELD_NAME]:
+        element[ELEMENT_DIFF_FIELD_NAME][element_field] = {}
 
+def _cast_list_of_string_to_list(list_of_strings: List[str]) -> str:
+    return "\n".join(map(str, list_of_strings))
 
 def track_changes_on_element(element: dict,
                              element_field: str,
-                             new_value: Union[str,List[str]],
+                             new_value: Union[str, List[str], None] = None,
                              instance_field: Optional[str] = None,
-                             original = None,
+                             original=None,
                              original_value: Optional[Union[str, List[str]]] = None):
     if original is None and original_value is None:
         return
+    if new_value is None:
+        return
+
+    _initialize_track_changes_element_field(element, element_field)
 
+    # optional js prop for react-diff-viewer-continued
     js_diff_viewer_props = {}
 
     _get_field = element_field if instance_field is None else instance_field
@@ -100,8 +126,8 @@ def track_changes_on_element(element: dict,
 
     if isinstance(new_value, list) and isinstance(original_value, list):
         # cast a list of strings with uris to a string with newlines
-        new_value = "\n".join(new_value)
-        original_value = "\n".join(original_value)
+        new_value = _cast_list_of_string_to_list(new_value)
+        original_value = _cast_list_of_string_to_list(original_value)
         js_diff_viewer_props['hideLineNumbers'] = False
         js_diff_viewer_props['splitView'] = False
 
@@ -112,15 +138,10 @@ def track_changes_on_element(element: dict,
     dmp.diff_cleanupSemantic(diff)
     changed: bool = any(i[0] != dmp.DIFF_EQUAL for i in diff)
     #  TODO maybe rename updated to new
-    changes = {'current': original_value,
-               'updated': new_value,
-               'changed': changed
-               }
-    changes.update(js_diff_viewer_props)
-    if element['updated_and_changed'].get(element_field) is None:
-        element['updated_and_changed'][element_field] = changes
-    else:
-        element['updated_and_changed'][element_field].update(changes)
+    element[ELEMENT_DIFF_FIELD_NAME][element_field]['current'] = original_value
+    element[ELEMENT_DIFF_FIELD_NAME][element_field]['updated'] = new_value
+    element[ELEMENT_DIFF_FIELD_NAME][element_field]['changed'] = changed
+    element[ELEMENT_DIFF_FIELD_NAME][element_field].update(js_diff_viewer_props)
 
 
 @dataclass(frozen=True)
@@ -135,7 +156,6 @@ class ElementImportHelper:
     model: Optional[models.Model] = field(default=None)
     model_path: Optional[str] = field(default=None)
     validators: Iterable[Callable] = field(default_factory=list)
-    serializer: Optional[Callable] = field(default=None)
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
     lang_fields: Sequence[str] = field(default_factory=list)
     foreign_fields: Sequence[str] = field(default_factory=list)
@@ -177,7 +197,7 @@ def set_lang_field(instance, field_name, element, original=None):
         element_field = lang_fields_value['element_key']
 
         track_changes_on_element(element, element_field,
-                                 field_value,
+                                 new_value=field_value,
                                  instance_field=field_lang_name,
                                  original=original)
         setattr(instance, field_lang_name, field_value)
@@ -190,7 +210,7 @@ def track_changes_on_uri_of_foreign_field(element, field_name, foreign_uri, orig
     original_foreign_uri = ''
     if original_foreign_instance:
         original_foreign_uri = getattr(original_foreign_instance, 'uri', '')
-    track_changes_on_element(element, field_name, foreign_uri, original_value=original_foreign_uri)
+    track_changes_on_element(element, field_name, new_value=foreign_uri, original_value=original_foreign_uri)
 
 def set_foreign_field(instance, field_name, element, uploaded_uris=None, original=None) -> None:
     if field_name not in element:
@@ -277,7 +297,8 @@ def set_extra_field(instance, field_name, element, questions_widget_types=None,
 
     setattr(instance, field_name, extra_value)
     # track changes
-    track_changes_on_element(element, field_name, extra_value, original=original)
+    track_changes_on_element(element, field_name, new_value=extra_value, original=original)
+
 
 def track_changes_m2m_instances(element, field_name,
                                 foreign_instances, original=None):
@@ -288,7 +309,7 @@ def track_changes_m2m_instances(element, field_name,
         return
     original_m2m_uris = list(original_m2m_instance.values_list('uri', flat=True))
     foreign_uris = [i.uri for i in foreign_instances]
-    track_changes_on_element(element, field_name, foreign_uris,
+    track_changes_on_element(element, field_name, new_value=foreign_uris,
                              original_value=original_m2m_uris)
 
 
@@ -310,15 +331,15 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
     through_instances = list(getattr(instance, through_name).all())
 
     _track_changes = {}
-    _track_changes['new_data'] = []
-    _track_changes['current_data'] = []
+    _track_changes[NEW_DATA_FIELD] = []
+    _track_changes[CURRENT_DATA_FIELD] = []
 
 
     if original is not None:
         try:
-            for _order, _through_instance in enumerate(getattr(original, field_name).all()):
-                _track_changes['current_data'].append({
-                    'uri': _through_instance.uri,
+            for _order, orig_field_instance in enumerate(getattr(original, through_name).order_by()):
+                _track_changes[CURRENT_DATA_FIELD].append({
+                    'uri': orig_field_instance.uri,
                     'order': _order,
                     'model': target_name
                 })
@@ -345,7 +366,7 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
                     # remove the through_instance from the through_instances list so that it won't get removed
                     through_instances.remove(through_instance)
                 if original is not None:
-                    _track_changes['new_data'].append(target_element)
+                    _track_changes[NEW_DATA_FIELD].append(target_element)
             except StopIteration:
                 # create a new item
                 if save:
@@ -355,7 +376,7 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
                         'order': order
                     }).save()
                 if original is not None:
-                    _track_changes['new_data'].append(target_element)
+                    _track_changes[NEW_DATA_FIELD].append(target_element)
 
         except target_model.DoesNotExist:
             message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} does not exist.'.format(
@@ -372,17 +393,18 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
         for through_instance in through_instances:
             if getattr(through_instance, target_name).uri_prefix == instance.uri_prefix:
                 through_instance.delete()
-    new_instance_data = sorted(_track_changes['new_data'], key=lambda k: k['order'])
-    original_instance_data = sorted(_track_changes['current_data'], key=lambda k: k['order'])
+    # sort the tracked changes by order in-place
+    new_instance_data = sorted(_track_changes[NEW_DATA_FIELD], key=lambda k: k['order'])
+    original_instance_data = sorted(_track_changes[CURRENT_DATA_FIELD], key=lambda k: k['order'])
     track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data)
 
 def track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data):
-    # sort the tracked changes by order in-place
-    _store = {'new_data': new_instance_data,  'current_data': original_instance_data}
-    element['updated_and_changed'][field_name] = _store
+    _initialize_track_changes_element_field(element, field_name)
+    element[ELEMENT_DIFF_FIELD_NAME][field_name][NEW_DATA_FIELD] = new_instance_data
+    element[ELEMENT_DIFF_FIELD_NAME][field_name][CURRENT_DATA_FIELD] = original_instance_data
     new_values = [i['uri'] for i in new_instance_data]
     original_values = [i['uri'] for i in original_instance_data]
-    track_changes_on_element(element, field_name, new_values, original_value=original_values)
+    track_changes_on_element(element, field_name, new_value=new_values, original_value=original_values)
 
 
 def set_m2m_instances(instance, element, field_name, original=None, save=None):
@@ -442,12 +464,12 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
     through_info = model_meta.get_field_info(through_model)
     target_model = through_info.forward_relations[target_name].related_model
     _track_changes = {}
-    _track_changes['new_data'] = []
-    _track_changes['current_data'] = []
+    _track_changes[NEW_DATA_FIELD] = []
+    _track_changes[CURRENT_DATA_FIELD] = []
     if original is not None:
         try:
-            for _order, _through_instance in enumerate(getattr(original, field_name).all()):
-                _track_changes['current_data'].append({
+            for _order, _through_instance in enumerate(getattr(original, through_name).order_by()):
+                _track_changes[CURRENT_DATA_FIELD].append({
                     'uri': _through_instance.uri,
                     'order': _order,
                     'model': target_name
@@ -479,7 +501,7 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
                 through_instance.order = order
                 through_instance.save()
             if original is not None:
-                _track_changes['new_data'].append(target_element)
+                _track_changes[NEW_DATA_FIELD].append(target_element)
 
         except target_model.DoesNotExist:
             message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} does not exist.'.format(
@@ -492,8 +514,8 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
             element['warnings'][target_uri].append(message)
             track_messages_on_element(element, field_name, warning=message)
     # sort the tracked changes by order in-place
-    new_instance_data = sorted(_track_changes['new_data'], key=lambda k: k['order'])
-    original_instance_data = sorted(_track_changes['current_data'], key=lambda k: k['order'])
+    new_instance_data = sorted(_track_changes[NEW_DATA_FIELD], key=lambda k: k['order'])
+    original_instance_data = sorted(_track_changes[CURRENT_DATA_FIELD], key=lambda k: k['order'])
     track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data)
 
 

From 291eecb6840598a77ffaa85413cb13d6e39c4bf5 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 5 Mar 2024 17:51:13 +0100
Subject: [PATCH 117/205] tests: add test for update options in optionset

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/test_import_options.py  |  43 +
 .../updated-and-changed/optionsets-1.json     | 933 ++++++++++++++++++
 .../updated-and-changed/optionsets-1.xml      |   1 -
 3 files changed, 976 insertions(+), 1 deletion(-)
 create mode 100644 testing/xml/elements/updated-and-changed/optionsets-1.json

diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 8ca1d0bfdd..92a60a8f9f 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -1,3 +1,4 @@
+import json
 from pathlib import Path
 
 import pytest
@@ -62,6 +63,46 @@ def test_update_optionsets_with_changed_fields(db, settings, updated_fields):
         assert test['updated_and_changed'] == imported['updated_and_changed']
 
 
+def test_update_optionsets_from_changed_xml(db, settings):
+    # Arrange, start test with fresh options in db
+    delete_all_objects([OptionSet, Option])
+
+    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
+    elements, root = read_xml_and_parse_to_elements(xml_file)
+    imported_elements = import_elements(elements)
+    assert len(root) == len(imported_elements) == 13
+    # Act, import from xml that has changes
+    xml_file_1 = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'updated-and-changed' / 'optionsets-1.xml'
+    elements_1, root_1 = read_xml_and_parse_to_elements(xml_file_1)
+    imported_elements_1 = import_elements(elements_1, save=False)
+
+    # Assert, compare with pre-defined json
+    json_file_1 = xml_file_1.with_suffix(".json")
+    _from_json_imported_elements_1 = json.loads(json_file_1.read_text())
+
+    # assert imported_elements_1 == from_json_imported_elements_1
+    changed_elements = [i for i in imported_elements_1 if i['changed']]
+    warnings_elements = [i for i in imported_elements_1 if i['warnings']]
+    assert len(changed_elements) == 5
+    assert len(warnings_elements) == 1
+
+    optionset_uri = "http://example.com/terms/options/one_two_three"
+    changed_uris = {i['uri']: i for i in changed_elements}
+    assert optionset_uri in changed_uris
+
+    # now save the elements_1
+    _imported_elements_1_save = import_elements(elements_1, save=True)
+    test_ordered_options = [
+        'http://example.com/terms/options/one_two_three/three',
+        'http://example.com/terms/options/one_two_three/two',
+        'http://example.com/terms/options/one_two_three/one',
+    ]
+    # get the ordered options (via .optionset_options) for this optionset from the db
+    optionset_1 = OptionSet.objects.get(uri=optionset_uri)
+    optionset_1_options = optionset_1.optionset_options.order_by('order').values_list('option__uri',flat=True)
+    for _test, _db in zip(test_ordered_options, optionset_1_options):
+        assert _test == _db
+
 def test_create_options(db, settings):
     Option.objects.all().delete()
 
@@ -108,6 +149,8 @@ def test_update_options_with_changed_fields(db, settings, updated_fields):
         assert test['updated_and_changed'] == imported['updated_and_changed']
 
 
+
+
 def test_create_legacy_options(db, settings):
     delete_all_objects([OptionSet, Option])
 
diff --git a/testing/xml/elements/updated-and-changed/optionsets-1.json b/testing/xml/elements/updated-and-changed/optionsets-1.json
new file mode 100644
index 0000000000..02f0a42029
--- /dev/null
+++ b/testing/xml/elements/updated-and-changed/optionsets-1.json
@@ -0,0 +1,933 @@
+[
+	{
+		"uri": "http://example.com/terms/options/condition/other",
+		"model": "options.option",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "condition/other",
+		"comment": null,
+		"text_en": "Other",
+		"help_en": null,
+		"text_de": "Sonstige",
+		"help_de": null,
+		"additional_input": "",
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": false,
+		"changed_fields": [],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "condition/other",
+				"updated": "condition/other",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"text_en": {
+				"current": "Other",
+				"updated": "Other",
+				"changed": false
+			},
+			"text_de": {
+				"current": "Sonstige",
+				"updated": "Sonstige",
+				"changed": false
+			},
+			"help_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"help_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/condition",
+		"model": "options.optionset",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "condition",
+		"comment": null,
+		"provider_key": null,
+		"options": [
+			{
+				"uri": "http://example.com/terms/options/condition/other",
+				"model": "options.option",
+				"order": "1"
+			}
+		],
+		"conditions": [
+			{
+				"uri": "http://example.com/terms/conditions/optionset_bool_is_false",
+				"model": "conditions.condition"
+			}
+		],
+		"warnings": {
+			"http://example.com/terms/conditions/optionset_bool_is_false": [
+				"Condition http://example.com/terms/conditions/optionset_bool_is_false for OptionSet http://example.com/terms/options/condition does not exist."
+			]
+		},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": true,
+		"changed_fields": [
+			"conditions"
+		],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "condition",
+				"updated": "condition",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"conditions": {
+				"errors": [],
+				"warnings": {
+					"http://example.com/terms/options/condition": [
+						"Condition http://example.com/terms/conditions/optionset_bool_is_false for OptionSet http://example.com/terms/options/condition does not exist."
+					]
+				},
+				"current": "http://example.com/terms/conditions/optionset_bool_is_true",
+				"updated": "",
+				"changed": true,
+				"hideLineNumbers": false,
+				"splitView": false
+			},
+			"options": {
+				"new_data": [
+					{
+						"uri": "http://example.com/terms/options/condition/other",
+						"model": "options.option",
+						"order": "1"
+					}
+				],
+				"current_data": [
+					{
+						"uri": "http://example.com/terms/options/condition/other",
+						"order": 0,
+						"model": "option"
+					}
+				],
+				"current": "http://example.com/terms/options/condition/other",
+				"updated": "http://example.com/terms/options/condition/other",
+				"changed": false,
+				"hideLineNumbers": false,
+				"splitView": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/one_two_three/three",
+		"model": "options.option",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "one_two_three/three",
+		"comment": null,
+		"text_en": "Three",
+		"help_en": null,
+		"text_de": "Drei",
+		"help_de": null,
+		"additional_input": "",
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": false,
+		"changed_fields": [],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "one_two_three/three",
+				"updated": "one_two_three/three",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"text_en": {
+				"current": "Three",
+				"updated": "Three",
+				"changed": false
+			},
+			"text_de": {
+				"current": "Drei",
+				"updated": "Drei",
+				"changed": false
+			},
+			"help_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"help_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/one_two_three/two",
+		"model": "options.option",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "one_two_three/two",
+		"comment": null,
+		"text_en": "Two",
+		"help_en": null,
+		"text_de": "Zwei",
+		"help_de": null,
+		"additional_input": "",
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": false,
+		"changed_fields": [],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "one_two_three/two",
+				"updated": "one_two_three/two",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"text_en": {
+				"current": "Two",
+				"updated": "Two",
+				"changed": false
+			},
+			"text_de": {
+				"current": "Zwei",
+				"updated": "Zwei",
+				"changed": false
+			},
+			"help_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"help_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/one_two_three/one",
+		"model": "options.option",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "one_two_three/one",
+		"comment": null,
+		"text_en": "One",
+		"help_en": null,
+		"text_de": "Eins",
+		"help_de": null,
+		"additional_input": "",
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": false,
+		"changed_fields": [],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "one_two_three/one",
+				"updated": "one_two_three/one",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"text_en": {
+				"current": "One",
+				"updated": "One",
+				"changed": false
+			},
+			"text_de": {
+				"current": "Eins",
+				"updated": "Eins",
+				"changed": false
+			},
+			"help_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"help_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/one_two_three",
+		"model": "options.optionset",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "one_two_three",
+		"comment": null,
+		"provider_key": null,
+		"options": [
+			{
+				"uri": "http://example.com/terms/options/one_two_three/three",
+				"model": "options.option",
+				"order": "1"
+			},
+			{
+				"uri": "http://example.com/terms/options/one_two_three/two",
+				"model": "options.option",
+				"order": "2"
+			},
+			{
+				"uri": "http://example.com/terms/options/one_two_three/one",
+				"model": "options.option",
+				"order": "3"
+			}
+		],
+		"conditions": null,
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": true,
+		"changed_fields": [
+			"options"
+		],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "one_two_three",
+				"updated": "one_two_three",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"options": {
+				"new_data": [
+					{
+						"uri": "http://example.com/terms/options/one_two_three/three",
+						"model": "options.option",
+						"order": "1"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three/two",
+						"model": "options.option",
+						"order": "2"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three/one",
+						"model": "options.option",
+						"order": "3"
+					}
+				],
+				"current_data": [
+					{
+						"uri": "http://example.com/terms/options/one_two_three/one",
+						"order": 0,
+						"model": "option"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three/two",
+						"order": 1,
+						"model": "option"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three/three",
+						"order": 2,
+						"model": "option"
+					}
+				],
+				"current": "http://example.com/terms/options/one_two_three/one\nhttp://example.com/terms/options/one_two_three/two\nhttp://example.com/terms/options/one_two_three/three",
+				"updated": "http://example.com/terms/options/one_two_three/three\nhttp://example.com/terms/options/one_two_three/two\nhttp://example.com/terms/options/one_two_three/one",
+				"changed": true,
+				"hideLineNumbers": false,
+				"splitView": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/one_two_three_other/textarea",
+		"model": "options.option",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "one_two_three_other/textarea",
+		"comment": null,
+		"text_en": "Text",
+		"help_en": null,
+		"text_de": "Text",
+		"help_de": null,
+		"additional_input": "",
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": false,
+		"changed_fields": [],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "one_two_three_other/textarea",
+				"updated": "one_two_three_other/textarea",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"text_en": {
+				"current": "Text",
+				"updated": "Text",
+				"changed": false
+			},
+			"text_de": {
+				"current": "Text",
+				"updated": "Text",
+				"changed": false
+			},
+			"help_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"help_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/one_two_three_other/text",
+		"model": "options.option",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "one_two_three_other/text",
+		"comment": null,
+		"text_en": "Text",
+		"help_en": null,
+		"text_de": "Text",
+		"help_de": null,
+		"additional_input": "",
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": false,
+		"changed_fields": [],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "one_two_three_other/text",
+				"updated": "one_two_three_other/text",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"text_en": {
+				"current": "Text",
+				"updated": "Text",
+				"changed": false
+			},
+			"text_de": {
+				"current": "Text",
+				"updated": "Text",
+				"changed": false
+			},
+			"help_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"help_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/one_two_three_other/three",
+		"model": "options.option",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "one_two_three_other/three",
+		"comment": null,
+		"text_en": "Three - 1",
+		"help_en": null,
+		"text_de": "Drei - 1",
+		"help_de": null,
+		"additional_input": "",
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": true,
+		"changed_fields": [
+			"text_en",
+			"text_de"
+		],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "one_two_three_other/three",
+				"updated": "one_two_three_other/three",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"text_en": {
+				"current": "Three",
+				"updated": "Three - 1",
+				"changed": true
+			},
+			"text_de": {
+				"current": "Drei",
+				"updated": "Drei - 1",
+				"changed": true
+			},
+			"help_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"help_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/one_two_three_other/two",
+		"model": "options.option",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "one_two_three_other/two",
+		"comment": null,
+		"text_en": "Two",
+		"help_en": null,
+		"text_de": "Zwei",
+		"help_de": null,
+		"additional_input": "",
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": false,
+		"changed_fields": [],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "one_two_three_other/two",
+				"updated": "one_two_three_other/two",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"text_en": {
+				"current": "Two",
+				"updated": "Two",
+				"changed": false
+			},
+			"text_de": {
+				"current": "Zwei",
+				"updated": "Zwei",
+				"changed": false
+			},
+			"help_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"help_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/one_two_three_other/one",
+		"model": "options.option",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "one_two_three_other/one",
+		"comment": null,
+		"text_en": "One - 3",
+		"help_en": null,
+		"text_de": "Eins - 3",
+		"help_de": null,
+		"additional_input": "",
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": true,
+		"changed_fields": [
+			"text_en",
+			"text_de"
+		],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "one_two_three_other/one",
+				"updated": "one_two_three_other/one",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"text_en": {
+				"current": "One",
+				"updated": "One - 3",
+				"changed": true
+			},
+			"text_de": {
+				"current": "Eins",
+				"updated": "Eins - 3",
+				"changed": true
+			},
+			"help_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"help_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_en": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"view_text_de": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/one_two_three_other",
+		"model": "options.optionset",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "one_two_three_other",
+		"comment": null,
+		"provider_key": null,
+		"options": [
+			{
+				"uri": "http://example.com/terms/options/one_two_three_other/textarea",
+				"model": "options.option",
+				"order": "1"
+			},
+			{
+				"uri": "http://example.com/terms/options/one_two_three_other/text",
+				"model": "options.option",
+				"order": "2"
+			},
+			{
+				"uri": "http://example.com/terms/options/one_two_three_other/three",
+				"model": "options.option",
+				"order": "3"
+			},
+			{
+				"uri": "http://example.com/terms/options/one_two_three_other/two",
+				"model": "options.option",
+				"order": "4"
+			},
+			{
+				"uri": "http://example.com/terms/options/one_two_three_other/one",
+				"model": "options.option",
+				"order": "5"
+			}
+		],
+		"conditions": null,
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": true,
+		"changed_fields": [
+			"options"
+		],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "one_two_three_other",
+				"updated": "one_two_three_other",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"options": {
+				"new_data": [
+					{
+						"uri": "http://example.com/terms/options/one_two_three_other/textarea",
+						"model": "options.option",
+						"order": "1"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three_other/text",
+						"model": "options.option",
+						"order": "2"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three_other/three",
+						"model": "options.option",
+						"order": "3"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three_other/two",
+						"model": "options.option",
+						"order": "4"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three_other/one",
+						"model": "options.option",
+						"order": "5"
+					}
+				],
+				"current_data": [
+					{
+						"uri": "http://example.com/terms/options/one_two_three_other/one",
+						"order": 0,
+						"model": "option"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three_other/two",
+						"order": 1,
+						"model": "option"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three_other/three",
+						"order": 2,
+						"model": "option"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three_other/text",
+						"order": 3,
+						"model": "option"
+					},
+					{
+						"uri": "http://example.com/terms/options/one_two_three_other/textarea",
+						"order": 4,
+						"model": "option"
+					}
+				],
+				"current": "http://example.com/terms/options/one_two_three_other/one\nhttp://example.com/terms/options/one_two_three_other/two\nhttp://example.com/terms/options/one_two_three_other/three\nhttp://example.com/terms/options/one_two_three_other/text\nhttp://example.com/terms/options/one_two_three_other/textarea",
+				"updated": "http://example.com/terms/options/one_two_three_other/textarea\nhttp://example.com/terms/options/one_two_three_other/text\nhttp://example.com/terms/options/one_two_three_other/three\nhttp://example.com/terms/options/one_two_three_other/two\nhttp://example.com/terms/options/one_two_three_other/one",
+				"changed": true,
+				"hideLineNumbers": false,
+				"splitView": false
+			}
+		}
+	},
+	{
+		"uri": "http://example.com/terms/options/plugin",
+		"model": "options.optionset",
+		"uri_prefix": "http://example.com/terms",
+		"uri_path": "plugin",
+		"comment": null,
+		"provider_key": "simple",
+		"options": null,
+		"conditions": null,
+		"warnings": {},
+		"errors": [],
+		"created": false,
+		"updated": true,
+		"changed": false,
+		"changed_fields": [],
+		"updated_and_changed": {
+			"uri_prefix": {
+				"current": "http://example.com/terms",
+				"updated": "http://example.com/terms",
+				"changed": false
+			},
+			"uri_path": {
+				"current": "plugin",
+				"updated": "plugin",
+				"changed": false
+			},
+			"comment": {
+				"current": "",
+				"updated": "",
+				"changed": false
+			},
+			"options": {
+				"new_data": [],
+				"current_data": [],
+				"current": "",
+				"updated": "",
+				"changed": false,
+				"hideLineNumbers": false,
+				"splitView": false
+			}
+		}
+	}
+]
diff --git a/testing/xml/elements/updated-and-changed/optionsets-1.xml b/testing/xml/elements/updated-and-changed/optionsets-1.xml
index e2349f919a..e4c6ff5174 100644
--- a/testing/xml/elements/updated-and-changed/optionsets-1.xml
+++ b/testing/xml/elements/updated-and-changed/optionsets-1.xml
@@ -31,7 +31,6 @@
 			<option dc:uri="http://example.com/terms/options/one_two_three/three" order="1"/>
 			<option dc:uri="http://example.com/terms/options/one_two_three/two" order="2"/>
 			<option dc:uri="http://example.com/terms/options/one_two_three/one" order="3"/>
-            <option dc:uri="http://example.com/terms/options/one_two_three/four" order="4"/>
 		</options>
 		<conditions/>
 	</optionset>

From 887948fe048d27cd1cb638c858457459297f8050 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 5 Mar 2024 17:52:03 +0100
Subject: [PATCH 118/205] refactor: use constant for diff field name

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/imports.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index e3bd9197e0..a0151f37b4 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -10,6 +10,7 @@
 
 from rdmo.conditions.imports import import_helper_condition
 from rdmo.core.imports import (
+    ELEMENT_DIFF_FIELD_NAME,
     check_permissions,
     get_or_return_instance,
     make_import_info_msg,
@@ -60,7 +61,7 @@
         'updated': bool,
         'changed': bool,
         'changed_fields': list,
-        'updated_and_changed': dict,
+        ELEMENT_DIFF_FIELD_NAME: dict,
     }
 
 
@@ -77,6 +78,10 @@ def import_elements(uploaded_elements: Dict, save: bool = True, request: Optiona
         imported_elements.append(element)
     return imported_elements
 
+def _initialize_import_element_dict(element: Dict) -> None:
+    # initialize element dict with default values
+    for _k,_val in IMPORT_ELEMENT_INIT_DICT.items():
+        element[_k] = _val()
 
 
 def import_element(
@@ -95,9 +100,7 @@ def import_element(
     if model_path is None:
         return element
 
-    # initialize element dict with default values
-    for _k,_val in IMPORT_ELEMENT_INIT_DICT.items():
-        element[_k] = _val()
+    _initialize_import_element_dict(element)
 
     user = request.user if request is not None else None
     import_helper = ELEMENT_IMPORT_HELPERS[model_path]
@@ -132,7 +135,7 @@ def import_element(
     _updated = not _created
     element['created'] = _created
     element['updated'] = _updated
-    # dict element['updated_and_changed'] is filled by tracking changes
+    # dict element[ELEMENT_DIFF_FIELD_NAME] is filled by tracking changes
 
     element = strip_uri_prefix_endswith_slash(element)
     # start to set values on the instance
@@ -142,7 +145,7 @@ def import_element(
         setattr(instance, common_field, common_value)
         if _updated and original:
             # track changes for common fields
-            track_changes_on_element(element, common_field, common_value, original=original)
+            track_changes_on_element(element, common_field, new_value=common_value, original=original)
     # set language fields
     for lang_field_name in lang_field_names:
         set_lang_field(instance, lang_field_name, element, original=original)
@@ -171,8 +174,8 @@ def import_element(
             set_reverse_m2m_through_instance(instance, element, **asdict(reverse_m2m_fields),
                                              original=original, save=save)
     # set aggregated changes potentially to True and a list of changed fields
-    if _updated and element['updated_and_changed']:
-        set_updated_and_changed_meta_info(element)
+    if _updated and element[ELEMENT_DIFF_FIELD_NAME]:
+        set_element_diff_field_meta_info(element)
 
     if save and settings.MULTISITE:
         # could be optimized with a bulk_create of through model later
@@ -183,6 +186,7 @@ def import_element(
 
     return element
 
+
 def strip_uri_prefix_endswith_slash(element: dict) -> dict:
     # handle URI Prefix ending with slash
     if 'uri_prefix' not in element:
@@ -191,7 +195,7 @@ def strip_uri_prefix_endswith_slash(element: dict) -> dict:
         element['uri_prefix'] = element['uri_prefix'].rstrip('/')
     return element
 
-def set_updated_and_changed_meta_info(element: dict) -> dict:
-    changed_fields = {k: val for k, val in element['updated_and_changed'].items() if val['changed']}
+def set_element_diff_field_meta_info(element: dict) -> None:
+    changed_fields = {k: val for k, val in element[ELEMENT_DIFF_FIELD_NAME].items() if val['changed']}
     element['changed'] = bool(changed_fields)
     element['changed_fields'] = list(changed_fields.keys())

From 9b62e67d03c2d4875332acec9ad9984334591c05 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 5 Mar 2024 17:52:54 +0100
Subject: [PATCH 119/205] tests: refactor helper function

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../management/tests/helpers_import_elements.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/rdmo/management/tests/helpers_import_elements.py b/rdmo/management/tests/helpers_import_elements.py
index cf12633eeb..721383299b 100644
--- a/rdmo/management/tests/helpers_import_elements.py
+++ b/rdmo/management/tests/helpers_import_elements.py
@@ -3,7 +3,7 @@
 from typing import Dict, List, Optional, Tuple
 
 from rdmo.core.imports import track_changes_on_element
-from rdmo.management.imports import set_updated_and_changed_meta_info
+from rdmo.management.imports import _initialize_import_element_dict, set_element_diff_field_meta_info
 
 UPDATE_FIELD_FUNCS = {
     'comment': lambda text: f"this is a test comment {text}",
@@ -23,22 +23,23 @@ def _test_helper_filter_updated_and_changed(elements: List[Dict], updated_fields
     changed_elements = filter(filter_func, elements)
     return list(changed_elements)
 
-def _test_helper_change_fields_elements(elements, fields_to_update: Optional[Tuple]=None, n=3) -> Tuple[Dict, List]:
+
+def _test_helper_change_fields_elements(elements,
+                                        fields_to_update: Optional[Tuple] = None,
+                                        n=3) -> OrderedDict:
     """ elements test preparation function """
 
     if len(elements) < n:
         raise ValueError("Length of elements should not be smaller than n.")
     _new_elements = OrderedDict()
-    for _n,(_uri, _element) in enumerate(elements.items()):
-        if _n <= n-1:
-            _element['updated_and_changed'] = {}
-            _element['changed'] = False
-            _element['changed_fields'] = []
+    for _n, (_uri, _element) in enumerate(elements.items()):
+        if _n <= n - 1:
+            _initialize_import_element_dict(_element)
             for field in fields_to_update:
                 original_value = _element[field] or ''
                 new_val = UPDATE_FIELD_FUNCS[field](_n)
                 track_changes_on_element(_element, field, new_val, original_value=original_value)
                 _element[field] = new_val
-            set_updated_and_changed_meta_info(_element)
+            set_element_diff_field_meta_info(_element)
         _new_elements[_uri] = _element
     return _new_elements

From 597cfcb8a3dd730add5dd9a206c9cb9db0d53c4f Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 6 Mar 2024 12:07:21 +0100
Subject: [PATCH 120/205] tests: fix frontend import test
 test_import_in_management

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/test_frontend_import.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/rdmo/management/tests/test_frontend_import.py b/rdmo/management/tests/test_frontend_import.py
index 3c46a84ca2..e1508e5147 100644
--- a/rdmo/management/tests/test_frontend_import.py
+++ b/rdmo/management/tests/test_frontend_import.py
@@ -30,7 +30,7 @@ def test_import_in_management(logged_in_user: Page) -> None:
 
     page = logged_in_user
     expect(page.get_by_role("heading", name="Management")).to_be_visible()
-    page.screenshot(path="screenshots/management-import-before-initial.png", full_page=True)
+    expect(page.locator("strong").filter(has_text="Catalogs")).to_be_visible()
     # choose the file to be imported
     page.locator("input[name=\"uploaded_file\"]").set_input_files("./testing/xml/elements/catalogs.xml")
     # click the import form submit button, this will take some time
@@ -45,15 +45,18 @@ def test_import_in_management(logged_in_user: Page) -> None:
     page.get_by_role("link", name="Show all").click()
     rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")
     # there are 2 rows per object displayed
-    expect(rows_displayed_in_ui).to_have_count(148*2)
+    expect(rows_displayed_in_ui).to_have_count(148 * 2)
     page.get_by_role("link", name="Hide all").click()
-    page.screenshot(path="screenshots/management-import-pre-hide-all.png", full_page=True)
+    expect(rows_displayed_in_ui).to_have_count(0)
+    page.screenshot(path="screenshots/management-import-pre-import.png", full_page=True)
     ## TODO test show changed elements
+    ## TODO test for filter changed elements
     ## TODO test select changed elements
     ## TODO test for warnings, errors
 
     # click the import button to start saving the instances to the db
     page.get_by_role("button", name="Import 148 elements").click()
+    expect(page.get_by_role("heading", name="Import successful")).to_be_visible()
     page.screenshot(path="screenshots/management-import-post-import.png", full_page=True)
     page.get_by_text("Created:").click()
     # go back to management page

From cd1213fcd05d64c7d8f6deed4b436433aac22138 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 6 Mar 2024 18:06:27 +0100
Subject: [PATCH 121/205] tests e2e: add test for optionsets import and rename

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../tests/test_frontend_import_options.py     | 76 +++++++++++++++++++
 ...t.py => test_frontend_import_questions.py} |  9 +--
 2 files changed, 78 insertions(+), 7 deletions(-)
 create mode 100644 rdmo/management/tests/test_frontend_import_options.py
 rename rdmo/management/tests/{test_frontend_import.py => test_frontend_import_questions.py} (90%)

diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
new file mode 100644
index 0000000000..c675a1f7b9
--- /dev/null
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -0,0 +1,76 @@
+# ruff: noqa: F811
+import os
+
+import pytest
+
+from playwright.sync_api import Page, expect
+
+from rdmo.options.models import Option, OptionSet
+
+from .fixtures_frontend import (
+    base_url_page,  # noqa: F401
+    e2e_tests_django_db_setup,  # noqa: F401
+    logged_in_user,  # noqa: F401
+)
+from .helpers_models import delete_all_objects
+
+pytestmark = pytest.mark.e2e
+
+# needed for playwright to run
+os.environ.setdefault("DJANGO_ALLOW_ASYNC_UNSAFE", "true")
+
+test_users = [('editor', 'editor')]
+import_xml = "./testing/xml/elements/optionsets.xml"
+import_xml_1 = "./testing/xml/elements/updated-and-changed/optionsets-1.xml"
+
+@pytest.mark.parametrize("username, password", test_users)  # consumed by fixture
+def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> None:
+    """Test that each content type is available through the navigation."""
+    delete_all_objects([OptionSet, Option])
+
+    page = logged_in_user
+    expect(page.get_by_role("heading", name="Management")).to_be_visible()
+    expect(page.locator("strong").filter(has_text="Catalogs")).to_be_visible()
+    ## 1. Import fresh optionset.xml
+    # choose the file to be imported
+    page.locator("input[name=\"uploaded_file\"]").set_input_files(import_xml)
+    # click the import form submit button, this will take some time
+    page.locator('#sidebar div.elements-sidebar form.upload-form.sidebar-form div.sidebar-form-button button.btn.btn-primary').click()  # noqa: E501
+    # wait for import to be finished with timeout 30s
+    expect(page.get_by_text("Import from: optionsets.xml")).to_be_visible(timeout=30_000)
+    ## TODO test if ImportInfo numbers are correct
+    # test the components of the import-before-import staging page
+    expect(page.get_by_text("Created: 13")).to_be_visible(timeout=30_000)
+    page.locator(".element-link").first.click()
+    page.get_by_role("link", name="Unselect all").click()
+    page.get_by_role("link", name="Select all", exact=True).click()
+    page.get_by_role("link", name="Show all", exact=True).click()
+    rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")
+    # there are 2 rows per object displayed
+    expect(rows_displayed_in_ui).to_have_count(13 * 2)
+
+    # click the import button to start saving the instances to the db
+    page.get_by_role("button", name="Import 13 elements").click()
+    expect(page.get_by_role("heading", name="Import successful")).to_be_visible()
+    page.screenshot(path="screenshots/management-import-post-import.png", full_page=True)
+    page.get_by_text("Created:").click()
+    # go back to management page
+    page.get_by_role("button", name="Back").click()
+    expect(page.get_by_role("heading", name="Management")).to_be_visible()
+    # assert all Model objects in db
+    assert OptionSet.objects.count() == 4
+    assert Option.objects.count() == 9
+
+    ## 2. import optionset-1.xml with changes
+    # choose the file to be imported
+    page.locator("input[name=\"uploaded_file\"]").set_input_files(import_xml_1)
+    # click the import form submit button, this will take some time
+    page.locator('#sidebar div.elements-sidebar form.upload-form.sidebar-form div.sidebar-form-button button.btn.btn-primary').click()  # noqa: E501
+    expect(page.get_by_text("Import from: optionsets-1.xml")).to_be_visible(timeout=40_000)
+    # assert changed elements
+    expect(page.get_by_text("Total: 13 Updated: 13 (Changed: 5) Warnings: 1")).to_be_visible(timeout=30_000)
+    expect(page.get_by_text("Filter changed (5)")).to_be_visible()
+    page.get_by_text("Filter changed (5)").click()
+    page.get_by_role("link", name="Show changes").click()
+    expect(page.get_by_text("http://example.com/terms/options/one_two_three/three").nth(1)).to_be_visible()
+    ## TODO test for warnings, errors
diff --git a/rdmo/management/tests/test_frontend_import.py b/rdmo/management/tests/test_frontend_import_questions.py
similarity index 90%
rename from rdmo/management/tests/test_frontend_import.py
rename to rdmo/management/tests/test_frontend_import_questions.py
index e1508e5147..4790a52933 100644
--- a/rdmo/management/tests/test_frontend_import.py
+++ b/rdmo/management/tests/test_frontend_import_questions.py
@@ -24,8 +24,8 @@
 test_users = [('editor', 'editor')]
 
 @pytest.mark.parametrize("username, password", test_users)  # consumed by fixture
-def test_import_in_management(logged_in_user: Page) -> None:
-    """Test that each content type is available through the navigation."""
+def test_import_catalogs_in_management(logged_in_user: Page) -> None:
+    """Test that the catalogs.xml can be imported correctly."""
     delete_all_objects([Catalog, Section, PageModel, QuestionSet, Question])
 
     page = logged_in_user
@@ -49,11 +49,6 @@ def test_import_in_management(logged_in_user: Page) -> None:
     page.get_by_role("link", name="Hide all").click()
     expect(rows_displayed_in_ui).to_have_count(0)
     page.screenshot(path="screenshots/management-import-pre-import.png", full_page=True)
-    ## TODO test show changed elements
-    ## TODO test for filter changed elements
-    ## TODO test select changed elements
-    ## TODO test for warnings, errors
-
     # click the import button to start saving the instances to the db
     page.get_by_role("button", name="Import 148 elements").click()
     expect(page.get_by_role("heading", name="Import successful")).to_be_visible()

From abc89d04944d0d677e9c11a72dbf54bd431abfc4 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 6 Mar 2024 18:07:30 +0100
Subject: [PATCH 122/205] refactor: move RDMO_MODELS to constants.py

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/constants.py | 14 ++++++++++++++
 rdmo/core/xml.py       | 15 +--------------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/rdmo/core/constants.py b/rdmo/core/constants.py
index 11451a838a..8c42a1b42e 100644
--- a/rdmo/core/constants.py
+++ b/rdmo/core/constants.py
@@ -99,3 +99,17 @@
     'unit': '',
     'widget_type': 'text',
 }
+
+RDMO_MODELS = {
+  'catalog': 'questions.catalog',
+  'section': 'questions.section',
+  'page': 'questions.page',
+  'questionset': 'questions.questionset',
+  'question': 'questions.question',
+  'attribute': 'domain.attribute',
+  'optionset': 'options.optionset',
+  'option': 'options.option',
+  'condition': 'conditions.condition',
+  'task': 'tasks.task',
+  'view': 'views.view'
+}
diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index c23c52841b..b522f488e7 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -11,23 +11,10 @@
 from packaging.version import Version, parse
 
 from rdmo import __version__ as VERSION
+from rdmo.core.constants import RDMO_MODELS
 
 logger = logging.getLogger(__name__)
 
-RDMO_MODELS = {
-  'catalog': 'questions.catalog',
-  'section': 'questions.section',
-  'page': 'questions.page',
-  'questionset': 'questions.questionset',
-  'question': 'questions.question',
-  'attribute': 'domain.attribute',
-  'optionset': 'options.optionset',
-  'option': 'options.option',
-  'condition': 'conditions.condition',
-  'task': 'tasks.task',
-  'view': 'views.view'
-}
-
 DEFAULT_RDMO_XML_VERSION = '1.11.0'
 ELEMENTS_USING_KEY = {RDMO_MODELS['attribute']}
 

From a3a2fa9780f783999bc1931118fa298a44913f2c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 6 Mar 2024 18:09:15 +0100
Subject: [PATCH 123/205] fix: get the correct original instance data for
 through models

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 9caee75926..4ac7cd7278 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -17,7 +17,7 @@
 
 from diff_match_patch import diff_match_patch
 
-from rdmo.core.constants import ELEMENT_COMMON_FIELDS, ELEMENT_IMPORT_EXTRA_FIELDS_DEFAULTS
+from rdmo.core.constants import ELEMENT_COMMON_FIELDS, ELEMENT_IMPORT_EXTRA_FIELDS_DEFAULTS, RDMO_MODELS
 from rdmo.core.utils import get_languages
 
 logger = logging.getLogger(__name__)
@@ -52,13 +52,22 @@ def generate_tempfile_name():
     return fn
 
 
-def get_or_return_instance(model: Callable, uri: Optional[str]=None) -> Tuple[models.Model, bool]:
+def get_or_return_instance(model: models.Model, uri: Optional[str]=None) -> Tuple[models.Model, bool]:
     if uri is None:
         return model(), True
     try:
         return model.objects.get(uri=uri), False
     except model.DoesNotExist:
         return model(), True
+    except model.MultipleObjectsReturned:
+        return model.objects.filter(uri=uri).first(), False
+
+def get_rdmo_model_path(target_name:str, field_name: str):
+    try:
+        return RDMO_MODELS[target_name]
+    except KeyError:
+        if 'parent' in target_name and 'questionset' in field_name:
+            return RDMO_MODELS[field_name]
 
 
 def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=None):
@@ -339,9 +348,9 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
         try:
             for _order, orig_field_instance in enumerate(getattr(original, through_name).order_by()):
                 _track_changes[CURRENT_DATA_FIELD].append({
-                    'uri': orig_field_instance.uri,
-                    'order': _order,
-                    'model': target_name
+                    'uri': getattr(orig_field_instance, target_name).uri,
+                    'order': orig_field_instance.order,
+                    'model': get_rdmo_model_path(target_name, field_name)
                 })
         except AttributeError:
             pass  # legacy elements miss the field_name
@@ -463,6 +472,7 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
     through_model = model_info.reverse_relations[through_name].related_model
     through_info = model_meta.get_field_info(through_model)
     target_model = through_info.forward_relations[target_name].related_model
+
     _track_changes = {}
     _track_changes[NEW_DATA_FIELD] = []
     _track_changes[CURRENT_DATA_FIELD] = []
@@ -470,9 +480,9 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
         try:
             for _order, _through_instance in enumerate(getattr(original, through_name).order_by()):
                 _track_changes[CURRENT_DATA_FIELD].append({
-                    'uri': _through_instance.uri,
-                    'order': _order,
-                    'model': target_name
+                    'uri': getattr(_through_instance, source_name).uri,
+                    'order': _through_instance.order,
+                    'model': get_rdmo_model_path(target_name, field_name),
                 })
         except AttributeError:
             pass  # legacy elements miss the field_name

From 316474195171400b9fbe179a088ed67885ee54d8 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 6 Mar 2024 18:10:56 +0100
Subject: [PATCH 124/205] tests: fix test_update_optionsets_from_changed_xml
 and add assertions

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/test_import_options.py  |  34 +-
 .../updated-and-changed/optionsets-1.json     | 933 ------------------
 2 files changed, 22 insertions(+), 945 deletions(-)
 delete mode 100644 testing/xml/elements/updated-and-changed/optionsets-1.json

diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 92a60a8f9f..7724a2281d 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -1,8 +1,8 @@
-import json
 from pathlib import Path
 
 import pytest
 
+from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
 from rdmo.management.imports import import_elements
 from rdmo.options.models import Option, OptionSet
 
@@ -76,33 +76,43 @@ def test_update_optionsets_from_changed_xml(db, settings):
     elements_1, root_1 = read_xml_and_parse_to_elements(xml_file_1)
     imported_elements_1 = import_elements(elements_1, save=False)
 
-    # Assert, compare with pre-defined json
-    json_file_1 = xml_file_1.with_suffix(".json")
-    _from_json_imported_elements_1 = json.loads(json_file_1.read_text())
-
-    # assert imported_elements_1 == from_json_imported_elements_1
+    # Assert imported_elements_1
     changed_elements = [i for i in imported_elements_1 if i['changed']]
     warnings_elements = [i for i in imported_elements_1 if i['warnings']]
     assert len(changed_elements) == 5
     assert len(warnings_elements) == 1
 
     optionset_uri = "http://example.com/terms/options/one_two_three"
+    test_original_options = [
+        'http://example.com/terms/options/one_two_three/one',
+        'http://example.com/terms/options/one_two_three/two',
+        'http://example.com/terms/options/one_two_three/three',
+    ]
+    # change the order of the options, as in the xml
+    test_changed_options = test_original_options[::-1]
+    _original_value = "\n".join(test_original_options)
+    _new_value = "\n".join(test_changed_options)
     changed_uris = {i['uri']: i for i in changed_elements}
     assert optionset_uri in changed_uris
+    changed_element = changed_uris[optionset_uri]
+    assert "options" in changed_element['changed_fields']
+    assert changed_element[ELEMENT_DIFF_FIELD_NAME]['options']['current'] == _original_value
+    assert changed_element[ELEMENT_DIFF_FIELD_NAME]['options']['updated'] == _new_value
 
     # now save the elements_1
     _imported_elements_1_save = import_elements(elements_1, save=True)
-    test_ordered_options = [
-        'http://example.com/terms/options/one_two_three/three',
-        'http://example.com/terms/options/one_two_three/two',
-        'http://example.com/terms/options/one_two_three/one',
-    ]
     # get the ordered options (via .optionset_options) for this optionset from the db
     optionset_1 = OptionSet.objects.get(uri=optionset_uri)
     optionset_1_options = optionset_1.optionset_options.order_by('order').values_list('option__uri',flat=True)
-    for _test, _db in zip(test_ordered_options, optionset_1_options):
+    for _test, _db in zip(test_changed_options, optionset_1_options):
         assert _test == _db
 
+    # Import again and test that there are no changes detected
+    imported_elements_2 = import_elements(elements_1, save=False)
+    assert len([i for i in imported_elements_2 if i['changed']]) == 0
+    assert len([i for i in imported_elements_2 if i['warnings']]) == 1
+
+
 def test_create_options(db, settings):
     Option.objects.all().delete()
 
diff --git a/testing/xml/elements/updated-and-changed/optionsets-1.json b/testing/xml/elements/updated-and-changed/optionsets-1.json
deleted file mode 100644
index 02f0a42029..0000000000
--- a/testing/xml/elements/updated-and-changed/optionsets-1.json
+++ /dev/null
@@ -1,933 +0,0 @@
-[
-	{
-		"uri": "http://example.com/terms/options/condition/other",
-		"model": "options.option",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "condition/other",
-		"comment": null,
-		"text_en": "Other",
-		"help_en": null,
-		"text_de": "Sonstige",
-		"help_de": null,
-		"additional_input": "",
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": false,
-		"changed_fields": [],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "condition/other",
-				"updated": "condition/other",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"text_en": {
-				"current": "Other",
-				"updated": "Other",
-				"changed": false
-			},
-			"text_de": {
-				"current": "Sonstige",
-				"updated": "Sonstige",
-				"changed": false
-			},
-			"help_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"help_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/condition",
-		"model": "options.optionset",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "condition",
-		"comment": null,
-		"provider_key": null,
-		"options": [
-			{
-				"uri": "http://example.com/terms/options/condition/other",
-				"model": "options.option",
-				"order": "1"
-			}
-		],
-		"conditions": [
-			{
-				"uri": "http://example.com/terms/conditions/optionset_bool_is_false",
-				"model": "conditions.condition"
-			}
-		],
-		"warnings": {
-			"http://example.com/terms/conditions/optionset_bool_is_false": [
-				"Condition http://example.com/terms/conditions/optionset_bool_is_false for OptionSet http://example.com/terms/options/condition does not exist."
-			]
-		},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": true,
-		"changed_fields": [
-			"conditions"
-		],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "condition",
-				"updated": "condition",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"conditions": {
-				"errors": [],
-				"warnings": {
-					"http://example.com/terms/options/condition": [
-						"Condition http://example.com/terms/conditions/optionset_bool_is_false for OptionSet http://example.com/terms/options/condition does not exist."
-					]
-				},
-				"current": "http://example.com/terms/conditions/optionset_bool_is_true",
-				"updated": "",
-				"changed": true,
-				"hideLineNumbers": false,
-				"splitView": false
-			},
-			"options": {
-				"new_data": [
-					{
-						"uri": "http://example.com/terms/options/condition/other",
-						"model": "options.option",
-						"order": "1"
-					}
-				],
-				"current_data": [
-					{
-						"uri": "http://example.com/terms/options/condition/other",
-						"order": 0,
-						"model": "option"
-					}
-				],
-				"current": "http://example.com/terms/options/condition/other",
-				"updated": "http://example.com/terms/options/condition/other",
-				"changed": false,
-				"hideLineNumbers": false,
-				"splitView": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/one_two_three/three",
-		"model": "options.option",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "one_two_three/three",
-		"comment": null,
-		"text_en": "Three",
-		"help_en": null,
-		"text_de": "Drei",
-		"help_de": null,
-		"additional_input": "",
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": false,
-		"changed_fields": [],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "one_two_three/three",
-				"updated": "one_two_three/three",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"text_en": {
-				"current": "Three",
-				"updated": "Three",
-				"changed": false
-			},
-			"text_de": {
-				"current": "Drei",
-				"updated": "Drei",
-				"changed": false
-			},
-			"help_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"help_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/one_two_three/two",
-		"model": "options.option",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "one_two_three/two",
-		"comment": null,
-		"text_en": "Two",
-		"help_en": null,
-		"text_de": "Zwei",
-		"help_de": null,
-		"additional_input": "",
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": false,
-		"changed_fields": [],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "one_two_three/two",
-				"updated": "one_two_three/two",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"text_en": {
-				"current": "Two",
-				"updated": "Two",
-				"changed": false
-			},
-			"text_de": {
-				"current": "Zwei",
-				"updated": "Zwei",
-				"changed": false
-			},
-			"help_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"help_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/one_two_three/one",
-		"model": "options.option",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "one_two_three/one",
-		"comment": null,
-		"text_en": "One",
-		"help_en": null,
-		"text_de": "Eins",
-		"help_de": null,
-		"additional_input": "",
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": false,
-		"changed_fields": [],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "one_two_three/one",
-				"updated": "one_two_three/one",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"text_en": {
-				"current": "One",
-				"updated": "One",
-				"changed": false
-			},
-			"text_de": {
-				"current": "Eins",
-				"updated": "Eins",
-				"changed": false
-			},
-			"help_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"help_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/one_two_three",
-		"model": "options.optionset",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "one_two_three",
-		"comment": null,
-		"provider_key": null,
-		"options": [
-			{
-				"uri": "http://example.com/terms/options/one_two_three/three",
-				"model": "options.option",
-				"order": "1"
-			},
-			{
-				"uri": "http://example.com/terms/options/one_two_three/two",
-				"model": "options.option",
-				"order": "2"
-			},
-			{
-				"uri": "http://example.com/terms/options/one_two_three/one",
-				"model": "options.option",
-				"order": "3"
-			}
-		],
-		"conditions": null,
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": true,
-		"changed_fields": [
-			"options"
-		],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "one_two_three",
-				"updated": "one_two_three",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"options": {
-				"new_data": [
-					{
-						"uri": "http://example.com/terms/options/one_two_three/three",
-						"model": "options.option",
-						"order": "1"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three/two",
-						"model": "options.option",
-						"order": "2"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three/one",
-						"model": "options.option",
-						"order": "3"
-					}
-				],
-				"current_data": [
-					{
-						"uri": "http://example.com/terms/options/one_two_three/one",
-						"order": 0,
-						"model": "option"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three/two",
-						"order": 1,
-						"model": "option"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three/three",
-						"order": 2,
-						"model": "option"
-					}
-				],
-				"current": "http://example.com/terms/options/one_two_three/one\nhttp://example.com/terms/options/one_two_three/two\nhttp://example.com/terms/options/one_two_three/three",
-				"updated": "http://example.com/terms/options/one_two_three/three\nhttp://example.com/terms/options/one_two_three/two\nhttp://example.com/terms/options/one_two_three/one",
-				"changed": true,
-				"hideLineNumbers": false,
-				"splitView": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/one_two_three_other/textarea",
-		"model": "options.option",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "one_two_three_other/textarea",
-		"comment": null,
-		"text_en": "Text",
-		"help_en": null,
-		"text_de": "Text",
-		"help_de": null,
-		"additional_input": "",
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": false,
-		"changed_fields": [],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "one_two_three_other/textarea",
-				"updated": "one_two_three_other/textarea",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"text_en": {
-				"current": "Text",
-				"updated": "Text",
-				"changed": false
-			},
-			"text_de": {
-				"current": "Text",
-				"updated": "Text",
-				"changed": false
-			},
-			"help_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"help_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/one_two_three_other/text",
-		"model": "options.option",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "one_two_three_other/text",
-		"comment": null,
-		"text_en": "Text",
-		"help_en": null,
-		"text_de": "Text",
-		"help_de": null,
-		"additional_input": "",
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": false,
-		"changed_fields": [],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "one_two_three_other/text",
-				"updated": "one_two_three_other/text",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"text_en": {
-				"current": "Text",
-				"updated": "Text",
-				"changed": false
-			},
-			"text_de": {
-				"current": "Text",
-				"updated": "Text",
-				"changed": false
-			},
-			"help_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"help_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/one_two_three_other/three",
-		"model": "options.option",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "one_two_three_other/three",
-		"comment": null,
-		"text_en": "Three - 1",
-		"help_en": null,
-		"text_de": "Drei - 1",
-		"help_de": null,
-		"additional_input": "",
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": true,
-		"changed_fields": [
-			"text_en",
-			"text_de"
-		],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "one_two_three_other/three",
-				"updated": "one_two_three_other/three",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"text_en": {
-				"current": "Three",
-				"updated": "Three - 1",
-				"changed": true
-			},
-			"text_de": {
-				"current": "Drei",
-				"updated": "Drei - 1",
-				"changed": true
-			},
-			"help_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"help_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/one_two_three_other/two",
-		"model": "options.option",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "one_two_three_other/two",
-		"comment": null,
-		"text_en": "Two",
-		"help_en": null,
-		"text_de": "Zwei",
-		"help_de": null,
-		"additional_input": "",
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": false,
-		"changed_fields": [],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "one_two_three_other/two",
-				"updated": "one_two_three_other/two",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"text_en": {
-				"current": "Two",
-				"updated": "Two",
-				"changed": false
-			},
-			"text_de": {
-				"current": "Zwei",
-				"updated": "Zwei",
-				"changed": false
-			},
-			"help_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"help_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/one_two_three_other/one",
-		"model": "options.option",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "one_two_three_other/one",
-		"comment": null,
-		"text_en": "One - 3",
-		"help_en": null,
-		"text_de": "Eins - 3",
-		"help_de": null,
-		"additional_input": "",
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": true,
-		"changed_fields": [
-			"text_en",
-			"text_de"
-		],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "one_two_three_other/one",
-				"updated": "one_two_three_other/one",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"text_en": {
-				"current": "One",
-				"updated": "One - 3",
-				"changed": true
-			},
-			"text_de": {
-				"current": "Eins",
-				"updated": "Eins - 3",
-				"changed": true
-			},
-			"help_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"help_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_en": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"view_text_de": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/one_two_three_other",
-		"model": "options.optionset",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "one_two_three_other",
-		"comment": null,
-		"provider_key": null,
-		"options": [
-			{
-				"uri": "http://example.com/terms/options/one_two_three_other/textarea",
-				"model": "options.option",
-				"order": "1"
-			},
-			{
-				"uri": "http://example.com/terms/options/one_two_three_other/text",
-				"model": "options.option",
-				"order": "2"
-			},
-			{
-				"uri": "http://example.com/terms/options/one_two_three_other/three",
-				"model": "options.option",
-				"order": "3"
-			},
-			{
-				"uri": "http://example.com/terms/options/one_two_three_other/two",
-				"model": "options.option",
-				"order": "4"
-			},
-			{
-				"uri": "http://example.com/terms/options/one_two_three_other/one",
-				"model": "options.option",
-				"order": "5"
-			}
-		],
-		"conditions": null,
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": true,
-		"changed_fields": [
-			"options"
-		],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "one_two_three_other",
-				"updated": "one_two_three_other",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"options": {
-				"new_data": [
-					{
-						"uri": "http://example.com/terms/options/one_two_three_other/textarea",
-						"model": "options.option",
-						"order": "1"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three_other/text",
-						"model": "options.option",
-						"order": "2"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three_other/three",
-						"model": "options.option",
-						"order": "3"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three_other/two",
-						"model": "options.option",
-						"order": "4"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three_other/one",
-						"model": "options.option",
-						"order": "5"
-					}
-				],
-				"current_data": [
-					{
-						"uri": "http://example.com/terms/options/one_two_three_other/one",
-						"order": 0,
-						"model": "option"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three_other/two",
-						"order": 1,
-						"model": "option"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three_other/three",
-						"order": 2,
-						"model": "option"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three_other/text",
-						"order": 3,
-						"model": "option"
-					},
-					{
-						"uri": "http://example.com/terms/options/one_two_three_other/textarea",
-						"order": 4,
-						"model": "option"
-					}
-				],
-				"current": "http://example.com/terms/options/one_two_three_other/one\nhttp://example.com/terms/options/one_two_three_other/two\nhttp://example.com/terms/options/one_two_three_other/three\nhttp://example.com/terms/options/one_two_three_other/text\nhttp://example.com/terms/options/one_two_three_other/textarea",
-				"updated": "http://example.com/terms/options/one_two_three_other/textarea\nhttp://example.com/terms/options/one_two_three_other/text\nhttp://example.com/terms/options/one_two_three_other/three\nhttp://example.com/terms/options/one_two_three_other/two\nhttp://example.com/terms/options/one_two_three_other/one",
-				"changed": true,
-				"hideLineNumbers": false,
-				"splitView": false
-			}
-		}
-	},
-	{
-		"uri": "http://example.com/terms/options/plugin",
-		"model": "options.optionset",
-		"uri_prefix": "http://example.com/terms",
-		"uri_path": "plugin",
-		"comment": null,
-		"provider_key": "simple",
-		"options": null,
-		"conditions": null,
-		"warnings": {},
-		"errors": [],
-		"created": false,
-		"updated": true,
-		"changed": false,
-		"changed_fields": [],
-		"updated_and_changed": {
-			"uri_prefix": {
-				"current": "http://example.com/terms",
-				"updated": "http://example.com/terms",
-				"changed": false
-			},
-			"uri_path": {
-				"current": "plugin",
-				"updated": "plugin",
-				"changed": false
-			},
-			"comment": {
-				"current": "",
-				"updated": "",
-				"changed": false
-			},
-			"options": {
-				"new_data": [],
-				"current_data": [],
-				"current": "",
-				"updated": "",
-				"changed": false,
-				"hideLineNumbers": false,
-				"splitView": false
-			}
-		}
-	}
-]

From ff1b8cbb3592242a950b195e52827a5aae38b905 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 7 Mar 2024 15:16:54 +0100
Subject: [PATCH 125/205] chore: at import add a message for validation error
 on full_clean

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 4ac7cd7278..65c9b90b42 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -529,16 +529,34 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
     track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data)
 
 
+def format_message_from_validation_error(exception: ValidationError) -> str:
+    message = '; '.join(['{}: {}'.format(key, ', '.join(messages)) for key, messages in exception.message_dict.items()])
+    return message
+
+
 def validate_instance(instance, element, *validators):
     exception_message = None
-    instance.full_clean()
+    try:
+        instance.full_clean()
+    except ValidationError as e:
+        exception_message = format_message_from_validation_error(e)
+        message = '{instance_model} {instance_uri} cannot be imported ({exception}).'.format(
+            instance_model=instance._meta.object_name,
+            instance_uri=element.get('uri'),
+            exception=exception_message
+        )
+        logger.info(message)
+        _key = "FullClean"
+        element['errors'].append(message)
+        track_messages_on_element(element, _key, error=message)
+        return
+
     for validator in validators:
         try:
             validator(instance if instance.id else None)(vars(instance))
         except ValidationError as e:
             try:
-                exception_message = '; '.join(['{}: {}'.format(key, ', '.join(messages))
-                                               for key, messages in e.message_dict.items()])
+                exception_message = format_message_from_validation_error(e)
             except AttributeError:
                 exception_message = ''.join(e.messages)
         except ObjectDoesNotExist as e:

From beab4ba55c7939376331680fd68ad363b9caede5 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 7 Mar 2024 15:34:54 +0100
Subject: [PATCH 126/205] js: update import element fields

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../management/assets/js/components/import/ImportElement.js | 2 --
 .../management/assets/js/components/import/common/Fields.js | 6 +++---
 .../assets/js/components/import/common/ImportInfo.js        | 6 +++---
 .../assets/js/components/sidebar/ImportSidebar.js           | 2 +-
 4 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index 1cd8764e33..7ed6da9ad2 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -6,7 +6,6 @@ import { CodeLink, WarningLink, ErrorLink, ShowLink, ShowUpdatedLink } from '../
 import Errors from './common/Errors'
 import Fields from './common/Fields'
 import Form from './common/Form'
-import Warnings from './common/Warnings'
 
 import { codeClass, verboseNames } from '../../constants/elements'
 import { isEmpty } from 'lodash'
@@ -36,7 +35,6 @@ const ImportElement = ({ config, element, importActions }) => {
         element.show && <>
           <Form config={config} element={element} updateElement={updateElement} />
           <Fields element={element} />
-          <Warnings element={element} showTitle={false}/>
           <Errors element={element} />
         </>
       }
diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index 811af79842..7a57f2cd4c 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -5,7 +5,6 @@ import isString from 'lodash/isString'
 import isUndefined from 'lodash/isUndefined'
 import truncate from 'lodash/truncate'
 import uniqueId from 'lodash/uniqueId'
-import isEmpty from 'lodash/isEmpty'
 
 
 import { codeClass } from '../../../constants/elements'
@@ -57,8 +56,9 @@ const Fields = ({ element }) => {
                   }
                 </div>
                 {
-                  isEmpty(element.errors) && !isEmpty(element.updated_and_changed) && element.updated &&
-                  key in element.updated_and_changed && <FieldsDiffs element={element} field={key}/>
+                  element.updated && element.changed &&
+                  key in element.updated_and_changed &&
+                  <FieldsDiffs element={element} field={key}/>
                   }
               </div>
             )
diff --git a/rdmo/management/assets/js/components/import/common/ImportInfo.js b/rdmo/management/assets/js/components/import/common/ImportInfo.js
index 59d156778d..2a2a4c3ac2 100644
--- a/rdmo/management/assets/js/components/import/common/ImportInfo.js
+++ b/rdmo/management/assets/js/components/import/common/ImportInfo.js
@@ -20,9 +20,9 @@ const ImportInfo = ({
   return (
     <div className="pull-right">
       {renderElementLengthInfo('Total', elementsLength)}
-      {renderElementLengthInfo('Updated', updatedLength)}
-      {updatedLength > 0 && <span>{' ('}{gettext('Changed')}{': '}{changedLength}{') '}</span>}
-      {renderElementLengthInfo('Created', createdLength)}
+      {renderElementLengthInfo('updated', updatedLength)}
+      {changedLength > 0 && <span>{' ('}{gettext('changed')}{': '}{changedLength}{') '}</span>}
+      {renderElementLengthInfo('created', createdLength)}
       {renderElementLengthInfo('Warnings', warningsLength)}
       {renderElementLengthInfo('Errors', errorsLength)}
     </div>
diff --git a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
index 140105c1df..d5a0244bc2 100644
--- a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
+++ b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
@@ -8,7 +8,7 @@ import Link from 'rdmo/core/assets/js/components/Link'
 const ImportSidebar = ({ config, imports, importActions }) => {
   const { elements, success } = imports
   const count = elements.filter(e => e.import).length
-  const updatedAndChangedElements = elements.filter(element => element.updated && !isEmpty(element.updated_and_changed))
+  const updatedAndChangedElements = elements.filter(element => element.updated && element.changed)
   const [uriPrefix, setUriPrefix] = useState('')
   const disabled = isNil(uriPrefix) || isEmpty(uriPrefix)
 

From 8c139d95297337ca662a9df4eb28db1937bc77b6 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 7 Mar 2024 15:48:39 +0100
Subject: [PATCH 127/205] tests: refactor frontend fixtures in conftest.py

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../tests/{fixtures_frontend.py => conftest.py}          | 0
 rdmo/management/tests/test_frontend_import_options.py    | 9 +--------
 rdmo/management/tests/test_frontend_import_questions.py  | 8 +-------
 ..._frontend.py => test_frontend_management_elements.py} | 7 -------
 4 files changed, 2 insertions(+), 22 deletions(-)
 rename rdmo/management/tests/{fixtures_frontend.py => conftest.py} (100%)
 rename rdmo/management/tests/{test_frontend.py => test_frontend_management_elements.py} (96%)

diff --git a/rdmo/management/tests/fixtures_frontend.py b/rdmo/management/tests/conftest.py
similarity index 100%
rename from rdmo/management/tests/fixtures_frontend.py
rename to rdmo/management/tests/conftest.py
diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
index c675a1f7b9..773a32a664 100644
--- a/rdmo/management/tests/test_frontend_import_options.py
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -7,11 +7,6 @@
 
 from rdmo.options.models import Option, OptionSet
 
-from .fixtures_frontend import (
-    base_url_page,  # noqa: F401
-    e2e_tests_django_db_setup,  # noqa: F401
-    logged_in_user,  # noqa: F401
-)
 from .helpers_models import delete_all_objects
 
 pytestmark = pytest.mark.e2e
@@ -46,9 +41,7 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     page.get_by_role("link", name="Select all", exact=True).click()
     page.get_by_role("link", name="Show all", exact=True).click()
     rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")
-    # there are 2 rows per object displayed
-    expect(rows_displayed_in_ui).to_have_count(13 * 2)
-
+    expect(rows_displayed_in_ui).to_have_count(13)
     # click the import button to start saving the instances to the db
     page.get_by_role("button", name="Import 13 elements").click()
     expect(page.get_by_role("heading", name="Import successful")).to_be_visible()
diff --git a/rdmo/management/tests/test_frontend_import_questions.py b/rdmo/management/tests/test_frontend_import_questions.py
index 4790a52933..48a567e515 100644
--- a/rdmo/management/tests/test_frontend_import_questions.py
+++ b/rdmo/management/tests/test_frontend_import_questions.py
@@ -9,11 +9,6 @@
 from rdmo.questions.models import Page as PageModel
 from rdmo.questions.models.questionset import QuestionSet
 
-from .fixtures_frontend import (
-    base_url_page,  # noqa: F401
-    e2e_tests_django_db_setup,  # noqa: F401
-    logged_in_user,  # noqa: F401
-)
 from .helpers_models import delete_all_objects
 
 pytestmark = pytest.mark.e2e
@@ -44,8 +39,7 @@ def test_import_catalogs_in_management(logged_in_user: Page) -> None:
     page.get_by_role("link", name="Select all", exact=True).click()
     page.get_by_role("link", name="Show all").click()
     rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")
-    # there are 2 rows per object displayed
-    expect(rows_displayed_in_ui).to_have_count(148 * 2)
+    expect(rows_displayed_in_ui).to_have_count(148)
     page.get_by_role("link", name="Hide all").click()
     expect(rows_displayed_in_ui).to_have_count(0)
     page.screenshot(path="screenshots/management-import-pre-import.png", full_page=True)
diff --git a/rdmo/management/tests/test_frontend.py b/rdmo/management/tests/test_frontend_management_elements.py
similarity index 96%
rename from rdmo/management/tests/test_frontend.py
rename to rdmo/management/tests/test_frontend_management_elements.py
index 2143ae777c..9eac650be9 100644
--- a/rdmo/management/tests/test_frontend.py
+++ b/rdmo/management/tests/test_frontend_management_elements.py
@@ -11,13 +11,6 @@
 from rdmo.domain.models import Attribute
 from rdmo.questions.models import Catalog
 
-from .fixtures_frontend import (
-    base_url_page,  # noqa: F401
-    e2e_tests_django_db_setup,  # noqa: F401
-    logged_in_user,  # noqa: F401
-)
-
-# logged_in_user,  # ruff: noqa: F811
 from .helpers_models import ModelHelper, model_helpers
 
 pytestmark = pytest.mark.e2e

From 736f96973d4f2d1dfb244a1416836f16ce386cdc Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 8 Mar 2024 11:59:03 +0100
Subject: [PATCH 128/205] tests: re-use constant ELEMENT_DIFF_FIELD_NAME for
 import tests

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/test_import_conditions.py |  3 ++-
 rdmo/management/tests/test_import_domain.py     |  3 ++-
 rdmo/management/tests/test_import_options.py    |  4 ++--
 rdmo/management/tests/test_import_questions.py  | 11 ++++++-----
 rdmo/management/tests/test_import_tasks.py      |  3 ++-
 rdmo/management/tests/test_import_views.py      |  3 ++-
 6 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/rdmo/management/tests/test_import_conditions.py b/rdmo/management/tests/test_import_conditions.py
index 6784039078..5452f97c68 100644
--- a/rdmo/management/tests/test_import_conditions.py
+++ b/rdmo/management/tests/test_import_conditions.py
@@ -3,6 +3,7 @@
 import pytest
 
 from rdmo.conditions.models import Condition
+from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
 from rdmo.management.imports import import_elements
 
 from .helpers_import_elements import (
@@ -53,7 +54,7 @@ def test_update_conditions_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 def test_create_legacy_conditions(db, settings):
diff --git a/rdmo/management/tests/test_import_domain.py b/rdmo/management/tests/test_import_domain.py
index 5700662b4e..4f92be8bb6 100644
--- a/rdmo/management/tests/test_import_domain.py
+++ b/rdmo/management/tests/test_import_domain.py
@@ -2,6 +2,7 @@
 
 import pytest
 
+from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
 from rdmo.domain.models import Attribute
 from rdmo.management.imports import import_elements
 
@@ -53,7 +54,7 @@ def test_update_attributes_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 def test_create_legacy_domain(db, settings):
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 7724a2281d..266d333b4d 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -60,7 +60,7 @@ def test_update_optionsets_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 def test_update_optionsets_from_changed_xml(db, settings):
@@ -156,7 +156,7 @@ def test_update_options_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 
diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py
index db2ac94a3b..d2ff7a0a3e 100644
--- a/rdmo/management/tests/test_import_questions.py
+++ b/rdmo/management/tests/test_import_questions.py
@@ -2,6 +2,7 @@
 
 import pytest
 
+from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
 from rdmo.management.imports import import_elements
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
@@ -64,7 +65,7 @@ def test_update_catalogs_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 def test_create_sections(db, settings):
@@ -114,7 +115,7 @@ def test_update_sections_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 def test_create_pages(db, settings):
@@ -163,7 +164,7 @@ def test_update_pages_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 def test_create_questionsets(db, settings):
@@ -214,7 +215,7 @@ def test_update_questionsets_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 def test_create_questions(db, settings):
@@ -261,7 +262,7 @@ def test_update_questions_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 def test_create_legacy_questions(db, settings):
diff --git a/rdmo/management/tests/test_import_tasks.py b/rdmo/management/tests/test_import_tasks.py
index ff169fb2dc..9c2a813629 100644
--- a/rdmo/management/tests/test_import_tasks.py
+++ b/rdmo/management/tests/test_import_tasks.py
@@ -2,6 +2,7 @@
 
 import pytest
 
+from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
 from rdmo.management.imports import import_elements
 from rdmo.tasks.models import Task
 
@@ -53,7 +54,7 @@ def test_update_tasks_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 def test_create_legacy_tasks(db, settings):
diff --git a/rdmo/management/tests/test_import_views.py b/rdmo/management/tests/test_import_views.py
index f3cf0017ed..c6418499ab 100644
--- a/rdmo/management/tests/test_import_views.py
+++ b/rdmo/management/tests/test_import_views.py
@@ -2,6 +2,7 @@
 
 import pytest
 
+from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
 from rdmo.management.imports import import_elements
 from rdmo.views.models import View
 
@@ -53,7 +54,7 @@ def test_update_views_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test['updated_and_changed'] == imported['updated_and_changed']
+        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
 
 
 def test_create_legacy_tasks(db, settings):

From c200a6158e5a354a54f061d13e57c0c50f68594c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 8 Mar 2024 14:01:33 +0100
Subject: [PATCH 129/205] js: fix import reducer selectChangedElements
 showChangedElements

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/js/reducers/importsReducer.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index bcb910278b..1826c1ecab 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -1,7 +1,6 @@
 import isArray from 'lodash/isArray'
 import isNil from 'lodash/isNil'
 import isUndefined from 'lodash/isUndefined'
-import { isEmpty } from 'lodash'
 
 import { buildUri } from '../utils/elements'
 
@@ -58,7 +57,7 @@ export default function importsReducer(state = initialState, action) {
       })}
     case 'import/selectChangedElements':
       return {...state, elements: state.elements.map(element => {
-        if (element.updated && !isEmpty(element.updated_and_changed) && !element.created ) {
+        if (element.updated && element.changed && !element.created ) {
           return {...element, import: action.value}
         }
         else if (action.value) {return {...element, import: !action.value}}
@@ -71,7 +70,7 @@ export default function importsReducer(state = initialState, action) {
       })}
     case 'import/showChangedElements':
       return {...state, elements: state.elements.map(element => {
-        if (element.updated && !isEmpty(element.updated_and_changed) && !element.created ) {
+        if (element.updated && element.changed && !element.created ) {
           return {...element, show: action.value}
         }
         else if (action.value) {return {...element, show: !action.value}}

From 9eb3af981c7e0411c2fc2c4094123315b051a6b3 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 8 Mar 2024 14:02:36 +0100
Subject: [PATCH 130/205] js: add nested list for changes in sidebar

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/sidebar/ImportSidebar.js    | 54 ++++++++++---------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
index d5a0244bc2..a24fd81cce 100644
--- a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
+++ b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
@@ -5,6 +5,8 @@ import isNil from 'lodash/isNil'
 
 import Link from 'rdmo/core/assets/js/components/Link'
 
+
+
 const ImportSidebar = ({ config, imports, importActions }) => {
   const { elements, success } = imports
   const count = elements.filter(e => e.import).length
@@ -51,20 +53,22 @@ const ImportSidebar = ({ config, imports, importActions }) => {
               {gettext('Select all')}
             </Link>
           </li>
-          {updatedAndChangedElements.length > 0 &&
-            <ul className="list-unstyled">
-            <li>
-              <Link onClick={() => importActions.selectChangedElements(true)}>
-                {gettext('Select changed')}
-              </Link>
-            </li>
+            {updatedAndChangedElements.length > 0 &&
             <li>
-              <Link onClick={() => importActions.selectChangedElements(false)}>
-                {gettext('Unselect changed')}
-              </Link>
+              <ul className="list-unstyled" style={{paddingLeft:'20px'}}>
+                <li>
+                  <Link onClick={() => importActions.selectChangedElements(true)}>
+                    {gettext('Select changed')}
+                  </Link>
+                </li>
+                <li>
+                  <Link onClick={() => importActions.selectChangedElements(false)}>
+                    {gettext('Unselect changed')}
+                  </Link>
+                </li>
+              </ul>
             </li>
-            </ul>
-          }
+            }
           <li>
             <Link onClick={() => importActions.selectElements(false)}>
               {gettext('Unselect all')}
@@ -80,18 +84,20 @@ const ImportSidebar = ({ config, imports, importActions }) => {
             </Link>
           </li>
           {updatedAndChangedElements.length > 0 &&
-            <ul className="list-unstyled">
-            <li>
-              <Link onClick={() => importActions.showChangedElements(true)}>
-                {gettext('Show changes')}
-              </Link>
-            </li>
-            <li>
-              <Link onClick={() => importActions.showChangedElements(false)}>
-                {gettext('Hide changes')}
-              </Link>
-            </li>
-            </ul>
+              <li>
+                <ul className="list-unstyled" style={{paddingLeft:'20px'}}>
+                  <li>
+                    <Link onClick={() => importActions.showChangedElements(true)}>
+                      {gettext('Show changes')}
+                    </Link>
+                  </li>
+                  <li>
+                    <Link onClick={() => importActions.showChangedElements(false)}>
+                      {gettext('Hide changes')}
+                    </Link>
+                  </li>
+                </ul>
+              </li>
           }
           <li>
             <Link onClick={() => importActions.showElements(false)}>

From 959efd154d90ff75464067335a9128afa26d6e0c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 4 Apr 2024 16:36:19 +0200
Subject: [PATCH 131/205] chore: remove unused file_path_exists

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 65c9b90b42..dfeca075db 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -4,7 +4,6 @@
 from collections import defaultdict
 from dataclasses import dataclass, field
 from os.path import join as pj
-from pathlib import Path
 from random import randint
 from typing import Callable, Iterable, List, Optional, Sequence, Tuple, Union
 
@@ -41,10 +40,6 @@ def handle_fetched_file(filedata):
     return tempfilename
 
 
-def file_path_exists(file_path):
-    return Path(file_path).exists()
-
-
 def generate_tempfile_name():
     t = int(round(time.time() * 1000))
     r = randint(10000, 99999)

From 151b27b53854847d58cb403dde85695fffab8405 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 5 Apr 2024 14:45:03 +0200
Subject: [PATCH 132/205] chore: add question util func
 get_widget_type_or_default

---
 rdmo/core/import_helpers.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 rdmo/core/import_helpers.py

diff --git a/rdmo/core/import_helpers.py b/rdmo/core/import_helpers.py
new file mode 100644
index 0000000000..e69de29bb2

From 83779235bcc9438eb167d4076d7a6ccf89fa574f Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 5 Apr 2024 14:45:41 +0200
Subject: [PATCH 133/205] chore: remove unused constants

---
 rdmo/core/constants.py | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/rdmo/core/constants.py b/rdmo/core/constants.py
index 8c42a1b42e..80bf2e353a 100644
--- a/rdmo/core/constants.py
+++ b/rdmo/core/constants.py
@@ -84,22 +84,6 @@
     'comment',
 )
 
-ELEMENT_IMPORT_EXTRA_FIELDS_DEFAULTS = {
-    'order': 0,
-    'available': True,
-    'template': '',
-    'relation': '',
-    'target_text': '',
-    'provider_key': '',
-    'additional_input': '',
-    'is_collection': False,
-    'is_optional': False,
-    'default_external_id': '',
-    'value_type': '',
-    'unit': '',
-    'widget_type': 'text',
-}
-
 RDMO_MODELS = {
   'catalog': 'questions.catalog',
   'section': 'questions.section',

From db404138810a38a7ca9b30b1e843afbc6af3351e Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 5 Apr 2024 17:47:08 +0200
Subject: [PATCH 134/205] chore: refactor import helpers, add extra_field
 default helper

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/import_helpers.py | 47 ++++++++++++++++++++++++++
 rdmo/core/imports.py        | 66 ++++++++++---------------------------
 rdmo/management/imports.py  | 17 ++++------
 3 files changed, 71 insertions(+), 59 deletions(-)

diff --git a/rdmo/core/import_helpers.py b/rdmo/core/import_helpers.py
index e69de29bb2..d473fe8f0b 100644
--- a/rdmo/core/import_helpers.py
+++ b/rdmo/core/import_helpers.py
@@ -0,0 +1,47 @@
+from dataclasses import dataclass, field
+from inspect import signature
+from typing import Callable, Iterable, Optional, Sequence, Union
+
+from django.db import models
+
+from rdmo.core.constants import ELEMENT_COMMON_FIELDS
+
+
+@dataclass(frozen=True)
+class ThroughInstanceMapper:
+    field_name: str
+    source_name: str
+    target_name: str
+    through_name: str
+
+
+@dataclass(frozen=True)
+class ExtraFieldDefaultHelper:
+    field_name: str
+    value: Optional[Union[str, bool, int]] = None
+    callback: Optional[Callable] = None
+
+    def get_default(self, **kwargs):
+        if self.callback is None:
+            return self.value
+
+        sig = signature(self.callback)
+        _kwargs = {k:val for k,val in kwargs.items() if k in sig.parameters}
+        _value = self.callback(**_kwargs)
+        return _value
+
+
+@dataclass(frozen=True)
+class ElementImportHelper:
+    model: Optional[models.Model] = field(default=None)
+    model_path: Optional[str] = field(default=None)
+    validators: Iterable[Callable] = field(default_factory=list)
+    common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
+    lang_fields: Sequence[str] = field(default_factory=list)
+    foreign_fields: Sequence[str] = field(default_factory=list)
+    extra_fields: Sequence[ExtraFieldDefaultHelper] = field(default_factory=list)
+    m2m_instance_fields: Sequence[str] = field(default_factory=list)
+    m2m_through_instance_fields: Sequence[ThroughInstanceMapper] = field(default_factory=list)
+    reverse_m2m_through_instance_fields: Sequence[ThroughInstanceMapper] = field(default_factory=list)
+    add_current_site_editors: bool = field(default=True)
+    add_current_site_sites: bool = field(default=False)
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index dfeca075db..0e52c60ef1 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -2,10 +2,9 @@
 import tempfile
 import time
 from collections import defaultdict
-from dataclasses import dataclass, field
 from os.path import join as pj
 from random import randint
-from typing import Callable, Iterable, List, Optional, Sequence, Tuple, Union
+from typing import List, Optional, Tuple, Union
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
@@ -16,7 +15,8 @@
 
 from diff_match_patch import diff_match_patch
 
-from rdmo.core.constants import ELEMENT_COMMON_FIELDS, ELEMENT_IMPORT_EXTRA_FIELDS_DEFAULTS, RDMO_MODELS
+from rdmo.core.constants import RDMO_MODELS
+from rdmo.core.import_helpers import ExtraFieldDefaultHelper
 from rdmo.core.utils import get_languages
 
 logger = logging.getLogger(__name__)
@@ -148,28 +148,6 @@ def track_changes_on_element(element: dict,
     element[ELEMENT_DIFF_FIELD_NAME][element_field].update(js_diff_viewer_props)
 
 
-@dataclass(frozen=True)
-class ThroughInstanceMapper:
-    field_name: str
-    source_name: str
-    target_name: str
-    through_name: str
-
-@dataclass(frozen=True)
-class ElementImportHelper:
-    model: Optional[models.Model] = field(default=None)
-    model_path: Optional[str] = field(default=None)
-    validators: Iterable[Callable] = field(default_factory=list)
-    common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
-    lang_fields: Sequence[str] = field(default_factory=list)
-    foreign_fields: Sequence[str] = field(default_factory=list)
-    extra_fields: Sequence[str] = field(default_factory=list)
-    m2m_instance_fields: Sequence[str] = field(default_factory=list)
-    m2m_through_instance_fields: Sequence[ThroughInstanceMapper] = field(default_factory=list)
-    reverse_m2m_through_instance_fields: Sequence[ThroughInstanceMapper] = field(default_factory=list)
-    add_current_site_editors: bool = field(default=True)
-    add_current_site_sites: bool = field(default=False)
-
 def get_lang_field_values(field_name: str,
                         element: Optional[dict] = None,
                         instance: Optional[models.Model] = None,
@@ -275,33 +253,23 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
         track_messages_on_element(element, field_name, error=message)
 
 
-def set_extra_field(instance, field_name, element, questions_widget_types=None, original=None) -> None:
+def set_extra_field(instance, field_name, element,
+                     extra_field_helper: Optional[ExtraFieldDefaultHelper]=None, original=None) -> None:
 
     element_value = element.get(field_name)
-    default_value = ELEMENT_IMPORT_EXTRA_FIELDS_DEFAULTS.get(field_name)
-    extra_value = element_value or default_value
-    if field_name == 'widget_type':
-        if element_value in questions_widget_types:
-            extra_value = element_value
-        else:
-            extra_value = default_value
-    if field_name == "path":
-        if instance.key and hasattr(instance, "build_path"):
-            extra_value = instance.build_path(instance.key, instance.parent)
-        else:
-            exception_message = _('This field may not be blank.')
-            message = '{instance_model} {instance_uri} cannot be imported (key: {exception}) .'.format(
-                instance_model=instance._meta.object_name,
-                instance_uri=element.get('uri'),
-                exception=exception_message
-            )
-            logger.info(message)
-            element['errors'].append(message)
-            track_messages_on_element(element, field_name, error=message)
 
-    setattr(instance, field_name, extra_value)
-    # track changes
-    track_changes_on_element(element, field_name, new_value=extra_value, original=original)
+    extra_value = None
+    if element_value is not None:
+        extra_value = element_value
+    elif extra_field_helper is not None:
+        # default_value
+        extra_value = extra_field_helper.get_default(instance=instance,
+                                                      key=field_name)
+
+    if extra_value is not None:
+        setattr(instance, field_name, extra_value)
+        # track changes
+        track_changes_on_element(element, field_name, new_value=extra_value, original=original)
 
 
 def track_changes_m2m_instances(element, field_name,
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index a0151f37b4..7bdcdffcb4 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -32,7 +32,6 @@
     import_helper_questionset,
     import_helper_section,
 )
-from rdmo.questions.utils import get_widget_types
 from rdmo.tasks.imports import import_helper_task
 from rdmo.views.imports import import_helper_view
 
@@ -69,11 +68,10 @@ def import_elements(uploaded_elements: Dict, save: bool = True, request: Optiona
     imported_elements = []
     uploaded_uris = set(uploaded_elements.keys())
     current_site = get_current_site(request)
-    questions_widget_types = get_widget_types()
     for _uri, uploaded_element in uploaded_elements.items():
-        element = import_element(element=uploaded_element, save=save, uploaded_uris=uploaded_uris,
-                                    request=request, current_site=current_site,
-                                    questions_widget_types=questions_widget_types)
+        element = import_element(element=uploaded_element, save=save,
+                                  uploaded_uris=uploaded_uris,
+                                    request=request, current_site=current_site)
         element['warnings'] = {k: val  for k, val in element['warnings'].items() if k not in uploaded_uris}
         imported_elements.append(element)
     return imported_elements
@@ -89,8 +87,7 @@ def import_element(
         save: bool = True,
         request: Optional[HttpRequest] = None,
         uploaded_uris: Optional[AbstractSet[str]] = None,
-        current_site = None,
-        questions_widget_types = None
+        current_site = None
     ) -> Dict:
 
     if element is None:
@@ -111,7 +108,7 @@ def import_element(
     common_fields = import_helper.common_fields
     lang_field_names = import_helper.lang_fields
     foreign_field_names = import_helper.foreign_fields
-    extra_field_names = import_helper.extra_fields
+    extra_field_helpers = import_helper.extra_fields
 
     uri = element.get('uri')
     # get or create instance from uri and model_path
@@ -153,8 +150,8 @@ def import_element(
     for foreign_field in foreign_field_names:
         set_foreign_field(instance, foreign_field, element, uploaded_uris=uploaded_uris)
     # set extra fields
-    for extra_field in extra_field_names:
-        set_extra_field(instance, extra_field, element, questions_widget_types=questions_widget_types)
+    for extra_field_helper in extra_field_helpers:
+        set_extra_field(instance, extra_field_helper.field_name, element, extra_field_helper=extra_field_helper)
     # call the validators on the instance
     validate_instance(instance, element, *validators)
 

From 592dbf781b1f0077fba4b5ecbfdb1c731180231e Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 5 Apr 2024 17:48:35 +0200
Subject: [PATCH 135/205] chore: add question util func
 get_widget_type_or_default

---
 rdmo/questions/utils.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/rdmo/questions/utils.py b/rdmo/questions/utils.py
index 84bed4874f..31d09d53ec 100644
--- a/rdmo/questions/utils.py
+++ b/rdmo/questions/utils.py
@@ -11,6 +11,14 @@ def get_widget_types():
     return [widget.key for widget in widgets.values()]
 
 
+def get_widget_type_or_default(key=None):
+    widget_types = get_widget_types()
+    if key in widget_types:
+        return key
+    else:
+        return widget_types[0]
+
+
 def get_widget_type_choices():
     widgets = get_plugins('QUESTIONS_WIDGETS')
     return [(widget.key, widget.label) for widget in widgets.values()]

From 877afd021cf8611f0dc7e7fcf5e4d7a71f5dd7bb Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 5 Apr 2024 17:50:01 +0200
Subject: [PATCH 136/205] chore: refactor import helpers extra fields

---
 rdmo/conditions/imports.py |  8 +++--
 rdmo/domain/imports.py     | 14 +++++---
 rdmo/options/imports.py    | 12 ++++---
 rdmo/questions/imports.py  | 70 +++++++++++++++++++++++---------------
 rdmo/tasks/imports.py      | 10 ++++--
 rdmo/views/imports.py      |  9 +++--
 6 files changed, 78 insertions(+), 45 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index c0eccaa4a2..566f1e4df7 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -1,5 +1,4 @@
-from rdmo.core.imports import ElementImportHelper
-
+from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
 from .models import Condition
 from .validators import ConditionLockedValidator, ConditionUniqueURIValidator
 
@@ -8,5 +7,8 @@
     model_path="conditions.condition",
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
     foreign_fields=('source', 'target_option'),
-    extra_fields=('relation', 'target_text')
+    extra_fields=(
+        ExtraFieldDefaultHelper(field_name='relation', value=''),
+        ExtraFieldDefaultHelper(field_name='target_text', value=''),
+    ),
 )
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 989a28607e..0459a1dd8d 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -1,19 +1,23 @@
 import logging
+from typing import Optional
 
-from rdmo.core.imports import (
-    ElementImportHelper,
-)
-
+from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
 from .models import Attribute
 from .validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
 
 logger = logging.getLogger(__name__)
 
+
+def get_default_path(instance: Optional[Attribute]=None):
+    if instance is not None:
+        return instance.build_path(instance.key, instance.parent)
+
+
 import_helper_attribute = ElementImportHelper(
     model=Attribute,
     model_path="domain.attribute",
     common_fields=('uri_prefix', 'key', 'comment'),
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     foreign_fields=('parent',),
-    extra_fields=('path',),
+    extra_fields=[ExtraFieldDefaultHelper(field_name='path', callback=get_default_path)],
 )
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index a6826beedb..086a2e1738 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -1,5 +1,4 @@
-from rdmo.core.imports import ElementImportHelper, ThroughInstanceMapper
-
+from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper, ThroughInstanceMapper
 from .models import Option, OptionSet
 from .validators import (
     OptionLockedValidator,
@@ -12,7 +11,10 @@
     model = OptionSet,
     model_path = "options.optionset",
     validators = (OptionSetLockedValidator, OptionSetUniqueURIValidator),
-    extra_fields = ('order', 'provider_key'),
+    extra_fields = (
+        ExtraFieldDefaultHelper(field_name='order'),
+        ExtraFieldDefaultHelper(field_name='provider_key', value=''),
+    ),
     m2m_instance_fields = ('conditions', ),
     m2m_through_instance_fields = [
         ThroughInstanceMapper(
@@ -29,7 +31,9 @@
     model_path = "options.option",
     validators = (OptionLockedValidator, OptionUniqueURIValidator),
     lang_fields = ('text', 'help', 'view_text'),
-    extra_fields = ('additional_input',),
+    extra_fields = (
+        ExtraFieldDefaultHelper(field_name='additional_input'),
+        ),
     reverse_m2m_through_instance_fields = [
         ThroughInstanceMapper(
             field_name='optionset',
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 7ae24ee27e..18cdcbfc15 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -1,6 +1,7 @@
-from rdmo.core.imports import ElementImportHelper, ThroughInstanceMapper
-
+from ..core.constants import VALUE_TYPE_TEXT
+from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper, ThroughInstanceMapper
 from .models import Catalog, Page, Question, QuestionSet, Section
+from .utils import get_widget_type_or_default
 from .validators import (
     CatalogLockedValidator,
     CatalogUniqueURIValidator,
@@ -19,7 +20,10 @@
     model_path="questions.catalog",
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
     lang_fields=('help', 'title'),
-    extra_fields = ('order', 'available'),
+    extra_fields = (
+        ExtraFieldDefaultHelper(field_name='order'),
+        ExtraFieldDefaultHelper(field_name='available'),
+    ),
     m2m_through_instance_fields=[
         ThroughInstanceMapper(
             field_name='sections', source_name='catalog',
@@ -54,7 +58,9 @@
     validators=(PageLockedValidator, PageUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
-    extra_fields = ('is_collection',),
+    extra_fields = (
+        ExtraFieldDefaultHelper(field_name='is_collection'),
+    ),
     m2m_instance_fields = ('conditions', ),
     m2m_through_instance_fields=[
         ThroughInstanceMapper(
@@ -74,13 +80,45 @@
     ]
 )
 
+import_helper_question = ElementImportHelper(
+    model=Question,
+    model_path="questions.question",
+    validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
+    lang_fields=('text', 'help', 'default_text', 'verbose_name'),
+    foreign_fields=('attribute', 'default_option'),
+    extra_fields=(
+        ExtraFieldDefaultHelper(field_name='is_collection'),
+        ExtraFieldDefaultHelper(field_name='is_optional'),
+        ExtraFieldDefaultHelper(field_name='default_external_id', value=''),
+        ExtraFieldDefaultHelper(field_name='widget_type', callback=get_widget_type_or_default),
+        ExtraFieldDefaultHelper(field_name='value_type', value=VALUE_TYPE_TEXT),
+        ExtraFieldDefaultHelper(field_name='minimum'),
+        ExtraFieldDefaultHelper(field_name='maximum'),
+        ExtraFieldDefaultHelper(field_name='step'),
+        ExtraFieldDefaultHelper(field_name='unit', value=''),
+        ExtraFieldDefaultHelper(field_name='width'),
+    ),
+    m2m_instance_fields=('conditions', 'optionsets'),
+    reverse_m2m_through_instance_fields=[
+        ThroughInstanceMapper(
+            field_name='page', source_name='question',
+            target_name='page', through_name='question_pages'
+        ),
+        ThroughInstanceMapper(
+            field_name='questionset', source_name='question',
+            target_name='questionset', through_name='question_questionsets'
+        )
+    ]
+)
 import_helper_questionset = ElementImportHelper(
     model = QuestionSet,
     model_path="questions.questionset",
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
-    extra_fields = ('is_collection',),
+    extra_fields=(
+        ExtraFieldDefaultHelper(field_name='is_collection'),
+    ),
     m2m_instance_fields=('conditions', ),
 
     m2m_through_instance_fields=[
@@ -104,25 +142,3 @@
         )
     ]
 )
-
-import_helper_question = ElementImportHelper(
-    model=Question,
-    model_path="questions.question",
-    validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
-    lang_fields=('text', 'help', 'default_text', 'verbose_name'),
-    foreign_fields=('attribute', 'default_option'),
-    extra_fields=('is_collection', 'is_optional', 'default_external_id',
-                  'widget_type', 'value_type', 'maximum', 'minimum', 'step',
-                  'unit', 'width'),
-    m2m_instance_fields=('conditions', 'optionsets'),
-    reverse_m2m_through_instance_fields=[
-        ThroughInstanceMapper(
-            field_name='page', source_name='question',
-            target_name='page', through_name='question_pages'
-        ),
-        ThroughInstanceMapper(
-            field_name='questionset', source_name='question',
-            target_name='questionset', through_name='question_questionsets'
-        )
-    ]
-)
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index ec33a31c49..92064efe78 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -1,5 +1,4 @@
-from rdmo.core.imports import ElementImportHelper
-
+from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
 from .models import Task
 from .validators import TaskLockedValidator, TaskUniqueURIValidator
 
@@ -9,6 +8,11 @@
     validators=(TaskLockedValidator, TaskUniqueURIValidator),
     lang_fields=('title', 'text'),
     foreign_fields=('start_attribute', 'end_attribute'),
-    extra_fields=('order', 'days_before', 'days_after', 'available'),
+    extra_fields=(
+        ExtraFieldDefaultHelper(field_name='order'),
+        ExtraFieldDefaultHelper(field_name='days_before'),
+        ExtraFieldDefaultHelper(field_name='days_after'),
+        ExtraFieldDefaultHelper(field_name='available'),
+    ),
     m2m_instance_fields=('catalogs', 'conditions'),
 )
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 084441f344..f515548746 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -1,5 +1,4 @@
-from rdmo.core.imports import ElementImportHelper
-
+from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
 from .models import View
 from .validators import ViewLockedValidator, ViewUniqueURIValidator
 
@@ -8,6 +7,10 @@
     model_path="views.view",
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
     lang_fields=('help', 'title'),
-    extra_fields=('order', 'template', 'available'),
+    extra_fields=(
+        ExtraFieldDefaultHelper(field_name='order'),
+        ExtraFieldDefaultHelper(field_name='template'),
+        ExtraFieldDefaultHelper(field_name='available'),
+    ),
     m2m_instance_fields=('catalogs',),
 )

From 20bd8ef5abc2483ee2370a17de970ed1e16036d8 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 5 Apr 2024 17:56:50 +0200
Subject: [PATCH 137/205] tests: refactor frontend import options test

---
 rdmo/management/tests/test_frontend_import_options.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
index 773a32a664..1a5048a7cf 100644
--- a/rdmo/management/tests/test_frontend_import_options.py
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -17,6 +17,7 @@
 test_users = [('editor', 'editor')]
 import_xml = "./testing/xml/elements/optionsets.xml"
 import_xml_1 = "./testing/xml/elements/updated-and-changed/optionsets-1.xml"
+OPTIONS_TOTAL_COUNT = 13
 
 @pytest.mark.parametrize("username, password", test_users)  # consumed by fixture
 def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> None:
@@ -35,15 +36,15 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     expect(page.get_by_text("Import from: optionsets.xml")).to_be_visible(timeout=30_000)
     ## TODO test if ImportInfo numbers are correct
     # test the components of the import-before-import staging page
-    expect(page.get_by_text("Created: 13")).to_be_visible(timeout=30_000)
+    expect(page.get_by_text(f"Created: {OPTIONS_TOTAL_COUNT}")).to_be_visible(timeout=30_000)
     page.locator(".element-link").first.click()
     page.get_by_role("link", name="Unselect all").click()
     page.get_by_role("link", name="Select all", exact=True).click()
     page.get_by_role("link", name="Show all", exact=True).click()
     rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")
-    expect(rows_displayed_in_ui).to_have_count(13)
+    expect(rows_displayed_in_ui).to_have_count(OPTIONS_TOTAL_COUNT)
     # click the import button to start saving the instances to the db
-    page.get_by_role("button", name="Import 13 elements").click()
+    page.get_by_role("button", name=f"Import {OPTIONS_TOTAL_COUNT} elements").click()
     expect(page.get_by_role("heading", name="Import successful")).to_be_visible()
     page.screenshot(path="screenshots/management-import-post-import.png", full_page=True)
     page.get_by_text("Created:").click()
@@ -61,7 +62,8 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     page.locator('#sidebar div.elements-sidebar form.upload-form.sidebar-form div.sidebar-form-button button.btn.btn-primary').click()  # noqa: E501
     expect(page.get_by_text("Import from: optionsets-1.xml")).to_be_visible(timeout=40_000)
     # assert changed elements
-    expect(page.get_by_text("Total: 13 Updated: 13 (Changed: 5) Warnings: 1")).to_be_visible(timeout=30_000)
+    expect(page.get_by_text(f"Total: {OPTIONS_TOTAL_COUNT} Updated: {OPTIONS_TOTAL_COUNT} (Changed: 5) Warnings: 1")).to_be_visible(timeout=30_000)  # noqa: E501
+
     expect(page.get_by_text("Filter changed (5)")).to_be_visible()
     page.get_by_text("Filter changed (5)").click()
     page.get_by_role("link", name="Show changes").click()

From ccfb8bd8231baa83880301f9a1f8ee6fd08c5f3d Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 9 Apr 2024 17:22:40 +0200
Subject: [PATCH 138/205] refactor: move element import diffs to react frontend

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py                          |  77 +++-----
 .../js/components/import/common/Fields.js     |   2 +-
 .../components/import/common/FieldsDiffs.js   |   4 +-
 .../assets/js/components/main/Import.js       | 171 +++++++++++++-----
 rdmo/management/imports.py                    |   8 +-
 .../tests/helpers_import_elements.py          |  35 +++-
 rdmo/management/tests/test_import_options.py  |  53 +++---
 7 files changed, 220 insertions(+), 130 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 0e52c60ef1..2e88d8ce0d 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -8,13 +8,10 @@
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models
-from django.utils.encoding import force_str
 from django.utils.translation import gettext_lazy as _
 
 from rest_framework.utils import model_meta
 
-from diff_match_patch import diff_match_patch
-
 from rdmo.core.constants import RDMO_MODELS
 from rdmo.core.import_helpers import ExtraFieldDefaultHelper
 from rdmo.core.utils import get_languages
@@ -25,6 +22,7 @@
 NEW_DATA_FIELD = "new_data"
 CURRENT_DATA_FIELD = "current_data"
 
+
 def handle_uploaded_file(filedata):
     tempfilename = generate_tempfile_name()
     with open(tempfilename, 'wb+') as destination:
@@ -47,7 +45,7 @@ def generate_tempfile_name():
     return fn
 
 
-def get_or_return_instance(model: models.Model, uri: Optional[str]=None) -> Tuple[models.Model, bool]:
+def get_or_return_instance(model: models.Model, uri: Optional[str] = None) -> Tuple[models.Model, bool]:
     if uri is None:
         return model(), True
     try:
@@ -57,7 +55,8 @@ def get_or_return_instance(model: models.Model, uri: Optional[str]=None) -> Tupl
     except model.MultipleObjectsReturned:
         return model.objects.filter(uri=uri).first(), False
 
-def get_rdmo_model_path(target_name:str, field_name: str):
+
+def get_rdmo_model_path(target_name: str, field_name: str):
     try:
         return RDMO_MODELS[target_name]
     except KeyError:
@@ -65,7 +64,7 @@ def get_rdmo_model_path(target_name:str, field_name: str):
             return RDMO_MODELS[field_name]
 
 
-def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str]=None):
+def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str] = None):
     if uri is None:
         return "%s, no uri" % verbose_name
     if created:
@@ -98,6 +97,7 @@ def track_messages_on_element(element: dict, element_field: str, warning: Option
         _initialize_tracking_field(element, element_field)
         _append_error(element, element_field, error)
 
+
 def _initialize_track_changes_element_field(element: dict, element_field: str):
     if ELEMENT_DIFF_FIELD_NAME not in element:
         element[ELEMENT_DIFF_FIELD_NAME] = {}
@@ -105,54 +105,35 @@ def _initialize_track_changes_element_field(element: dict, element_field: str):
     if element_field and element_field not in element[ELEMENT_DIFF_FIELD_NAME]:
         element[ELEMENT_DIFF_FIELD_NAME][element_field] = {}
 
+
 def _cast_list_of_string_to_list(list_of_strings: List[str]) -> str:
     return "\n".join(map(str, list_of_strings))
 
+
 def track_changes_on_element(element: dict,
                              element_field: str,
                              new_value: Union[str, List[str], None] = None,
                              instance_field: Optional[str] = None,
                              original=None,
                              original_value: Optional[Union[str, List[str]]] = None):
-    if original is None and original_value is None:
-        return
-    if new_value is None:
+    if (original is None and original_value is None) or new_value is None:
         return
 
     _initialize_track_changes_element_field(element, element_field)
 
-    # optional js prop for react-diff-viewer-continued
-    js_diff_viewer_props = {}
-
-    _get_field = element_field if instance_field is None else instance_field
     if original_value is None:
+        _get_field = element_field if instance_field is None else instance_field
         original_value = getattr(original, _get_field, '')
 
-    if isinstance(new_value, list) and isinstance(original_value, list):
-        # cast a list of strings with uris to a string with newlines
-        new_value = _cast_list_of_string_to_list(new_value)
-        original_value = _cast_list_of_string_to_list(original_value)
-        js_diff_viewer_props['hideLineNumbers'] = False
-        js_diff_viewer_props['splitView'] = False
-
-    new_value = force_str(new_value)
-    original_value = force_str(original_value)
-    dmp = diff_match_patch()
-    diff = dmp.diff_main(original_value, new_value)
-    dmp.diff_cleanupSemantic(diff)
-    changed: bool = any(i[0] != dmp.DIFF_EQUAL for i in diff)
-    #  TODO maybe rename updated to new
-    element[ELEMENT_DIFF_FIELD_NAME][element_field]['current'] = original_value
-    element[ELEMENT_DIFF_FIELD_NAME][element_field]['updated'] = new_value
-    element[ELEMENT_DIFF_FIELD_NAME][element_field]['changed'] = changed
-    element[ELEMENT_DIFF_FIELD_NAME][element_field].update(js_diff_viewer_props)
+    element[ELEMENT_DIFF_FIELD_NAME][element_field][CURRENT_DATA_FIELD] = original_value
+    element[ELEMENT_DIFF_FIELD_NAME][element_field][NEW_DATA_FIELD] = new_value
 
 
 def get_lang_field_values(field_name: str,
-                        element: Optional[dict] = None,
-                        instance: Optional[models.Model] = None,
-                        get_by_lang_field_key: bool = True):
-    if (element is not None and instance is not None):
+                          element: Optional[dict] = None,
+                          instance: Optional[models.Model] = None,
+                          get_by_lang_field_key: bool = True):
+    if element is not None and instance is not None:
         raise ValueError("Please choose one of each")
 
     ret = []
@@ -171,6 +152,7 @@ def get_lang_field_values(field_name: str,
         ret.append(row)
     return ret
 
+
 def set_lang_field(instance, field_name, element, original=None):
     languages_field_values = get_lang_field_values(field_name, element=element)
     for lang_fields_value in languages_field_values:
@@ -184,6 +166,7 @@ def set_lang_field(instance, field_name, element, original=None):
                                  original=original)
         setattr(instance, field_lang_name, field_value)
 
+
 def track_changes_on_uri_of_foreign_field(element, field_name, foreign_uri, original=None):
     if original is None:
         return
@@ -194,6 +177,7 @@ def track_changes_on_uri_of_foreign_field(element, field_name, foreign_uri, orig
         original_foreign_uri = getattr(original_foreign_instance, 'uri', '')
     track_changes_on_element(element, field_name, new_value=foreign_uri, original_value=original_foreign_uri)
 
+
 def set_foreign_field(instance, field_name, element, uploaded_uris=None, original=None) -> None:
     if field_name not in element:
         return
@@ -205,7 +189,8 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
         return
 
     if 'uri' not in foreign_element:
-        message = 'Foreign model can not be assigned on {instance_model}.{field_name} {instance_uri} due to missing uri.'.format( # noqa: E501
+        message = 'Foreign model can not be assigned on {instance_model}.{field_name} {instance_uri} due to missing uri.'.format(
+            # noqa: E501
             instance_model=instance._meta.object_name,
             instance_uri=element.get('uri'),
             field_name=field_name
@@ -237,25 +222,25 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
             setattr(instance, field_name, foreign_instance)
         _foreign_uri = foreign_uri if foreign_instance is not None else ""
         track_changes_on_uri_of_foreign_field(element,
-                                    field_name,
-                                    _foreign_uri,
-                                    original=original)
+                                              field_name,
+                                              _foreign_uri,
+                                              original=original)
     except ValueError:
-        message = '{foreign_model} {foreign_uri} can not be assigned on {instance_model}.{field_name} {instance_uri} .'.format( # noqa: E501
+        message = '{foreign_model} {foreign_uri} can not be assigned on {instance_model}.{field_name} {instance_uri} .'.format(
+            # noqa: E501
             foreign_model=foreign_model._meta.object_name,
             foreign_uri=foreign_uri,
             instance_model=instance._meta.object_name,
             instance_uri=element.get('uri'),
             field_name=field_name,
-            )
+        )
         logger.info(message)
         element['errors'][foreign_uri].append(message)
         track_messages_on_element(element, field_name, error=message)
 
 
 def set_extra_field(instance, field_name, element,
-                     extra_field_helper: Optional[ExtraFieldDefaultHelper]=None, original=None) -> None:
-
+                    extra_field_helper: Optional[ExtraFieldDefaultHelper] = None, original=None) -> None:
     element_value = element.get(field_name)
 
     extra_value = None
@@ -264,7 +249,7 @@ def set_extra_field(instance, field_name, element,
     elif extra_field_helper is not None:
         # default_value
         extra_value = extra_field_helper.get_default(instance=instance,
-                                                      key=field_name)
+                                                     key=field_name)
 
     if extra_value is not None:
         setattr(instance, field_name, extra_value)
@@ -306,7 +291,6 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
     _track_changes[NEW_DATA_FIELD] = []
     _track_changes[CURRENT_DATA_FIELD] = []
 
-
     if original is not None:
         try:
             for _order, orig_field_instance in enumerate(getattr(original, through_name).order_by()):
@@ -370,6 +354,7 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
     original_instance_data = sorted(_track_changes[CURRENT_DATA_FIELD], key=lambda k: k['order'])
     track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data)
 
+
 def track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data):
     _initialize_track_changes_element_field(element, field_name)
     element[ELEMENT_DIFF_FIELD_NAME][field_name][NEW_DATA_FIELD] = new_instance_data
@@ -416,8 +401,6 @@ def set_m2m_instances(instance, element, field_name, original=None, save=None):
                                 foreign_instances, original=original)
 
 
-
-
 def set_reverse_m2m_through_instance(instance, element, field_name=None, source_name=None,
                                      target_name=None, through_name=None,
                                      original=None, save=None) -> None:
diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index 7a57f2cd4c..b4e1762876 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -26,7 +26,7 @@ const excludeKeys = [
   'warnings',
   'updated_and_changed',
   'changed',
-  'changed_fields',
+  'changedFields',
 ]
 
 const Fields = ({ element }) => {
diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index 555588f5c3..f2ea354e47 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -11,8 +11,8 @@ const FieldsDiffs = ({ element, field }) => {
       return null
   }
   const fieldDiffData = element.updated_and_changed[field]
-  const newVal = fieldDiffData.updated ?? ''
-  const oldVal = fieldDiffData.current ?? ''
+  const newVal = fieldDiffData.newValue ?? ''
+  const oldVal = fieldDiffData.oldValue ?? ''
   const changed = fieldDiffData.changed ?? false
   const hideLineNumbers = fieldDiffData.hideLineNumbers ?? true
   const splitView = fieldDiffData.splitView ?? true
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index aa8dc3d259..31c3ad9019 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -1,6 +1,7 @@
 import React from 'react'
-import PropTypes from 'prop-types'
 import isEmpty from 'lodash/isEmpty'
+import { DiffMethod } from 'react-diff-viewer-continued'
+import PropTypes from 'prop-types'
 
 import ImportElement from '../import/ImportElement'
 import ImportSuccessElement from '../import/ImportSuccessElement'
@@ -10,82 +11,160 @@ import ImportInfo from '../import/common/ImportInfo'
 import ImportFilters from '../import/common/ImportFilters'
 import get from 'lodash/get'
 
-const Import = ({ config, imports, configActions, importActions }) => {
-  const { file, elements, success } = imports
+function getDiffData( diffData ) {
+    const OriginalValue = diffData.current_data || ''
+    const NewValue = diffData.new_data || ''
 
-  const updatedAndChangedElements = elements.filter(element => element.updated && element.changed)
+  let OriginalValueStr = OriginalValue
+  let NewValueStr = NewValue
+    if (Array.isArray(NewValue) && Array.isArray(OriginalValue)) {
+      // cast Array to String, joined by newline
+      OriginalValueStr = OriginalValue.join('\n')
+      NewValueStr = NewValue.join('\n')
+      console.log('isArray', NewValue, OriginalValueStr, NewValueStr)
+      diffData.hideLineNumbers = false
+      diffData.splitView = false
+      diffData.compareMethod = DiffMethod.LINES
+    }
+    else {
+      OriginalValueStr = OriginalValue.toString()
+      NewValueStr = NewValue.toString()
+    }
+    const equality = NewValueStr === OriginalValueStr
+    diffData.changed = !equality
+    diffData.newValue = NewValueStr
+    diffData.oldValue = OriginalValueStr
+    return diffData
+
+}
 
-  const updatedElements = elements.filter(element => element.updated)
-  const createdElements = elements.filter(element => element.created)
+function getDiffsForUpdatedElement( element )  {
+  let changedElement = false
+  let changedFields = []
+  Object.entries(element.updated_and_changed).sort().map(([key, diffData]) => {
+    const elementFieldDiff = getDiffData(diffData)
+    console.log('setDiffsAllFields', key, elementFieldDiff)
+    if (elementFieldDiff.changed ?? true) {
+      changedFields.push(key)
+      changedElement = true
+    }
+    element.updated_and_changed[key] = elementFieldDiff
+  })
+  element.changedFields = changedFields
+  element.changed = changedElement
+  // this.element.updated_and_changed = updatedAndChangedElement
+  return element
+}
 
-  const importWarnings = elements.filter(element => !isEmpty(element.warnings))
-  const importErrors = elements.filter(element => !isEmpty(element.errors))
 
-  const searchString = get(config, 'filter.import.elements.search', '')
-  const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
-  const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
+class ImportedElementsDiffsManager {
+  constructor( elements ) {
+    // Assign the RGB values as a property of `this`.
+    this.elementsImported = elements
+    this.elements = []
+    this.createdElements = []
+    this.updatedElements = []
+    this.changedElements = []
+    this.importWarnings = []
+    this.importErrors = []
+  }
+  setElementsDiff() {
+    this.elementsImported.forEach(elementRaw => {
+      const element = getDiffsForUpdatedElement(elementRaw)
+      this.elements.push(element)
+      if (element.updated ?? true) {
+        this.updatedElements.push(element)
+        if (element.changed ?? true) {
+          this.changedElements.push(element)
+        }
+      } else {
+        if (element.created ?? true) {
+          this.createdElements.push(element)
+          }
+        }
+      if (!isEmpty(element.warnings)) {
+        this.importWarnings.push(element)
+      }
+      if (!isEmpty(element.errors)) {
+        this.importErrors.push(element)
+      }
+      })
+    }
+}
 
-  // filter func callbacks
-  const filterByChanged = (elements, selectFilterChanged, updatedAndChangedElements) => {
-    if (selectFilterChanged === true && updatedAndChangedElements.length > 0) {
-    return updatedAndChangedElements
+function filterElementsByChanged (elements, selectFilterChanged) {
+    if (selectFilterChanged === true) {
+      return elements.filter((element) => element.changed)
   } else {
     return elements
   }}
-  const filterByUriSearch = (elements, searchString) => {
-    if (searchString) {
-      const lowercaseSearch = searchString.toLowerCase()
-      return elements.filter((element) =>
-        element.uri.toLowerCase().includes(lowercaseSearch)
-          // || element.title.toLowerCase().includes(lowercaseSearch)
-      )
-    } else {
-      return elements
-    }
+
+function filterElementsByUri (elements, searchString) {
+  if (searchString) {
+    const lowercaseSearch = searchString.toLowerCase()
+    return elements.filter((element) =>
+      element.uri.toLowerCase().includes(lowercaseSearch)
+    )} else {
+    return elements
   }
-    const filterByUriPrefix = (elements, searchUriPrefix) => {
+}
+
+function filterElementsByUriPrefix(elements, searchUriPrefix) {
     if (searchUriPrefix) {
       return elements.filter((element) =>
         element.uri_prefix.toLowerCase().includes(searchUriPrefix)
-          // || element.title.toLowerCase().includes(lowercaseSearch)
-      )
-    } else {
-      return elements
+      )} else {
+        return elements
+      }
     }
-  }
 
-  const filteredElements = filterByUriSearch(
-                                  filterByUriPrefix(
-                                        filterByChanged(elements, selectFilterChanged, updatedAndChangedElements),
-                                  selectedUriPrefix),
-                          searchString)
+function filterElements(elements, selectFilterChanged, selectedUriPrefix, searchString) {
+  const filteredElements = filterElementsByUri(
+    filterElementsByUriPrefix(
+      filterElementsByChanged(elements, selectFilterChanged),
+      selectedUriPrefix),
+    searchString)
+  return filteredElements
+}
+
+const Import = ({ config, imports, configActions, importActions }) => {
+  const { file, elements, success } = imports
+
+  const elementsDiff = new ImportedElementsDiffsManager(elements)
+  elementsDiff.setElementsDiff()
+
+  const searchString = get(config, 'filter.import.elements.search', '')
+  const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
+  const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
+
+  const filteredElements = filterElements(elements, selectFilterChanged, selectedUriPrefix, searchString)
 
   return (
     <div className="panel panel-default panel-import">
       <div className="panel-heading">
         <strong>{gettext('Import')} from: {file.name}</strong>
-        <ImportInfo elementsLength={elements.length} createdLength={createdElements.length}
-                        updatedLength={updatedElements.length} changedLength={updatedAndChangedElements.length}
-                        warningsLength={importWarnings.length} errorsLength={importErrors.length}/>
+        <ImportInfo elementsLength={elementsDiff.elements.length} createdLength={elementsDiff.createdElements.length}
+                        updatedLength={elementsDiff.updatedElements.length} changedLength={elementsDiff.changedElements.length}
+                        warningsLength={elementsDiff.importWarnings.length} errorsLength={elementsDiff.importErrors.length}/>
 
       </div>
         {
-          updatedAndChangedElements.length > 0 &&
-          <ImportFilters config={config} elements={elements}
-                         updatedAndChanged={updatedAndChangedElements}
+          elementsDiff.changedElements.length > 0 &&
+          <ImportFilters config={config} elements={elementsDiff.elements}
+                         updatedAndChanged={elementsDiff.changedElements}
                          filteredElements={filteredElements}
                          configActions={configActions}
           />
         }
 
         {
-        importWarnings.length > 0 &&
-          <ImportWarningsPanel config={config} elements={importWarnings} importActions={importActions}
+        elementsDiff.importWarnings.length > 0 &&
+          <ImportWarningsPanel config={config} elements={elementsDiff.importWarnings} importActions={importActions}
                                configActions={configActions}/>
         }
         {
-        importErrors.length > 0 &&
-          <ImportErrorsPanel config={config} elements={importErrors} importActions={importActions}
+        elementsDiff.importErrors.length > 0 &&
+          <ImportErrorsPanel config={config} elements={elementsDiff.importErrors} importActions={importActions}
                                configActions={configActions}/>
         }
         <ul className="list-group">
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 7bdcdffcb4..694f0b5ff9 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -58,8 +58,6 @@
         'errors': list,
         'created': bool,
         'updated': bool,
-        'changed': bool,
-        'changed_fields': list,
         ELEMENT_DIFF_FIELD_NAME: dict,
     }
 
@@ -171,8 +169,8 @@ def import_element(
             set_reverse_m2m_through_instance(instance, element, **asdict(reverse_m2m_fields),
                                              original=original, save=save)
     # set aggregated changes potentially to True and a list of changed fields
-    if _updated and element[ELEMENT_DIFF_FIELD_NAME]:
-        set_element_diff_field_meta_info(element)
+    # if _updated and element[ELEMENT_DIFF_FIELD_NAME]:
+    #     set_element_diff_field_meta_info(element)
 
     if save and settings.MULTISITE:
         # could be optimized with a bulk_create of through model later
@@ -192,7 +190,7 @@ def strip_uri_prefix_endswith_slash(element: dict) -> dict:
         element['uri_prefix'] = element['uri_prefix'].rstrip('/')
     return element
 
-def set_element_diff_field_meta_info(element: dict) -> None:
+def _set_element_diff_field_meta_info(element: dict) -> None:
     changed_fields = {k: val for k, val in element[ELEMENT_DIFF_FIELD_NAME].items() if val['changed']}
     element['changed'] = bool(changed_fields)
     element['changed_fields'] = list(changed_fields.keys())
diff --git a/rdmo/management/tests/helpers_import_elements.py b/rdmo/management/tests/helpers_import_elements.py
index 721383299b..2646098e1d 100644
--- a/rdmo/management/tests/helpers_import_elements.py
+++ b/rdmo/management/tests/helpers_import_elements.py
@@ -1,9 +1,9 @@
 from collections import OrderedDict
 from functools import partial
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, Union
 
-from rdmo.core.imports import track_changes_on_element
-from rdmo.management.imports import _initialize_import_element_dict, set_element_diff_field_meta_info
+from rdmo.core.imports import track_changes_on_element, ELEMENT_DIFF_FIELD_NAME, NEW_DATA_FIELD, CURRENT_DATA_FIELD
+from rdmo.management.imports import _initialize_import_element_dict
 
 UPDATE_FIELD_FUNCS = {
     'comment': lambda text: f"this is a test comment {text}",
@@ -13,9 +13,33 @@
 
 
 def filter_changed_fields(element, updated_fields=None) -> bool:
+    _changed = element.get('changed', False)
     if updated_fields is None:
-        return element.get('changed', False)
-    return element.get('changed', False) and any(i in updated_fields for i in element.get('changed_fields', []))
+        return _changed
+    changes = element.get(ELEMENT_DIFF_FIELD_NAME, {})
+    for field, diff in changes.items():
+        if field not in updated_fields:
+            continue
+        _new_value = diff.get(NEW_DATA_FIELD)
+        _current_value = diff.get(CURRENT_DATA_FIELD)
+        if _new_value != _current_value:
+            return True
+    return _changed
+
+def get_changed_elements(elements: List[Dict]) -> Dict[str, Dict[str,Union[bool,str]]]:
+    changed_elements = {}
+    for element in elements:
+
+        changed_fields = []
+        for key, diff_field in element[ELEMENT_DIFF_FIELD_NAME].items():
+            if diff_field[NEW_DATA_FIELD] != diff_field[CURRENT_DATA_FIELD]:
+                changed_fields += key
+        if changed_fields:
+            changed_elements[element['uri']] = {
+                'changed': bool(changed_fields),
+                'changed_fields': changed_fields,
+            }
+    return changed_elements
 
 
 def _test_helper_filter_updated_and_changed(elements: List[Dict], updated_fields: Optional[Tuple]) -> List[Dict]:
@@ -40,6 +64,5 @@ def _test_helper_change_fields_elements(elements,
                 new_val = UPDATE_FIELD_FUNCS[field](_n)
                 track_changes_on_element(_element, field, new_val, original_value=original_value)
                 _element[field] = new_val
-            set_element_diff_field_meta_info(_element)
         _new_elements[_uri] = _element
     return _new_elements
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 266d333b4d..9c763d4049 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -2,19 +2,31 @@
 
 import pytest
 
-from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
+from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME, NEW_DATA_FIELD, CURRENT_DATA_FIELD
 from rdmo.management.imports import import_elements
 from rdmo.options.models import Option, OptionSet
 
 from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
+    get_changed_elements
 )
 from .helpers_models import delete_all_objects
 from .helpers_xml import read_xml_and_parse_to_elements
 
 fields_to_be_changed = (('comment',),)
 
+test_optionset = {
+    'original': {
+        "uri": "http://example.com/terms/options/one_two_three",
+        "options": [
+            'http://example.com/terms/options/one_two_three/one',
+            'http://example.com/terms/options/one_two_three/two',
+            'http://example.com/terms/options/one_two_three/three',
+            ],
+        },
+    }
+
 def test_create_optionsets(db, settings):
     delete_all_objects([OptionSet, Option])
 
@@ -75,41 +87,36 @@ def test_update_optionsets_from_changed_xml(db, settings):
     xml_file_1 = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'updated-and-changed' / 'optionsets-1.xml'
     elements_1, root_1 = read_xml_and_parse_to_elements(xml_file_1)
     imported_elements_1 = import_elements(elements_1, save=False)
-
-    # Assert imported_elements_1
-    changed_elements = [i for i in imported_elements_1 if i['changed']]
+    assert imported_elements_1
+    assert [i for i in imported_elements_1 if i[ELEMENT_DIFF_FIELD_NAME]]
     warnings_elements = [i for i in imported_elements_1 if i['warnings']]
-    assert len(changed_elements) == 5
     assert len(warnings_elements) == 1
 
-    optionset_uri = "http://example.com/terms/options/one_two_three"
-    test_original_options = [
-        'http://example.com/terms/options/one_two_three/one',
-        'http://example.com/terms/options/one_two_three/two',
-        'http://example.com/terms/options/one_two_three/three',
-    ]
+    changed_elements = get_changed_elements(imported_elements_1)
+
+    assert test_optionset['original']['uri'] in changed_elements
+    assert len([i for i in  changed_elements.values() if i]) == 5
+
     # change the order of the options, as in the xml
-    test_changed_options = test_original_options[::-1]
-    _original_value = "\n".join(test_original_options)
-    _new_value = "\n".join(test_changed_options)
-    changed_uris = {i['uri']: i for i in changed_elements}
-    assert optionset_uri in changed_uris
-    changed_element = changed_uris[optionset_uri]
-    assert "options" in changed_element['changed_fields']
-    assert changed_element[ELEMENT_DIFF_FIELD_NAME]['options']['current'] == _original_value
-    assert changed_element[ELEMENT_DIFF_FIELD_NAME]['options']['updated'] == _new_value
+    optionset_element = [i for i in imported_elements_1 if i['uri'] == test_optionset['original']['uri']][0]
+    test_optionset_changed_options = test_optionset['original']['options'][::-1]  # the test changes are simply the reversed order of the options
+    assert optionset_element
+    assert "options" in optionset_element[ELEMENT_DIFF_FIELD_NAME]
+    assert optionset_element[ELEMENT_DIFF_FIELD_NAME]['options'][CURRENT_DATA_FIELD] == test_optionset['original']['options']
+    assert optionset_element[ELEMENT_DIFF_FIELD_NAME]['options'][NEW_DATA_FIELD] == test_optionset_changed_options
 
     # now save the elements_1
     _imported_elements_1_save = import_elements(elements_1, save=True)
     # get the ordered options (via .optionset_options) for this optionset from the db
-    optionset_1 = OptionSet.objects.get(uri=optionset_uri)
+    optionset_1 = OptionSet.objects.get(uri=test_optionset['original']['uri'])
     optionset_1_options = optionset_1.optionset_options.order_by('order').values_list('option__uri',flat=True)
-    for _test, _db in zip(test_changed_options, optionset_1_options):
+    for _test, _db in zip(test_optionset_changed_options, optionset_1_options):
         assert _test == _db
 
     # Import again and test that there are no changes detected
     imported_elements_2 = import_elements(elements_1, save=False)
-    assert len([i for i in imported_elements_2 if i['changed']]) == 0
+    changed_elements_2 = get_changed_elements(imported_elements_2)
+    assert len(changed_elements_2) == 0
     assert len([i for i in imported_elements_2 if i['warnings']]) == 1
 
 

From 636647e21222292654a2565aae207c853ef3fed9 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 10 Apr 2024 09:26:03 +0200
Subject: [PATCH 139/205] style: make ruff and ignore long lines

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py                             |  6 ++----
 rdmo/management/tests/helpers_import_elements.py |  2 +-
 rdmo/management/tests/test_import_options.py     | 11 ++++++-----
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 2e88d8ce0d..1bc9658d30 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -189,8 +189,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
         return
 
     if 'uri' not in foreign_element:
-        message = 'Foreign model can not be assigned on {instance_model}.{field_name} {instance_uri} due to missing uri.'.format(
-            # noqa: E501
+        message = 'Foreign model can not be assigned on {instance_model}.{field_name} {instance_uri} due to missing uri.'.format(  # noqa: E501
             instance_model=instance._meta.object_name,
             instance_uri=element.get('uri'),
             field_name=field_name
@@ -226,8 +225,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
                                               _foreign_uri,
                                               original=original)
     except ValueError:
-        message = '{foreign_model} {foreign_uri} can not be assigned on {instance_model}.{field_name} {instance_uri} .'.format(
-            # noqa: E501
+        message = '{foreign_model} {foreign_uri} can not be assigned on {instance_model}.{field_name} {instance_uri} .'.format(  # noqa: E501
             foreign_model=foreign_model._meta.object_name,
             foreign_uri=foreign_uri,
             instance_model=instance._meta.object_name,
diff --git a/rdmo/management/tests/helpers_import_elements.py b/rdmo/management/tests/helpers_import_elements.py
index 2646098e1d..53c9e6d649 100644
--- a/rdmo/management/tests/helpers_import_elements.py
+++ b/rdmo/management/tests/helpers_import_elements.py
@@ -2,7 +2,7 @@
 from functools import partial
 from typing import Dict, List, Optional, Tuple, Union
 
-from rdmo.core.imports import track_changes_on_element, ELEMENT_DIFF_FIELD_NAME, NEW_DATA_FIELD, CURRENT_DATA_FIELD
+from rdmo.core.imports import CURRENT_DATA_FIELD, ELEMENT_DIFF_FIELD_NAME, NEW_DATA_FIELD, track_changes_on_element
 from rdmo.management.imports import _initialize_import_element_dict
 
 UPDATE_FIELD_FUNCS = {
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 9c763d4049..d0b9a0aab0 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -2,14 +2,14 @@
 
 import pytest
 
-from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME, NEW_DATA_FIELD, CURRENT_DATA_FIELD
+from rdmo.core.imports import CURRENT_DATA_FIELD, ELEMENT_DIFF_FIELD_NAME, NEW_DATA_FIELD
 from rdmo.management.imports import import_elements
 from rdmo.options.models import Option, OptionSet
 
 from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
-    get_changed_elements
+    get_changed_elements,
 )
 from .helpers_models import delete_all_objects
 from .helpers_xml import read_xml_and_parse_to_elements
@@ -98,11 +98,12 @@ def test_update_optionsets_from_changed_xml(db, settings):
     assert len([i for i in  changed_elements.values() if i]) == 5
 
     # change the order of the options, as in the xml
-    optionset_element = [i for i in imported_elements_1 if i['uri'] == test_optionset['original']['uri']][0]
-    test_optionset_changed_options = test_optionset['original']['options'][::-1]  # the test changes are simply the reversed order of the options
+    optionset_element = next([i for i in imported_elements_1 if i['uri'] == test_optionset['original']['uri']])
+    # the test changes are simply the reversed order of the options
+    test_optionset_changed_options = test_optionset['original']['options'][::-1]
     assert optionset_element
     assert "options" in optionset_element[ELEMENT_DIFF_FIELD_NAME]
-    assert optionset_element[ELEMENT_DIFF_FIELD_NAME]['options'][CURRENT_DATA_FIELD] == test_optionset['original']['options']
+    assert optionset_element[ELEMENT_DIFF_FIELD_NAME]['options'][CURRENT_DATA_FIELD] == test_optionset['original']['options']  # noqa: E501
     assert optionset_element[ELEMENT_DIFF_FIELD_NAME]['options'][NEW_DATA_FIELD] == test_optionset_changed_options
 
     # now save the elements_1

From 7af9211daffaa4038abf28627b39a10025b08ea7 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 10 Apr 2024 15:19:01 +0200
Subject: [PATCH 140/205] tests: fix test iterator

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/test_import_options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index d0b9a0aab0..dd43823d1e 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -98,7 +98,7 @@ def test_update_optionsets_from_changed_xml(db, settings):
     assert len([i for i in  changed_elements.values() if i]) == 5
 
     # change the order of the options, as in the xml
-    optionset_element = next([i for i in imported_elements_1 if i['uri'] == test_optionset['original']['uri']])
+    optionset_element = next(filter(lambda x: x['uri'] == test_optionset['original']['uri'], imported_elements_1))
     # the test changes are simply the reversed order of the options
     test_optionset_changed_options = test_optionset['original']['options'][::-1]
     assert optionset_element

From 0ffa04111b6f5d4f7eef0573f8b555e9cd305755 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 11 Apr 2024 10:27:09 +0200
Subject: [PATCH 141/205] tests: rename e2e screenshot files

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/test_frontend_import_options.py   | 4 ++--
 rdmo/management/tests/test_frontend_import_questions.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
index 1a5048a7cf..3cd28045a1 100644
--- a/rdmo/management/tests/test_frontend_import_options.py
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -46,7 +46,7 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     # click the import button to start saving the instances to the db
     page.get_by_role("button", name=f"Import {OPTIONS_TOTAL_COUNT} elements").click()
     expect(page.get_by_role("heading", name="Import successful")).to_be_visible()
-    page.screenshot(path="screenshots/management-import-post-import.png", full_page=True)
+    page.screenshot(path="screenshots/management-import-optionsets-post-import.png", full_page=True)
     page.get_by_text("Created:").click()
     # go back to management page
     page.get_by_role("button", name="Back").click()
@@ -63,9 +63,9 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     expect(page.get_by_text("Import from: optionsets-1.xml")).to_be_visible(timeout=40_000)
     # assert changed elements
     expect(page.get_by_text(f"Total: {OPTIONS_TOTAL_COUNT} Updated: {OPTIONS_TOTAL_COUNT} (Changed: 5) Warnings: 1")).to_be_visible(timeout=30_000)  # noqa: E501
-
     expect(page.get_by_text("Filter changed (5)")).to_be_visible()
     page.get_by_text("Filter changed (5)").click()
     page.get_by_role("link", name="Show changes").click()
     expect(page.get_by_text("http://example.com/terms/options/one_two_three/three").nth(1)).to_be_visible()
+    page.screenshot(path="screenshots/management-import-optionsets-1-changes.png", full_page=True)
     ## TODO test for warnings, errors
diff --git a/rdmo/management/tests/test_frontend_import_questions.py b/rdmo/management/tests/test_frontend_import_questions.py
index 48a567e515..368634f26f 100644
--- a/rdmo/management/tests/test_frontend_import_questions.py
+++ b/rdmo/management/tests/test_frontend_import_questions.py
@@ -42,11 +42,11 @@ def test_import_catalogs_in_management(logged_in_user: Page) -> None:
     expect(rows_displayed_in_ui).to_have_count(148)
     page.get_by_role("link", name="Hide all").click()
     expect(rows_displayed_in_ui).to_have_count(0)
-    page.screenshot(path="screenshots/management-import-pre-import.png", full_page=True)
+    page.screenshot(path="screenshots/management-import-catalogs-pre.png", full_page=True)
     # click the import button to start saving the instances to the db
     page.get_by_role("button", name="Import 148 elements").click()
     expect(page.get_by_role("heading", name="Import successful")).to_be_visible()
-    page.screenshot(path="screenshots/management-import-post-import.png", full_page=True)
+    page.screenshot(path="screenshots/management-import-catalogs-post.png", full_page=True)
     page.get_by_text("Created:").click()
     # go back to management page
     page.get_by_role("button", name="Back").click()

From 0372a3abbe185124484b901ace82b46cafd4b396 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 26 Apr 2024 17:24:00 +0200
Subject: [PATCH 142/205] chore: fix errors on element and catch
 MultipleObjectsReturned

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 1bc9658d30..f77ed6d8a6 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -195,7 +195,8 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
             field_name=field_name
         )
         logger.info(message)
-        element['errors'][element.get('uri')].append(message)
+        # [element.get('uri')]
+        element['errors'].append(message)  # errors is a list
         track_messages_on_element(element, field_name, error=message)
         return
 
@@ -216,6 +217,17 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
         logger.info(message)
         element['warnings'][foreign_uri].append(message)
         track_messages_on_element(element, field_name, warning=message)
+    except foreign_model.MultipleObjectsReturned:
+        message = 'Multiple objects for {foreign_model} {foreign_uri} for {instance_model} {instance_uri} exist.'.format(    # noqa: E501
+            foreign_model=foreign_model._meta.object_name,
+            foreign_uri=foreign_uri,
+            instance_model=instance._meta.object_name,
+            instance_uri=element.get('uri')
+        )
+        logger.info(message)
+        element['errors'].append(message)  # errors is a list
+        track_messages_on_element(element, field_name, error=message)
+
     try:
         if foreign_instance is not None:
             setattr(instance, field_name, foreign_instance)
@@ -233,7 +245,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
             field_name=field_name,
         )
         logger.info(message)
-        element['errors'][foreign_uri].append(message)
+        element['errors'].append(message)
         track_messages_on_element(element, field_name, error=message)
 
 

From 26c4c3cfae677701061e484406b7f649a2d2c00e Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 26 Apr 2024 17:56:43 +0200
Subject: [PATCH 143/205] js: refactor Import diff compononent into hook and
 utils

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/main/Import.js       | 159 +++---------------
 .../assets/js/hooks/useImportElements.js      |  25 +++
 rdmo/management/assets/js/utils/filter.js     |   3 +-
 rdmo/management/assets/js/utils/getDiff.js    |  36 ++++
 .../assets/js/utils/importFilters.js          |  33 ++++
 .../assets/js/utils/processElementDiffs.js    |  33 ++++
 6 files changed, 154 insertions(+), 135 deletions(-)
 create mode 100644 rdmo/management/assets/js/hooks/useImportElements.js
 create mode 100644 rdmo/management/assets/js/utils/getDiff.js
 create mode 100644 rdmo/management/assets/js/utils/importFilters.js
 create mode 100644 rdmo/management/assets/js/utils/processElementDiffs.js

diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 31c3ad9019..8e4a3ca40f 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -1,7 +1,6 @@
 import React from 'react'
-import isEmpty from 'lodash/isEmpty'
-import { DiffMethod } from 'react-diff-viewer-continued'
 import PropTypes from 'prop-types'
+import get from 'lodash/get'
 
 import ImportElement from '../import/ImportElement'
 import ImportSuccessElement from '../import/ImportSuccessElement'
@@ -9,165 +8,57 @@ import ImportWarningsPanel from '../import/ImportWarningsPanel'
 import ImportErrorsPanel from '../import/ImportErrorsPanel'
 import ImportInfo from '../import/common/ImportInfo'
 import ImportFilters from '../import/common/ImportFilters'
-import get from 'lodash/get'
-
-function getDiffData( diffData ) {
-    const OriginalValue = diffData.current_data || ''
-    const NewValue = diffData.new_data || ''
-
-  let OriginalValueStr = OriginalValue
-  let NewValueStr = NewValue
-    if (Array.isArray(NewValue) && Array.isArray(OriginalValue)) {
-      // cast Array to String, joined by newline
-      OriginalValueStr = OriginalValue.join('\n')
-      NewValueStr = NewValue.join('\n')
-      console.log('isArray', NewValue, OriginalValueStr, NewValueStr)
-      diffData.hideLineNumbers = false
-      diffData.splitView = false
-      diffData.compareMethod = DiffMethod.LINES
-    }
-    else {
-      OriginalValueStr = OriginalValue.toString()
-      NewValueStr = NewValue.toString()
-    }
-    const equality = NewValueStr === OriginalValueStr
-    diffData.changed = !equality
-    diffData.newValue = NewValueStr
-    diffData.oldValue = OriginalValueStr
-    return diffData
-
-}
-
-function getDiffsForUpdatedElement( element )  {
-  let changedElement = false
-  let changedFields = []
-  Object.entries(element.updated_and_changed).sort().map(([key, diffData]) => {
-    const elementFieldDiff = getDiffData(diffData)
-    console.log('setDiffsAllFields', key, elementFieldDiff)
-    if (elementFieldDiff.changed ?? true) {
-      changedFields.push(key)
-      changedElement = true
-    }
-    element.updated_and_changed[key] = elementFieldDiff
-  })
-  element.changedFields = changedFields
-  element.changed = changedElement
-  // this.element.updated_and_changed = updatedAndChangedElement
-  return element
-}
+import useFilteredElements from '../../utils/importFilters'
+import {useImportElements} from '../../hooks/useImportElements'
 
 
-class ImportedElementsDiffsManager {
-  constructor( elements ) {
-    // Assign the RGB values as a property of `this`.
-    this.elementsImported = elements
-    this.elements = []
-    this.createdElements = []
-    this.updatedElements = []
-    this.changedElements = []
-    this.importWarnings = []
-    this.importErrors = []
-  }
-  setElementsDiff() {
-    this.elementsImported.forEach(elementRaw => {
-      const element = getDiffsForUpdatedElement(elementRaw)
-      this.elements.push(element)
-      if (element.updated ?? true) {
-        this.updatedElements.push(element)
-        if (element.changed ?? true) {
-          this.changedElements.push(element)
-        }
-      } else {
-        if (element.created ?? true) {
-          this.createdElements.push(element)
-          }
-        }
-      if (!isEmpty(element.warnings)) {
-        this.importWarnings.push(element)
-      }
-      if (!isEmpty(element.errors)) {
-        this.importErrors.push(element)
-      }
-      })
-    }
-}
-
-function filterElementsByChanged (elements, selectFilterChanged) {
-    if (selectFilterChanged === true) {
-      return elements.filter((element) => element.changed)
-  } else {
-    return elements
-  }}
-
-function filterElementsByUri (elements, searchString) {
-  if (searchString) {
-    const lowercaseSearch = searchString.toLowerCase()
-    return elements.filter((element) =>
-      element.uri.toLowerCase().includes(lowercaseSearch)
-    )} else {
-    return elements
-  }
-}
-
-function filterElementsByUriPrefix(elements, searchUriPrefix) {
-    if (searchUriPrefix) {
-      return elements.filter((element) =>
-        element.uri_prefix.toLowerCase().includes(searchUriPrefix)
-      )} else {
-        return elements
-      }
-    }
-
-function filterElements(elements, selectFilterChanged, selectedUriPrefix, searchString) {
-  const filteredElements = filterElementsByUri(
-    filterElementsByUriPrefix(
-      filterElementsByChanged(elements, selectFilterChanged),
-      selectedUriPrefix),
-    searchString)
-  return filteredElements
-}
-
 const Import = ({ config, imports, configActions, importActions }) => {
   const { file, elements, success } = imports
 
-  const elementsDiff = new ImportedElementsDiffsManager(elements)
-  elementsDiff.setElementsDiff()
+   const {
+    elementsImported,
+    createdElements,
+    updatedElements,
+    changedElements,
+    importWarnings,
+    importErrors
+  } = useImportElements(elements)
 
   const searchString = get(config, 'filter.import.elements.search', '')
   const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
   const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
 
-  const filteredElements = filterElements(elements, selectFilterChanged, selectedUriPrefix, searchString)
+  const filteredElements = useFilteredElements(elementsImported, selectFilterChanged, selectedUriPrefix, searchString)
 
   return (
-    <div className="panel panel-default panel-import">
-      <div className="panel-heading">
+    <div className='panel panel-default panel-import'>
+      <div className='panel-heading'>
         <strong>{gettext('Import')} from: {file.name}</strong>
-        <ImportInfo elementsLength={elementsDiff.elements.length} createdLength={elementsDiff.createdElements.length}
-                        updatedLength={elementsDiff.updatedElements.length} changedLength={elementsDiff.changedElements.length}
-                        warningsLength={elementsDiff.importWarnings.length} errorsLength={elementsDiff.importErrors.length}/>
+        <ImportInfo elementsLength={elementsImported.length} createdLength={createdElements.length}
+                        updatedLength={updatedElements.length} changedLength={changedElements.length}
+                        warningsLength={importWarnings.length} errorsLength={importErrors.length}/>
 
       </div>
         {
-          elementsDiff.changedElements.length > 0 &&
-          <ImportFilters config={config} elements={elementsDiff.elements}
-                         updatedAndChanged={elementsDiff.changedElements}
+          changedElements.length > 0 &&
+          <ImportFilters config={config} elements={elementsImported}
+                         updatedAndChanged={changedElements}
                          filteredElements={filteredElements}
                          configActions={configActions}
           />
         }
 
         {
-        elementsDiff.importWarnings.length > 0 &&
-          <ImportWarningsPanel config={config} elements={elementsDiff.importWarnings} importActions={importActions}
+        importWarnings.length > 0 &&
+          <ImportWarningsPanel config={config} elements={importWarnings} importActions={importActions}
                                configActions={configActions}/>
         }
         {
-        elementsDiff.importErrors.length > 0 &&
-          <ImportErrorsPanel config={config} elements={elementsDiff.importErrors} importActions={importActions}
+        importErrors.length > 0 &&
+          <ImportErrorsPanel config={config} elements={importErrors} importActions={importActions}
                                configActions={configActions}/>
         }
-        <ul className="list-group">
+        <ul className='list-group'>
           {
             filteredElements.map((element, index) => {
               if (success) {
diff --git a/rdmo/management/assets/js/hooks/useImportElements.js b/rdmo/management/assets/js/hooks/useImportElements.js
new file mode 100644
index 0000000000..aa2656e3d2
--- /dev/null
+++ b/rdmo/management/assets/js/hooks/useImportElements.js
@@ -0,0 +1,25 @@
+import {useMemo} from 'react'
+import isEmpty from 'lodash/isEmpty'
+import processElementDiffs from '../utils/processElementDiffs'
+
+
+export const useImportElements = (elements) => {
+  return useMemo(() => {
+    const elementsImported = elements.map(element => processElementDiffs(element))
+
+    const createdElements = elementsImported.filter(element => element.created)
+    const updatedElements = elementsImported.filter(element => element.updated)
+    const changedElements = updatedElements.filter(element => element.changed)
+    const importWarnings = elementsImported.filter(element => !isEmpty(element.warnings))
+    const importErrors = elementsImported.filter(element => !isEmpty(element.errors))
+
+    return {
+      elementsImported,
+      createdElements,
+      updatedElements,
+      changedElements,
+      importWarnings,
+      importErrors
+    }
+  }, [elements])
+}
diff --git a/rdmo/management/assets/js/utils/filter.js b/rdmo/management/assets/js/utils/filter.js
index 324d0c6fed..5b5a318810 100644
--- a/rdmo/management/assets/js/utils/filter.js
+++ b/rdmo/management/assets/js/utils/filter.js
@@ -37,6 +37,7 @@ const filterEditor = (editor, element) => {
   return isEmpty(editor) || element.editors.includes(toNumber(editor))
 }
 
+
 const getUriPrefixes = (elements) => {
   return elements.reduce((acc, cur) => {
     if (!acc.includes(cur.uri_prefix)) {
@@ -61,4 +62,4 @@ const getExportParams = (filter) => {
   return exportParams.toString()
 }
 
-export { filterElement, getUriPrefixes, getExportParams }
+export { filterElement, getUriPrefixes, getExportParams, filterUriPrefix, filterSearch }
diff --git a/rdmo/management/assets/js/utils/getDiff.js b/rdmo/management/assets/js/utils/getDiff.js
new file mode 100644
index 0000000000..868a1a0c5c
--- /dev/null
+++ b/rdmo/management/assets/js/utils/getDiff.js
@@ -0,0 +1,36 @@
+import { DiffMethod } from 'react-diff-viewer-continued'
+
+function getDiff(currentData, updatedData) {
+  let originalValueStr = currentData || ''
+  let newValueStr = updatedData || ''
+  let hideLineNumbers = true
+  let splitView = true
+  let compareMethod = DiffMethod.CHARS
+
+  if (Array.isArray(originalValueStr) && Array.isArray(newValueStr)) {
+    // Cast array to string, joined by newline
+    originalValueStr = originalValueStr.join('\n')
+    newValueStr = newValueStr.join('\n')
+    hideLineNumbers = false
+    splitView = false
+    compareMethod = DiffMethod.LINES
+  } else {
+    // Cast to string
+    originalValueStr = originalValueStr.toString()
+    newValueStr = newValueStr.toString()
+  }
+
+  const changed = newValueStr !== originalValueStr
+
+  // Return a structured object
+  return {
+    oldValue: originalValueStr,
+    newValue: newValueStr,
+    changed: changed,
+    hideLineNumbers: hideLineNumbers,
+    splitView: splitView,
+    compareMethod: compareMethod
+  }
+}
+
+export default getDiff
diff --git a/rdmo/management/assets/js/utils/importFilters.js b/rdmo/management/assets/js/utils/importFilters.js
new file mode 100644
index 0000000000..cb31c4be31
--- /dev/null
+++ b/rdmo/management/assets/js/utils/importFilters.js
@@ -0,0 +1,33 @@
+import { useMemo } from 'react'
+import { filterUriPrefix, filterSearch} from './filter'
+
+
+const filterChanged = (selectFilterChanged, element) => {
+  return selectFilterChanged ? element.changed : true
+}
+
+function filterElementsByChanged(elements, selectFilterChanged) {
+  if (!selectFilterChanged) return elements
+  return elements.filter((element) => filterChanged(selectFilterChanged, element))
+}
+
+function filterElementsByUri(elements, searchString) {
+  if (!searchString) return elements
+  return elements.filter((element) => filterSearch(searchString, element))
+}
+
+function filterElementsByUriPrefix(elements, searchUriPrefix) {
+  if (!searchUriPrefix) return elements
+  return elements.filter((element) => filterUriPrefix(searchUriPrefix, element))
+}
+
+const useFilteredElements = (elements, selectFilterChanged, selectedUriPrefix, searchString) => {
+  return useMemo(() => {
+    let filteredElements = filterElementsByChanged(elements, selectFilterChanged)
+    filteredElements = filterElementsByUriPrefix(filteredElements, selectedUriPrefix)
+    filteredElements = filterElementsByUri(filteredElements, searchString)
+    return filteredElements
+  }, [elements, selectFilterChanged, selectedUriPrefix, searchString])
+}
+
+export default useFilteredElements
diff --git a/rdmo/management/assets/js/utils/processElementDiffs.js b/rdmo/management/assets/js/utils/processElementDiffs.js
new file mode 100644
index 0000000000..4c8856ef27
--- /dev/null
+++ b/rdmo/management/assets/js/utils/processElementDiffs.js
@@ -0,0 +1,33 @@
+// utils/processElementDiffs.js
+import getDiff from './getDiff' // Make sure the path is correct
+
+function processElementDiffs(element) {
+  let changedElement = false
+  let changedFields = []
+
+  const updatedAndChanged = element.updated_and_changed
+
+  // Iterate over each field that might have changed
+  const updatedWithDiffs = Object.entries(updatedAndChanged).reduce((acc, [key, { current_data, new_data }]) => {
+    const elementFieldDiff = getDiff(current_data, new_data)
+
+    // Determine if the field has changed
+    if (elementFieldDiff.changed) {
+      changedFields.push(key)
+      changedElement = true
+    }
+
+    // Update the accumulator with new diff data
+    acc[key] = elementFieldDiff
+    return acc
+  }, {})
+
+  return {
+    ...element,
+    updated_and_changed: updatedWithDiffs,
+    changedFields: changedFields,
+    changed: changedElement
+  }
+}
+
+export default processElementDiffs

From b8df2d6c81b2756bec98e3ef03e9e6f6f30d75f3 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 26 Apr 2024 17:59:56 +0200
Subject: [PATCH 144/205] js draft: update sidebar

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/sidebar/ImportSidebar.js    | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
index a24fd81cce..26b844c2a7 100644
--- a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
+++ b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
@@ -4,13 +4,22 @@ import isEmpty from 'lodash/isEmpty'
 import isNil from 'lodash/isNil'
 
 import Link from 'rdmo/core/assets/js/components/Link'
-
+import {useImportElements} from '../../hooks/useImportElements'
 
 
 const ImportSidebar = ({ config, imports, importActions }) => {
   const { elements, success } = imports
+
+   const {
+    // elementsImported,
+    // createdElements,
+    // updatedElements,
+    changedElements,
+    // importWarnings,
+    // importErrors
+  } = useImportElements(elements)
+
   const count = elements.filter(e => e.import).length
-  const updatedAndChangedElements = elements.filter(element => element.updated && element.changed)
   const [uriPrefix, setUriPrefix] = useState('')
   const disabled = isNil(uriPrefix) || isEmpty(uriPrefix)
 
@@ -53,7 +62,7 @@ const ImportSidebar = ({ config, imports, importActions }) => {
               {gettext('Select all')}
             </Link>
           </li>
-            {updatedAndChangedElements.length > 0 &&
+            {changedElements.length > 0 &&
             <li>
               <ul className="list-unstyled" style={{paddingLeft:'20px'}}>
                 <li>
@@ -83,15 +92,17 @@ const ImportSidebar = ({ config, imports, importActions }) => {
               {gettext('Show all')}
             </Link>
           </li>
-          {updatedAndChangedElements.length > 0 &&
+          {changedElements.length > 0 &&
               <li>
                 <ul className="list-unstyled" style={{paddingLeft:'20px'}}>
                   <li>
+                    {/* TODO fix action showChangedElements */}
                     <Link onClick={() => importActions.showChangedElements(true)}>
                       {gettext('Show changes')}
                     </Link>
                   </li>
                   <li>
+                    {/* TODO fix action showChangedElements */}
                     <Link onClick={() => importActions.showChangedElements(false)}>
                       {gettext('Hide changes')}
                     </Link>

From b3fac446d28d1309c2ac45cb469b072bf8c7693f Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Apr 2024 14:44:15 +0200
Subject: [PATCH 145/205] chore: catch MultipleObjectsReturned at import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index f77ed6d8a6..25aa6255c8 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -218,15 +218,16 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
         element['warnings'][foreign_uri].append(message)
         track_messages_on_element(element, field_name, warning=message)
     except foreign_model.MultipleObjectsReturned:
-        message = 'Multiple objects for {foreign_model} {foreign_uri} for {instance_model} {instance_uri} exist.'.format(    # noqa: E501
+        message = '{foreign_model} {foreign_uri} for {instance_model} {instance_uri} returns multiple objects.'.format(
             foreign_model=foreign_model._meta.object_name,
             foreign_uri=foreign_uri,
             instance_model=instance._meta.object_name,
             instance_uri=element.get('uri')
         )
         logger.info(message)
-        element['errors'].append(message)  # errors is a list
-        track_messages_on_element(element, field_name, error=message)
+        element['warnings'][foreign_uri].append(message)
+        track_messages_on_element(element, field_name, warning=message)
+
 
     try:
         if foreign_instance is not None:
@@ -354,6 +355,16 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
             logger.info(message)
             element['warnings'][target_uri].append(message)
             track_messages_on_element(element, field_name, warning=message)
+        except target_model.MultipleObjectsReturned:
+            message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} returns multiple objects.'.format(  # noqa: E501
+                target_model=target_model._meta.object_name,
+                target_uri=target_uri,
+                instance_model=instance._meta.object_name,
+                instance_uri=element.get('uri')
+            )
+            logger.info(message)
+            element['warnings'][target_uri].append(message)
+            track_messages_on_element(element, field_name, warning=message)
     if save:
         # remove the remainders of the items list
         for through_instance in through_instances:
@@ -509,7 +520,7 @@ def validate_instance(instance, element, *validators):
 
     for validator in validators:
         try:
-            validator(instance if instance.id else None)(vars(instance))
+            validator(instance=instance if instance.id else None)(vars(instance))
         except ValidationError as e:
             try:
                 exception_message = format_message_from_validation_error(e)

From 043c4f5e8830db04dcf9e70f474ccd4780531566 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Apr 2024 14:47:21 +0200
Subject: [PATCH 146/205] tests frontent: add expect form-group and take
 screenshot before last expect

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/test_frontend_import_options.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
index 3cd28045a1..71b6c26812 100644
--- a/rdmo/management/tests/test_frontend_import_options.py
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -64,8 +64,9 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     # assert changed elements
     expect(page.get_by_text(f"Total: {OPTIONS_TOTAL_COUNT} Updated: {OPTIONS_TOTAL_COUNT} (Changed: 5) Warnings: 1")).to_be_visible(timeout=30_000)  # noqa: E501
     expect(page.get_by_text("Filter changed (5)")).to_be_visible()
-    page.get_by_text("Filter changed (5)").click()
     page.get_by_role("link", name="Show changes").click()
-    expect(page.get_by_text("http://example.com/terms/options/one_two_three/three").nth(1)).to_be_visible()
+    expect(page.locator(".col-sm-6 > .form-group").first).to_be_visible(timeout=30_000)
+    # take a screenshot of the import page
     page.screenshot(path="screenshots/management-import-optionsets-1-changes.png", full_page=True)
+    expect(page.get_by_text("http://example.com/terms/options/one_two_three/three").nth(1)).to_be_visible()
     ## TODO test for warnings, errors

From a04227f93ac6c6b945c4a25ea41248c1e191757c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Apr 2024 15:04:42 +0200
Subject: [PATCH 147/205] js: update ShowUpdatedLink

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/js/components/common/Links.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/rdmo/management/assets/js/components/common/Links.js b/rdmo/management/assets/js/components/common/Links.js
index ac5b57a6e4..1ef7e777ee 100644
--- a/rdmo/management/assets/js/components/common/Links.js
+++ b/rdmo/management/assets/js/components/common/Links.js
@@ -241,15 +241,16 @@ WarningLink.propTypes = {
   onClick: PropTypes.func.isRequired
 }
 
-const ShowUpdatedLink = ({ show= false, onClick }) => {
+const ShowUpdatedLink = ({ show= false, disabled= false, onClick }) => {
   return (
     show &&
-    <Link className="element-link fa fa-pencil" onClick={onClick} />
+    <Link  className='element-link fa fa-pencil' onClick={onClick} disabled={disabled}/>
   )
 }
 
 ShowUpdatedLink.propTypes = {
   show: PropTypes.bool,
+  disabled: PropTypes.bool,
   onClick: PropTypes.func.isRequired
 }
 

From 634fa5adaa861c087f11abbdb5908a04ca64df31 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Apr 2024 15:06:15 +0200
Subject: [PATCH 148/205] js: move process elements to importsReducer, update
 filters and sidebar

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/import/ImportElement.js  |  2 +-
 .../js/components/import/common/ImportFilters.js  | 10 +++++-----
 .../assets/js/components/main/Import.js           | 13 ++++++-------
 .../assets/js/components/sidebar/ImportSidebar.js |  2 --
 .../assets/js/hooks/useImportElements.js          | 15 ++++++---------
 .../assets/js/reducers/importsReducer.js          |  6 ++++--
 rdmo/management/assets/js/utils/getDiff.js        |  2 +-
 rdmo/management/assets/js/utils/importFilters.js  |  3 +--
 8 files changed, 24 insertions(+), 29 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index 7ed6da9ad2..63cad75348 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -20,7 +20,7 @@ const ImportElement = ({ config, element, importActions }) => {
       <div className="pull-right">
         <WarningLink show={!isEmpty(element.warnings)} onClick={updateShowField} />
         <ErrorLink show={!isEmpty(element.errors)} onClick={updateShowField} />
-        <ShowUpdatedLink show={(element.changed && !element.created)} onClick={updateShowField} />
+        <ShowUpdatedLink show={(element.changed && !element.created)} disabled={true} onClick={updateShowField} />
         <ShowLink show={element.show} onClick={updateShowField} />
 
       </div>
diff --git a/rdmo/management/assets/js/components/import/common/ImportFilters.js b/rdmo/management/assets/js/components/import/common/ImportFilters.js
index e0471b72b1..7e3dd4deb2 100644
--- a/rdmo/management/assets/js/components/import/common/ImportFilters.js
+++ b/rdmo/management/assets/js/components/import/common/ImportFilters.js
@@ -5,7 +5,7 @@ import get from 'lodash/get'
 import {getUriPrefixes} from '../../../utils/filter'
 import {Checkbox} from '../../common/Checkboxes'
 
-const ImportFilters = ({ config, elements, updatedAndChanged, filteredElements, configActions }) => {
+const ImportFilters = ({ config, elements, changedElements, filteredElements, configActions }) => {
   const updateFilterString = (value) => configActions.updateConfig('filter.import.elements.search', value)
   const getValueFilterString = () => get(config, 'filter.import.elements.search', '')
   const updateFilterUriPrefix = (value) => configActions.updateConfig('filter.import.elements.uri_prefix', value)
@@ -29,17 +29,17 @@ const ImportFilters = ({ config, elements, updatedAndChanged, filteredElements,
       </div>
       <div className="row-checkbox">
         {
-          updatedAndChanged.length > 0 &&
+          elements.length > 0 &&
           <div className="pull-left">
             <div className="checkboxes">
               <span className="mr-10">{gettext('Changed:')}</span>
               <Checkbox label={<code
-                className="code-questions">{gettext('Filter changed')}{' ('}{updatedAndChanged.length}{') '}</code>}
+                className="code-questions">{gettext('Filter changed')}{' ('}{changedElements.length}{') '}</code>}
                         value={getValueFilterChanged()} onChange={updateFilterChanged}/>
             </div>
           </div>
         }
-        { filteredElements.length > 0 &&
+        { elements.length > 0 &&
           <div className="pull-right">
             <span>{gettext('Shown')}: {filteredElements.length} / {elements.length} </span>
           </div>
@@ -55,7 +55,7 @@ const ImportFilters = ({ config, elements, updatedAndChanged, filteredElements,
 ImportFilters.propTypes = {
   config: PropTypes.object.isRequired,
   elements: PropTypes.array.isRequired,
-  updatedAndChanged: PropTypes.array.isRequired,
+  changedElements: PropTypes.array.isRequired,
   filteredElements: PropTypes.array.isRequired,
   configActions: PropTypes.object.isRequired,
 }
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 8e4a3ca40f..92e80c0a46 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -14,9 +14,8 @@ import {useImportElements} from '../../hooks/useImportElements'
 
 const Import = ({ config, imports, configActions, importActions }) => {
   const { file, elements, success } = imports
-
+    // the elements are already processed by processElementDiffs in the importsReducer
    const {
-    elementsImported,
     createdElements,
     updatedElements,
     changedElements,
@@ -28,21 +27,21 @@ const Import = ({ config, imports, configActions, importActions }) => {
   const selectedUriPrefix = get(config, 'filter.import.elements.uri_prefix', '')
   const selectFilterChanged = get(config, 'filter.import.elements.changed', false)
 
-  const filteredElements = useFilteredElements(elementsImported, selectFilterChanged, selectedUriPrefix, searchString)
+  const filteredElements = useFilteredElements(elements, selectFilterChanged, selectedUriPrefix, searchString)
 
   return (
     <div className='panel panel-default panel-import'>
       <div className='panel-heading'>
         <strong>{gettext('Import')} from: {file.name}</strong>
-        <ImportInfo elementsLength={elementsImported.length} createdLength={createdElements.length}
+        <ImportInfo elementsLength={elements.length} createdLength={createdElements.length}
                         updatedLength={updatedElements.length} changedLength={changedElements.length}
                         warningsLength={importWarnings.length} errorsLength={importErrors.length}/>
 
       </div>
         {
-          changedElements.length > 0 &&
-          <ImportFilters config={config} elements={elementsImported}
-                         updatedAndChanged={changedElements}
+          elements.length > 0 &&
+          <ImportFilters config={config} elements={elements}
+                         changedElements={changedElements}
                          filteredElements={filteredElements}
                          configActions={configActions}
           />
diff --git a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
index 26b844c2a7..2f8c6b5883 100644
--- a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
+++ b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
@@ -96,13 +96,11 @@ const ImportSidebar = ({ config, imports, importActions }) => {
               <li>
                 <ul className="list-unstyled" style={{paddingLeft:'20px'}}>
                   <li>
-                    {/* TODO fix action showChangedElements */}
                     <Link onClick={() => importActions.showChangedElements(true)}>
                       {gettext('Show changes')}
                     </Link>
                   </li>
                   <li>
-                    {/* TODO fix action showChangedElements */}
                     <Link onClick={() => importActions.showChangedElements(false)}>
                       {gettext('Hide changes')}
                     </Link>
diff --git a/rdmo/management/assets/js/hooks/useImportElements.js b/rdmo/management/assets/js/hooks/useImportElements.js
index aa2656e3d2..f00abc3ef3 100644
--- a/rdmo/management/assets/js/hooks/useImportElements.js
+++ b/rdmo/management/assets/js/hooks/useImportElements.js
@@ -1,20 +1,17 @@
 import {useMemo} from 'react'
 import isEmpty from 'lodash/isEmpty'
-import processElementDiffs from '../utils/processElementDiffs'
-
 
 export const useImportElements = (elements) => {
   return useMemo(() => {
-    const elementsImported = elements.map(element => processElementDiffs(element))
-
-    const createdElements = elementsImported.filter(element => element.created)
-    const updatedElements = elementsImported.filter(element => element.updated)
+    // the elements are already processed by processElementDiffs in the importsReducer
+    const createdElements = elements.filter(element => element.created)
+    const updatedElements = elements.filter(element => element.updated)
+    // collects elements with updated AND changed
     const changedElements = updatedElements.filter(element => element.changed)
-    const importWarnings = elementsImported.filter(element => !isEmpty(element.warnings))
-    const importErrors = elementsImported.filter(element => !isEmpty(element.errors))
+    const importWarnings = elements.filter(element => !isEmpty(element.warnings))
+    const importErrors = elements.filter(element => !isEmpty(element.errors))
 
     return {
-      elementsImported,
       createdElements,
       updatedElements,
       changedElements,
diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index 1826c1ecab..68f4a02e66 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -3,6 +3,7 @@ import isNil from 'lodash/isNil'
 import isUndefined from 'lodash/isUndefined'
 
 import { buildUri } from '../utils/elements'
+import processElementDiffs from '../utils/processElementDiffs'
 
 
 const initialState = {
@@ -24,6 +25,7 @@ export default function importsReducer(state = initialState, action) {
       return {...state, elements: [], errors: [], success: false}
     case 'import/uploadFileSuccess':
       return {...state, elements: action.elements.map(element => {
+        element = processElementDiffs(element)
         if (['questions.catalogs', 'tasks.task', 'views.view'].includes(element.model)) {
           element.available = true
         }
@@ -57,7 +59,7 @@ export default function importsReducer(state = initialState, action) {
       })}
     case 'import/selectChangedElements':
       return {...state, elements: state.elements.map(element => {
-        if (element.updated && element.changed && !element.created ) {
+        if (element.changed && !element.created ) {
           return {...element, import: action.value}
         }
         else if (action.value) {return {...element, import: !action.value}}
@@ -70,7 +72,7 @@ export default function importsReducer(state = initialState, action) {
       })}
     case 'import/showChangedElements':
       return {...state, elements: state.elements.map(element => {
-        if (element.updated && element.changed && !element.created ) {
+        if (element.changed && !element.created ) {
           return {...element, show: action.value}
         }
         else if (action.value) {return {...element, show: !action.value}}
diff --git a/rdmo/management/assets/js/utils/getDiff.js b/rdmo/management/assets/js/utils/getDiff.js
index 868a1a0c5c..6f671c1679 100644
--- a/rdmo/management/assets/js/utils/getDiff.js
+++ b/rdmo/management/assets/js/utils/getDiff.js
@@ -4,7 +4,7 @@ function getDiff(currentData, updatedData) {
   let originalValueStr = currentData || ''
   let newValueStr = updatedData || ''
   let hideLineNumbers = true
-  let splitView = true
+  let splitView = false
   let compareMethod = DiffMethod.CHARS
 
   if (Array.isArray(originalValueStr) && Array.isArray(newValueStr)) {
diff --git a/rdmo/management/assets/js/utils/importFilters.js b/rdmo/management/assets/js/utils/importFilters.js
index cb31c4be31..b97c5ce829 100644
--- a/rdmo/management/assets/js/utils/importFilters.js
+++ b/rdmo/management/assets/js/utils/importFilters.js
@@ -1,9 +1,8 @@
 import { useMemo } from 'react'
 import { filterUriPrefix, filterSearch} from './filter'
 
-
 const filterChanged = (selectFilterChanged, element) => {
-  return selectFilterChanged ? element.changed : true
+  return element.changed
 }
 
 function filterElementsByChanged(elements, selectFilterChanged) {

From e337d8e9b697fd95f3438d4bb6e5af32cce45b30 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Apr 2024 17:56:52 +0200
Subject: [PATCH 149/205] chore: add space to no permissions to import message

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 25aa6255c8..a390a18c75 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -554,6 +554,6 @@ def check_permissions(instance: models.Model, element_uri: str, user: models.Mod
         perms = [f'{app_label}.add_{model_name}_object']
 
     if not user.has_perms(perms, instance):
-        message = _('You have no permissions to import') + f'{instance._meta.object_name} {element_uri}.'
+        message = _('You have no permissions to import') + f' {instance._meta.object_name} {element_uri}.'
         logger.info(message)
         return message

From 337bfa6c6bc32427faf72fc0b69b9822fd3b9def Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 29 Apr 2024 17:58:35 +0200
Subject: [PATCH 150/205] tests frontent: fix floating navigation in screenshot

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/test_frontend_import_options.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
index 71b6c26812..aa5aa43179 100644
--- a/rdmo/management/tests/test_frontend_import_options.py
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -67,6 +67,8 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     page.get_by_role("link", name="Show changes").click()
     expect(page.locator(".col-sm-6 > .form-group").first).to_be_visible(timeout=30_000)
     # take a screenshot of the import page
-    page.screenshot(path="screenshots/management-import-optionsets-1-changes.png", full_page=True)
     expect(page.get_by_text("http://example.com/terms/options/one_two_three/three").nth(1)).to_be_visible()
+    page.locator("body").press("Home")
+    expect(page.get_by_role("link", name="Management", exact=True)).to_be_visible()
+    page.screenshot(path="screenshots/management-import-optionsets-1-changes.png", full_page=True)
     ## TODO test for warnings, errors

From 848680ff249a1982794c4b814d93d00f79f937b9 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 28 May 2024 10:07:59 +0200
Subject: [PATCH 151/205] build(npm): update package-lock.json

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 package-lock.json | 313 ++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 290 insertions(+), 23 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 861f0ce3a9..6f76f7e118 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -13,6 +13,7 @@
         "@uiw/react-codemirror": "^4.19.9",
         "bootstrap-sass": "^3.4.1",
         "classnames": "^2.3.2",
+        "date-fns": "^3.6.0",
         "font-awesome": "4.7.0",
         "jquery": "^3.6.0",
         "js-cookie": "^2.2.1",
@@ -21,10 +22,12 @@
         "prop-types": "^15.7.2",
         "react": "^18.2.0",
         "react-bootstrap": "0.33.1",
+        "react-datepicker": "6.6.0",
         "react-diff-viewer-continued": "^3.3.1",
         "react-dnd": "^16.0.1",
         "react-dnd-html5-backend": "^16.0.1",
         "react-dom": "^18.2.0",
+        "react-dropzone": "^10.0.0",
         "react-redux": "^7.2.4",
         "react-select": "^5.7.0",
         "redux": "^4.1.1",
@@ -120,9 +123,9 @@
       }
     },
     "node_modules/@babel/compat-data": {
-      "version": "7.17.10",
-      "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.17.10.tgz",
-      "integrity": "sha512-GZt/TCsG70Ms19gfZO1tM4CVnXsPgEPBCpJu+Qz3L0LUDsY5nZqFZglIoPC1kIYOtNBZlrnFT+klg12vFGZXrw==",
+      "version": "7.24.6",
+      "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.24.6.tgz",
+      "integrity": "sha512-aC2DGhBq5eEdyXWqrDInSqQjO0k8xtPRf5YylULqx8MCd6jBtzqfta/3ETMRpuKIc5hyswfO80ObyA1MvkCcUQ==",
       "devOptional": true,
       "engines": {
         "node": ">=6.9.0"
@@ -132,6 +135,7 @@
       "version": "7.23.3",
       "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.23.3.tgz",
       "integrity": "sha512-Jg+msLuNuCJDyBvFv5+OKOUjWMZgd85bKjbICd3zWrKAo+bJ49HJufi7CQE0q0uR8NGyO6xkCACScNqyjHSZew==",
+      "devOptional": true,
       "dependencies": {
         "@ampproject/remapping": "^2.2.0",
         "@babel/code-frame": "^7.22.13",
@@ -160,12 +164,14 @@
     "node_modules/@babel/core/node_modules/convert-source-map": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
-      "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="
+      "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==",
+      "devOptional": true
     },
     "node_modules/@babel/generator": {
       "version": "7.23.4",
       "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.4.tgz",
       "integrity": "sha512-esuS49Cga3HcThFNebGhlgsrVLkvhqvYDTzgjfFFlHJcIfLe5jFmRRfCQ1KuBfc4Jrtn3ndLgKWAKjBE+IraYQ==",
+      "devOptional": true,
       "dependencies": {
         "@babel/types": "^7.23.4",
         "@jridgewell/gen-mapping": "^0.3.2",
@@ -180,6 +186,7 @@
       "version": "0.3.3",
       "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
       "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==",
+      "devOptional": true,
       "dependencies": {
         "@jridgewell/set-array": "^1.0.1",
         "@jridgewell/sourcemap-codec": "^1.4.10",
@@ -217,6 +224,7 @@
       "version": "7.22.15",
       "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz",
       "integrity": "sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw==",
+      "devOptional": true,
       "dependencies": {
         "@babel/compat-data": "^7.22.9",
         "@babel/helper-validator-option": "^7.22.15",
@@ -232,6 +240,7 @@
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
       "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+      "devOptional": true,
       "dependencies": {
         "yallist": "^3.0.2"
       }
@@ -239,7 +248,8 @@
     "node_modules/@babel/helper-compilation-targets/node_modules/yallist": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
-      "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
+      "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
+      "devOptional": true
     },
     "node_modules/@babel/helper-create-class-features-plugin": {
       "version": "7.22.15",
@@ -301,6 +311,7 @@
       "version": "7.22.20",
       "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz",
       "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==",
+      "devOptional": true,
       "engines": {
         "node": ">=6.9.0"
       }
@@ -309,6 +320,7 @@
       "version": "7.23.0",
       "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz",
       "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==",
+      "devOptional": true,
       "dependencies": {
         "@babel/template": "^7.22.15",
         "@babel/types": "^7.23.0"
@@ -321,6 +333,7 @@
       "version": "7.22.5",
       "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz",
       "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==",
+      "devOptional": true,
       "dependencies": {
         "@babel/types": "^7.22.5"
       },
@@ -355,6 +368,7 @@
       "version": "7.23.3",
       "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz",
       "integrity": "sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==",
+      "devOptional": true,
       "dependencies": {
         "@babel/helper-environment-visitor": "^7.22.20",
         "@babel/helper-module-imports": "^7.22.15",
@@ -385,6 +399,7 @@
       "version": "7.22.5",
       "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz",
       "integrity": "sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg==",
+      "dev": true,
       "engines": {
         "node": ">=6.9.0"
       }
@@ -427,6 +442,7 @@
       "version": "7.22.5",
       "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz",
       "integrity": "sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==",
+      "devOptional": true,
       "dependencies": {
         "@babel/types": "^7.22.5"
       },
@@ -450,6 +466,7 @@
       "version": "7.22.6",
       "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz",
       "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==",
+      "devOptional": true,
       "dependencies": {
         "@babel/types": "^7.22.5"
       },
@@ -477,6 +494,7 @@
       "version": "7.22.15",
       "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz",
       "integrity": "sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA==",
+      "devOptional": true,
       "engines": {
         "node": ">=6.9.0"
       }
@@ -499,6 +517,7 @@
       "version": "7.23.4",
       "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.23.4.tgz",
       "integrity": "sha512-HfcMizYz10cr3h29VqyfGL6ZWIjTwWfvYBMsBVGwpcbhNGe3wQ1ZXZRPzZoAHhd9OqHadHqjQ89iVKINXnbzuw==",
+      "devOptional": true,
       "dependencies": {
         "@babel/template": "^7.22.15",
         "@babel/traverse": "^7.23.4",
@@ -525,6 +544,7 @@
       "version": "7.23.4",
       "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.4.tgz",
       "integrity": "sha512-vf3Xna6UEprW+7t6EtOmFpHNAuxw3xqPZghy+brsnusscJRW5BMUzzHZc5ICjULee81WeUV2jjakG09MDglJXQ==",
+      "devOptional": true,
       "bin": {
         "parser": "bin/babel-parser.js"
       },
@@ -713,6 +733,7 @@
       "version": "7.23.3",
       "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.23.3.tgz",
       "integrity": "sha512-EB2MELswq55OHUoRZLGg/zC7QWUKfNLpE57m/S2yr1uEneIgsTgrSzXP3NXEsMkVn76OlaVVnzN+ugObuYGwhg==",
+      "dev": true,
       "dependencies": {
         "@babel/helper-plugin-utils": "^7.22.5"
       },
@@ -1839,6 +1860,7 @@
       "version": "7.22.15",
       "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",
       "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==",
+      "devOptional": true,
       "dependencies": {
         "@babel/code-frame": "^7.22.13",
         "@babel/parser": "^7.22.15",
@@ -1852,6 +1874,7 @@
       "version": "7.23.4",
       "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.4.tgz",
       "integrity": "sha512-IYM8wSUwunWTB6tFC2dkKZhxbIjHoWemdK+3f8/wq8aKhbUscxD5MX72ubd90fxvFknaLPeGw5ycU84V1obHJg==",
+      "devOptional": true,
       "dependencies": {
         "@babel/code-frame": "^7.23.4",
         "@babel/generator": "^7.23.4",
@@ -2240,6 +2263,37 @@
         "@floating-ui/core": "^1.2.1"
       }
     },
+    "node_modules/@floating-ui/react": {
+      "version": "0.26.16",
+      "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.26.16.tgz",
+      "integrity": "sha512-HEf43zxZNAI/E781QIVpYSF3K2VH4TTYZpqecjdsFkjsaU1EbaWcM++kw0HXFffj7gDUcBFevX8s0rQGQpxkow==",
+      "dependencies": {
+        "@floating-ui/react-dom": "^2.1.0",
+        "@floating-ui/utils": "^0.2.0",
+        "tabbable": "^6.0.0"
+      },
+      "peerDependencies": {
+        "react": ">=16.8.0",
+        "react-dom": ">=16.8.0"
+      }
+    },
+    "node_modules/@floating-ui/react-dom": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.0.tgz",
+      "integrity": "sha512-lNzj5EQmEKn5FFKc04+zasr09h/uX8RtJRNj5gUXsSQIXHVWTVh+hVAg1vOMCexkX8EgvemMvIFpQfkosnVNyA==",
+      "dependencies": {
+        "@floating-ui/dom": "^1.0.0"
+      },
+      "peerDependencies": {
+        "react": ">=16.8.0",
+        "react-dom": ">=16.8.0"
+      }
+    },
+    "node_modules/@floating-ui/utils": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.2.tgz",
+      "integrity": "sha512-J4yDIIthosAsRZ5CPYP/jQvUAQtlZTTD/4suA08/FEnlxqW3sKS9iAhgsa9VYLZ6vDHn/ixJgIqRQPotoBjxIw=="
+    },
     "node_modules/@humanwhocodes/config-array": {
       "version": "0.11.13",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz",
@@ -2290,6 +2344,7 @@
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz",
       "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==",
+      "devOptional": true,
       "engines": {
         "node": ">=6.0.0"
       }
@@ -2330,12 +2385,14 @@
     "node_modules/@jridgewell/sourcemap-codec": {
       "version": "1.4.15",
       "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
-      "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg=="
+      "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==",
+      "devOptional": true
     },
     "node_modules/@jridgewell/trace-mapping": {
       "version": "0.3.20",
       "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz",
       "integrity": "sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==",
+      "devOptional": true,
       "dependencies": {
         "@jridgewell/resolve-uri": "^3.1.0",
         "@jridgewell/sourcemap-codec": "^1.4.14"
@@ -3017,6 +3074,14 @@
         "has-symbols": "^1.0.3"
       }
     },
+    "node_modules/attr-accept": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.2.tgz",
+      "integrity": "sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg==",
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/available-typed-arrays": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
@@ -3154,6 +3219,7 @@
       "version": "4.22.1",
       "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.22.1.tgz",
       "integrity": "sha512-FEVc202+2iuClEhZhrWy6ZiAcRLvNMyYcxZ8raemul1DYVOVdFsbqckWLdsixQZCpJlwe77Z3UTalE7jsjnKfQ==",
+      "devOptional": true,
       "funding": [
         {
           "type": "opencollective",
@@ -3213,6 +3279,7 @@
       "version": "1.0.30001564",
       "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001564.tgz",
       "integrity": "sha512-DqAOf+rhof+6GVx1y+xzbFPeOumfQnhYzVnZD6LAXijR77yPtm9mfOcqOnT3mpnJiZVT+kwLAFnRlZcIz+c6bg==",
+      "devOptional": true,
       "funding": [
         {
           "type": "opencollective",
@@ -3296,6 +3363,14 @@
         "node": ">=6"
       }
     },
+    "node_modules/clsx": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
+      "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/codemirror": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-6.0.1.tgz",
@@ -3526,6 +3601,15 @@
       "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.11.tgz",
       "integrity": "sha512-sa6P2wJ+CAbgyy4KFssIb/JNMLxFvKF1pCYCSXS8ZMuqZnMsrxqI2E5sPyoTpxoPU/gVZMzr2zjOfg8GIZOMsw=="
     },
+    "node_modules/date-fns": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-3.6.0.tgz",
+      "integrity": "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/kossnocorp"
+      }
+    },
     "node_modules/debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -3638,7 +3722,8 @@
     "node_modules/electron-to-chromium": {
       "version": "1.4.592",
       "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.592.tgz",
-      "integrity": "sha512-D3NOkROIlF+d5ixnz7pAf3Lu/AuWpd6AYgI9O67GQXMXTcCP1gJQRotOq35eQy5Sb4hez33XH1YdTtILA7Udww=="
+      "integrity": "sha512-D3NOkROIlF+d5ixnz7pAf3Lu/AuWpd6AYgI9O67GQXMXTcCP1gJQRotOq35eQy5Sb4hez33XH1YdTtILA7Udww==",
+      "devOptional": true
     },
     "node_modules/emojis-list": {
       "version": "3.0.0",
@@ -4347,6 +4432,17 @@
         "url": "https://opencollective.com/webpack"
       }
     },
+    "node_modules/file-selector": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-0.1.19.tgz",
+      "integrity": "sha512-kCWw3+Aai8Uox+5tHCNgMFaUdgidxvMnLWO6fM5sZ0hA2wlHP5/DHGF0ECe84BiB95qdJbKNEJhWKVDvMN+JDQ==",
+      "dependencies": {
+        "tslib": "^2.0.1"
+      },
+      "engines": {
+        "node": ">= 10"
+      }
+    },
     "node_modules/fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -5758,7 +5854,8 @@
     "node_modules/node-releases": {
       "version": "2.0.13",
       "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.13.tgz",
-      "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ=="
+      "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==",
+      "devOptional": true
     },
     "node_modules/normalize-path": {
       "version": "3.0.0",
@@ -6266,6 +6363,22 @@
         "loose-envify": "^1.0.0"
       }
     },
+    "node_modules/react-datepicker": {
+      "version": "6.6.0",
+      "resolved": "https://registry.npmjs.org/react-datepicker/-/react-datepicker-6.6.0.tgz",
+      "integrity": "sha512-ERC0/Q4pPC9bNIcGUpdCbHc+oCxhkU3WI3UOGHkyJ3A9fqALCYpEmLc5S5xvAd7DuCDdbsyW97oRPM6pWWwjww==",
+      "dependencies": {
+        "@floating-ui/react": "^0.26.2",
+        "clsx": "^2.1.0",
+        "date-fns": "^3.3.1",
+        "prop-types": "^15.7.2",
+        "react-onclickoutside": "^6.13.0"
+      },
+      "peerDependencies": {
+        "react": "^16.9.0 || ^17 || ^18",
+        "react-dom": "^16.9.0 || ^17 || ^18"
+      }
+    },
     "node_modules/react-diff-viewer-continued": {
       "version": "3.3.1",
       "resolved": "https://registry.npmjs.org/react-diff-viewer-continued/-/react-diff-viewer-continued-3.3.1.tgz",
@@ -6334,6 +6447,22 @@
         "react": "^18.2.0"
       }
     },
+    "node_modules/react-dropzone": {
+      "version": "10.2.2",
+      "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-10.2.2.tgz",
+      "integrity": "sha512-U5EKckXVt6IrEyhMMsgmHQiWTGLudhajPPG77KFSvgsMqNEHSyGpqWvOMc5+DhEah/vH4E1n+J5weBNLd5VtyA==",
+      "dependencies": {
+        "attr-accept": "^2.0.0",
+        "file-selector": "^0.1.12",
+        "prop-types": "^15.7.2"
+      },
+      "engines": {
+        "node": ">= 8"
+      },
+      "peerDependencies": {
+        "react": ">= 16.8"
+      }
+    },
     "node_modules/react-is": {
       "version": "16.13.1",
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
@@ -6344,6 +6473,19 @@
       "resolved": "https://registry.npmjs.org/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz",
       "integrity": "sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA=="
     },
+    "node_modules/react-onclickoutside": {
+      "version": "6.13.1",
+      "resolved": "https://registry.npmjs.org/react-onclickoutside/-/react-onclickoutside-6.13.1.tgz",
+      "integrity": "sha512-LdrrxK/Yh9zbBQdFbMTXPp3dTSN9B+9YJQucdDu3JNKRrbdU+H+/TVONJoWtOwy4II8Sqf1y/DTI6w/vGPYW0w==",
+      "funding": {
+        "type": "individual",
+        "url": "https://github.com/Pomax/react-onclickoutside/blob/master/FUNDING.md"
+      },
+      "peerDependencies": {
+        "react": "^15.5.x || ^16.x || ^17.x || ^18.x",
+        "react-dom": "^15.5.x || ^16.x || ^17.x || ^18.x"
+      }
+    },
     "node_modules/react-overlays": {
       "version": "0.9.3",
       "resolved": "https://registry.npmjs.org/react-overlays/-/react-overlays-0.9.3.tgz",
@@ -7131,6 +7273,11 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/tabbable": {
+      "version": "6.2.0",
+      "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz",
+      "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew=="
+    },
     "node_modules/tapable": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
@@ -7242,6 +7389,11 @@
         "node": ">=8.0"
       }
     },
+    "node_modules/tslib": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
+      "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
+    },
     "node_modules/type-check": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
@@ -7404,6 +7556,7 @@
       "version": "1.0.13",
       "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz",
       "integrity": "sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==",
+      "devOptional": true,
       "funding": [
         {
           "type": "opencollective",
@@ -7807,14 +7960,16 @@
       }
     },
     "@babel/compat-data": {
-      "version": "7.23.3",
-      "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.23.3.tgz",
-      "integrity": "sha512-BmR4bWbDIoFJmJ9z2cZ8Gmm2MXgEDgjdWgpKmKWUt54UGFJdlj31ECtbaDvCG/qVdG3AQ1SfpZEs01lUFbzLOQ=="
+      "version": "7.24.6",
+      "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.24.6.tgz",
+      "integrity": "sha512-aC2DGhBq5eEdyXWqrDInSqQjO0k8xtPRf5YylULqx8MCd6jBtzqfta/3ETMRpuKIc5hyswfO80ObyA1MvkCcUQ==",
+      "devOptional": true
     },
     "@babel/core": {
       "version": "7.23.3",
       "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.23.3.tgz",
       "integrity": "sha512-Jg+msLuNuCJDyBvFv5+OKOUjWMZgd85bKjbICd3zWrKAo+bJ49HJufi7CQE0q0uR8NGyO6xkCACScNqyjHSZew==",
+      "devOptional": true,
       "requires": {
         "@ampproject/remapping": "^2.2.0",
         "@babel/code-frame": "^7.22.13",
@@ -7836,7 +7991,8 @@
         "convert-source-map": {
           "version": "2.0.0",
           "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
-          "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="
+          "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==",
+          "devOptional": true
         }
       }
     },
@@ -7844,6 +8000,7 @@
       "version": "7.23.4",
       "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.4.tgz",
       "integrity": "sha512-esuS49Cga3HcThFNebGhlgsrVLkvhqvYDTzgjfFFlHJcIfLe5jFmRRfCQ1KuBfc4Jrtn3ndLgKWAKjBE+IraYQ==",
+      "devOptional": true,
       "requires": {
         "@babel/types": "^7.23.4",
         "@jridgewell/gen-mapping": "^0.3.2",
@@ -7855,6 +8012,7 @@
           "version": "0.3.3",
           "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
           "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==",
+          "devOptional": true,
           "requires": {
             "@jridgewell/set-array": "^1.0.1",
             "@jridgewell/sourcemap-codec": "^1.4.10",
@@ -7885,6 +8043,7 @@
       "version": "7.22.15",
       "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz",
       "integrity": "sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw==",
+      "devOptional": true,
       "requires": {
         "@babel/compat-data": "^7.22.9",
         "@babel/helper-validator-option": "^7.22.15",
@@ -7897,6 +8056,7 @@
           "version": "5.1.1",
           "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
           "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+          "devOptional": true,
           "requires": {
             "yallist": "^3.0.2"
           }
@@ -7904,7 +8064,8 @@
         "yallist": {
           "version": "3.1.1",
           "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
-          "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
+          "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
+          "devOptional": true
         }
       }
     },
@@ -7952,12 +8113,14 @@
     "@babel/helper-environment-visitor": {
       "version": "7.22.20",
       "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz",
-      "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA=="
+      "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==",
+      "devOptional": true
     },
     "@babel/helper-function-name": {
       "version": "7.23.0",
       "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz",
       "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==",
+      "devOptional": true,
       "requires": {
         "@babel/template": "^7.22.15",
         "@babel/types": "^7.23.0"
@@ -7967,6 +8130,7 @@
       "version": "7.22.5",
       "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz",
       "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==",
+      "devOptional": true,
       "requires": {
         "@babel/types": "^7.22.5"
       }
@@ -7992,6 +8156,7 @@
       "version": "7.23.3",
       "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz",
       "integrity": "sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==",
+      "devOptional": true,
       "requires": {
         "@babel/helper-environment-visitor": "^7.22.20",
         "@babel/helper-module-imports": "^7.22.15",
@@ -8012,7 +8177,8 @@
     "@babel/helper-plugin-utils": {
       "version": "7.22.5",
       "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz",
-      "integrity": "sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg=="
+      "integrity": "sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg==",
+      "dev": true
     },
     "@babel/helper-remap-async-to-generator": {
       "version": "7.22.20",
@@ -8040,6 +8206,7 @@
       "version": "7.22.5",
       "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz",
       "integrity": "sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==",
+      "devOptional": true,
       "requires": {
         "@babel/types": "^7.22.5"
       }
@@ -8057,6 +8224,7 @@
       "version": "7.22.6",
       "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz",
       "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==",
+      "devOptional": true,
       "requires": {
         "@babel/types": "^7.22.5"
       }
@@ -8074,7 +8242,8 @@
     "@babel/helper-validator-option": {
       "version": "7.22.15",
       "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz",
-      "integrity": "sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA=="
+      "integrity": "sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA==",
+      "devOptional": true
     },
     "@babel/helper-wrap-function": {
       "version": "7.22.20",
@@ -8091,6 +8260,7 @@
       "version": "7.23.4",
       "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.23.4.tgz",
       "integrity": "sha512-HfcMizYz10cr3h29VqyfGL6ZWIjTwWfvYBMsBVGwpcbhNGe3wQ1ZXZRPzZoAHhd9OqHadHqjQ89iVKINXnbzuw==",
+      "devOptional": true,
       "requires": {
         "@babel/template": "^7.22.15",
         "@babel/traverse": "^7.23.4",
@@ -8110,7 +8280,8 @@
     "@babel/parser": {
       "version": "7.23.4",
       "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.4.tgz",
-      "integrity": "sha512-vf3Xna6UEprW+7t6EtOmFpHNAuxw3xqPZghy+brsnusscJRW5BMUzzHZc5ICjULee81WeUV2jjakG09MDglJXQ=="
+      "integrity": "sha512-vf3Xna6UEprW+7t6EtOmFpHNAuxw3xqPZghy+brsnusscJRW5BMUzzHZc5ICjULee81WeUV2jjakG09MDglJXQ==",
+      "devOptional": true
     },
     "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": {
       "version": "7.23.3",
@@ -8234,6 +8405,7 @@
       "version": "7.23.3",
       "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.23.3.tgz",
       "integrity": "sha512-EB2MELswq55OHUoRZLGg/zC7QWUKfNLpE57m/S2yr1uEneIgsTgrSzXP3NXEsMkVn76OlaVVnzN+ugObuYGwhg==",
+      "dev": true,
       "requires": {
         "@babel/helper-plugin-utils": "^7.22.5"
       }
@@ -8985,6 +9157,7 @@
       "version": "7.22.15",
       "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",
       "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==",
+      "devOptional": true,
       "requires": {
         "@babel/code-frame": "^7.22.13",
         "@babel/parser": "^7.22.15",
@@ -8995,6 +9168,7 @@
       "version": "7.23.4",
       "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.4.tgz",
       "integrity": "sha512-IYM8wSUwunWTB6tFC2dkKZhxbIjHoWemdK+3f8/wq8aKhbUscxD5MX72ubd90fxvFknaLPeGw5ycU84V1obHJg==",
+      "devOptional": true,
       "requires": {
         "@babel/code-frame": "^7.23.4",
         "@babel/generator": "^7.23.4",
@@ -9325,6 +9499,29 @@
         "@floating-ui/core": "^1.2.1"
       }
     },
+    "@floating-ui/react": {
+      "version": "0.26.16",
+      "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.26.16.tgz",
+      "integrity": "sha512-HEf43zxZNAI/E781QIVpYSF3K2VH4TTYZpqecjdsFkjsaU1EbaWcM++kw0HXFffj7gDUcBFevX8s0rQGQpxkow==",
+      "requires": {
+        "@floating-ui/react-dom": "^2.1.0",
+        "@floating-ui/utils": "^0.2.0",
+        "tabbable": "^6.0.0"
+      }
+    },
+    "@floating-ui/react-dom": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.0.tgz",
+      "integrity": "sha512-lNzj5EQmEKn5FFKc04+zasr09h/uX8RtJRNj5gUXsSQIXHVWTVh+hVAg1vOMCexkX8EgvemMvIFpQfkosnVNyA==",
+      "requires": {
+        "@floating-ui/dom": "^1.0.0"
+      }
+    },
+    "@floating-ui/utils": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.2.tgz",
+      "integrity": "sha512-J4yDIIthosAsRZ5CPYP/jQvUAQtlZTTD/4suA08/FEnlxqW3sKS9iAhgsa9VYLZ6vDHn/ixJgIqRQPotoBjxIw=="
+    },
     "@humanwhocodes/config-array": {
       "version": "0.11.13",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz",
@@ -9361,7 +9558,8 @@
     "@jridgewell/resolve-uri": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz",
-      "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA=="
+      "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==",
+      "devOptional": true
     },
     "@jridgewell/set-array": {
       "version": "1.1.0",
@@ -9395,12 +9593,14 @@
     "@jridgewell/sourcemap-codec": {
       "version": "1.4.15",
       "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
-      "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg=="
+      "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==",
+      "devOptional": true
     },
     "@jridgewell/trace-mapping": {
       "version": "0.3.20",
       "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz",
       "integrity": "sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==",
+      "devOptional": true,
       "requires": {
         "@jridgewell/resolve-uri": "^3.1.0",
         "@jridgewell/sourcemap-codec": "^1.4.14"
@@ -9976,6 +10176,11 @@
         "has-symbols": "^1.0.3"
       }
     },
+    "attr-accept": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.2.tgz",
+      "integrity": "sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg=="
+    },
     "available-typed-arrays": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
@@ -10078,6 +10283,7 @@
       "version": "4.22.1",
       "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.22.1.tgz",
       "integrity": "sha512-FEVc202+2iuClEhZhrWy6ZiAcRLvNMyYcxZ8raemul1DYVOVdFsbqckWLdsixQZCpJlwe77Z3UTalE7jsjnKfQ==",
+      "devOptional": true,
       "requires": {
         "caniuse-lite": "^1.0.30001541",
         "electron-to-chromium": "^1.4.535",
@@ -10110,7 +10316,8 @@
     "caniuse-lite": {
       "version": "1.0.30001564",
       "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001564.tgz",
-      "integrity": "sha512-DqAOf+rhof+6GVx1y+xzbFPeOumfQnhYzVnZD6LAXijR77yPtm9mfOcqOnT3mpnJiZVT+kwLAFnRlZcIz+c6bg=="
+      "integrity": "sha512-DqAOf+rhof+6GVx1y+xzbFPeOumfQnhYzVnZD6LAXijR77yPtm9mfOcqOnT3mpnJiZVT+kwLAFnRlZcIz+c6bg==",
+      "devOptional": true
     },
     "chalk": {
       "version": "2.4.2",
@@ -10160,6 +10367,11 @@
         "shallow-clone": "^3.0.0"
       }
     },
+    "clsx": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
+      "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="
+    },
     "codemirror": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-6.0.1.tgz",
@@ -10337,6 +10549,11 @@
       "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.11.tgz",
       "integrity": "sha512-sa6P2wJ+CAbgyy4KFssIb/JNMLxFvKF1pCYCSXS8ZMuqZnMsrxqI2E5sPyoTpxoPU/gVZMzr2zjOfg8GIZOMsw=="
     },
+    "date-fns": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-3.6.0.tgz",
+      "integrity": "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww=="
+    },
     "debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -10423,7 +10640,8 @@
     "electron-to-chromium": {
       "version": "1.4.592",
       "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.592.tgz",
-      "integrity": "sha512-D3NOkROIlF+d5ixnz7pAf3Lu/AuWpd6AYgI9O67GQXMXTcCP1gJQRotOq35eQy5Sb4hez33XH1YdTtILA7Udww=="
+      "integrity": "sha512-D3NOkROIlF+d5ixnz7pAf3Lu/AuWpd6AYgI9O67GQXMXTcCP1gJQRotOq35eQy5Sb4hez33XH1YdTtILA7Udww==",
+      "devOptional": true
     },
     "emojis-list": {
       "version": "3.0.0",
@@ -10954,6 +11172,14 @@
         }
       }
     },
+    "file-selector": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-0.1.19.tgz",
+      "integrity": "sha512-kCWw3+Aai8Uox+5tHCNgMFaUdgidxvMnLWO6fM5sZ0hA2wlHP5/DHGF0ECe84BiB95qdJbKNEJhWKVDvMN+JDQ==",
+      "requires": {
+        "tslib": "^2.0.1"
+      }
+    },
     "fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -11712,7 +11938,8 @@
     "jsesc": {
       "version": "2.5.2",
       "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",
-      "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA=="
+      "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==",
+      "devOptional": true
     },
     "json-parse-even-better-errors": {
       "version": "2.3.1",
@@ -11969,7 +12196,8 @@
     "node-releases": {
       "version": "2.0.13",
       "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.13.tgz",
-      "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ=="
+      "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==",
+      "devOptional": true
     },
     "normalize-path": {
       "version": "3.0.0",
@@ -12321,6 +12549,18 @@
         }
       }
     },
+    "react-datepicker": {
+      "version": "6.6.0",
+      "resolved": "https://registry.npmjs.org/react-datepicker/-/react-datepicker-6.6.0.tgz",
+      "integrity": "sha512-ERC0/Q4pPC9bNIcGUpdCbHc+oCxhkU3WI3UOGHkyJ3A9fqALCYpEmLc5S5xvAd7DuCDdbsyW97oRPM6pWWwjww==",
+      "requires": {
+        "@floating-ui/react": "^0.26.2",
+        "clsx": "^2.1.0",
+        "date-fns": "^3.3.1",
+        "prop-types": "^15.7.2",
+        "react-onclickoutside": "^6.13.0"
+      }
+    },
     "react-diff-viewer-continued": {
       "version": "3.3.1",
       "resolved": "https://registry.npmjs.org/react-diff-viewer-continued/-/react-diff-viewer-continued-3.3.1.tgz",
@@ -12362,6 +12602,16 @@
         "scheduler": "^0.23.0"
       }
     },
+    "react-dropzone": {
+      "version": "10.2.2",
+      "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-10.2.2.tgz",
+      "integrity": "sha512-U5EKckXVt6IrEyhMMsgmHQiWTGLudhajPPG77KFSvgsMqNEHSyGpqWvOMc5+DhEah/vH4E1n+J5weBNLd5VtyA==",
+      "requires": {
+        "attr-accept": "^2.0.0",
+        "file-selector": "^0.1.12",
+        "prop-types": "^15.7.2"
+      }
+    },
     "react-is": {
       "version": "16.13.1",
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
@@ -12372,6 +12622,12 @@
       "resolved": "https://registry.npmjs.org/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz",
       "integrity": "sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA=="
     },
+    "react-onclickoutside": {
+      "version": "6.13.1",
+      "resolved": "https://registry.npmjs.org/react-onclickoutside/-/react-onclickoutside-6.13.1.tgz",
+      "integrity": "sha512-LdrrxK/Yh9zbBQdFbMTXPp3dTSN9B+9YJQucdDu3JNKRrbdU+H+/TVONJoWtOwy4II8Sqf1y/DTI6w/vGPYW0w==",
+      "requires": {}
+    },
     "react-overlays": {
       "version": "0.9.3",
       "resolved": "https://registry.npmjs.org/react-overlays/-/react-overlays-0.9.3.tgz",
@@ -12954,6 +13210,11 @@
       "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
       "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w=="
     },
+    "tabbable": {
+      "version": "6.2.0",
+      "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz",
+      "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew=="
+    },
     "tapable": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
@@ -13026,6 +13287,11 @@
         "is-number": "^7.0.0"
       }
     },
+    "tslib": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
+      "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
+    },
     "type-check": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
@@ -13143,6 +13409,7 @@
       "version": "1.0.13",
       "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz",
       "integrity": "sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==",
+      "devOptional": true,
       "requires": {
         "escalade": "^3.1.1",
         "picocolors": "^1.0.0"

From 7990fadd7a131a58c5e8f52669ca1db05d7068d1 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 28 May 2024 10:59:40 +0200
Subject: [PATCH 152/205] chore: fix style and minor changes

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/conditions/imports.py                    |  3 +-
 rdmo/core/constants.py                        |  6 ---
 rdmo/core/import_helpers.py                   |  7 +++-
 rdmo/core/imports.py                          |  5 +--
 rdmo/domain/imports.py                        |  3 +-
 .../js/components/sidebar/ImportSidebar.js    |  5 ---
 rdmo/management/imports.py                    | 38 +++++++++----------
 rdmo/options/imports.py                       |  3 +-
 rdmo/questions/imports.py                     |  3 +-
 rdmo/tasks/imports.py                         |  3 +-
 rdmo/views/imports.py                         |  3 +-
 11 files changed, 35 insertions(+), 44 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index 566f1e4df7..695b72ff44 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -1,4 +1,5 @@
-from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+
 from .models import Condition
 from .validators import ConditionLockedValidator, ConditionUniqueURIValidator
 
diff --git a/rdmo/core/constants.py b/rdmo/core/constants.py
index 80bf2e353a..7e96075a81 100644
--- a/rdmo/core/constants.py
+++ b/rdmo/core/constants.py
@@ -78,12 +78,6 @@
     "pib": {"base": 1024, "power": 5},
 }
 
-ELEMENT_COMMON_FIELDS = (
-    'uri_prefix',
-    'uri_path',
-    'comment',
-)
-
 RDMO_MODELS = {
   'catalog': 'questions.catalog',
   'section': 'questions.section',
diff --git a/rdmo/core/import_helpers.py b/rdmo/core/import_helpers.py
index d473fe8f0b..70af6a1eb1 100644
--- a/rdmo/core/import_helpers.py
+++ b/rdmo/core/import_helpers.py
@@ -4,8 +4,11 @@
 
 from django.db import models
 
-from rdmo.core.constants import ELEMENT_COMMON_FIELDS
-
+ELEMENT_COMMON_FIELDS = (
+    'uri_prefix',
+    'uri_path',
+    'comment',
+)
 
 @dataclass(frozen=True)
 class ThroughInstanceMapper:
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index a390a18c75..1ddd61bd1d 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -131,8 +131,7 @@ def track_changes_on_element(element: dict,
 
 def get_lang_field_values(field_name: str,
                           element: Optional[dict] = None,
-                          instance: Optional[models.Model] = None,
-                          get_by_lang_field_key: bool = True):
+                          instance: Optional[models.Model] = None):
     if element is not None and instance is not None:
         raise ValueError("Please choose one of each")
 
@@ -140,8 +139,6 @@ def get_lang_field_values(field_name: str,
     for lang_code, lang_verbose_name, lang_field in get_languages():
         name_code = f'{field_name}_{lang_code}'
         name_field = f'{field_name}_{lang_field}'
-        # get_key = name_field if get_by_lang_field_key else name_code
-        # set_key = name_code if get_by_lang_field_key else name_field
         row = {}
         row['element_key'] = name_code
         row['instance_field'] = name_field
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 0459a1dd8d..2a174a3830 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -1,7 +1,8 @@
 import logging
 from typing import Optional
 
-from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+
 from .models import Attribute
 from .validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
 
diff --git a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
index 2f8c6b5883..eaef0a7d46 100644
--- a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
+++ b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
@@ -11,12 +11,7 @@ const ImportSidebar = ({ config, imports, importActions }) => {
   const { elements, success } = imports
 
    const {
-    // elementsImported,
-    // createdElements,
-    // updatedElements,
     changedElements,
-    // importWarnings,
-    // importErrors
   } = useImportElements(elements)
 
   const count = elements.filter(e => e.import).length
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 694f0b5ff9..bffebdf3be 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -38,7 +38,6 @@
 logger = logging.getLogger(__name__)
 
 
-# mapping is redundant, since each ImportHelper has a .model_path attribute
 ELEMENT_IMPORT_HELPERS = {
     "conditions.condition": import_helper_condition,
     "domain.attribute": import_helper_attribute,
@@ -51,7 +50,7 @@
     "questions.question": import_helper_question,
     "tasks.task": import_helper_task,
     "views.view": import_helper_view
-    }
+}
 
 IMPORT_ELEMENT_INIT_DICT = {
         'warnings': lambda: defaultdict(list),
@@ -110,27 +109,27 @@ def import_element(
 
     uri = element.get('uri')
     # get or create instance from uri and model_path
-    instance, _created = get_or_return_instance(model, uri=uri)
+    instance, created = get_or_return_instance(model, uri=uri)
 
     # keep a copy of the original
     # when the element is updated
     # needs to be created here, else the changes will be overwritten
-    original = copy.deepcopy(instance) if not _created else None
+    original = copy.deepcopy(instance) if not created else None
 
     # prepare a log message
-    _msg = make_import_info_msg(model._meta.verbose_name, _created, uri=uri)
+    msg = make_import_info_msg(model._meta.verbose_name, created, uri=uri)
 
     # check the change or add permissions for the user on the instance
-    _perms_error_msg = check_permissions(instance, uri, user)
-    if _perms_error_msg:
-        # when there is an error msg, the import could be stopped and return
-        element["errors"].append(_perms_error_msg)
+    perms_error_msg = check_permissions(instance, uri, user)
+    if perms_error_msg:
+        # when there is an error msg, the import can be stopped and return
+        element["errors"].append(perms_error_msg)
         return element
 
-    _updated = not _created
-    element['created'] = _created
-    element['updated'] = _updated
-    # dict element[ELEMENT_DIFF_FIELD_NAME] is filled by tracking changes
+    updated = not created
+    element['created'] = created
+    element['updated'] = updated
+    # the dict element[ELEMENT_DIFF_FIELD_NAME] is filled by tracking changes
 
     element = strip_uri_prefix_endswith_slash(element)
     # start to set values on the instance
@@ -138,7 +137,7 @@ def import_element(
     for common_field in common_fields:
         common_value = element.get(common_field) or ''
         setattr(instance, common_field, common_value)
-        if _updated and original:
+        if updated and original:
             # track changes for common fields
             track_changes_on_element(element, common_field, new_value=common_value, original=original)
     # set language fields
@@ -154,12 +153,13 @@ def import_element(
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
+        # when there is an error msg, the import can be stopped and return
         return element
     if save:
-        logger.info(_msg)
+        logger.info(msg)
         instance.save()
-    # breakpoint()
-    if save or _updated:
+    if save or updated:
+        # this part updates the related fields of the instance
         for m2m_field in import_helper.m2m_instance_fields:
             set_m2m_instances(instance, element, m2m_field, original=original, save=save)
         for m2m_through_fields in import_helper.m2m_through_instance_fields:
@@ -168,12 +168,8 @@ def import_element(
         for reverse_m2m_fields in import_helper.reverse_m2m_through_instance_fields:
             set_reverse_m2m_through_instance(instance, element, **asdict(reverse_m2m_fields),
                                              original=original, save=save)
-    # set aggregated changes potentially to True and a list of changed fields
-    # if _updated and element[ELEMENT_DIFF_FIELD_NAME]:
-    #     set_element_diff_field_meta_info(element)
 
     if save and settings.MULTISITE:
-        # could be optimized with a bulk_create of through model later
         if import_helper.add_current_site_editors:
             instance.editors.add(current_site)
         if import_helper.add_current_site_sites:
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 086a2e1738..0ba6ca5344 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -1,4 +1,5 @@
-from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper, ThroughInstanceMapper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper, ThroughInstanceMapper
+
 from .models import Option, OptionSet
 from .validators import (
     OptionLockedValidator,
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 18cdcbfc15..f554a55c2a 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -1,5 +1,6 @@
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper, ThroughInstanceMapper
+
 from ..core.constants import VALUE_TYPE_TEXT
-from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper, ThroughInstanceMapper
 from .models import Catalog, Page, Question, QuestionSet, Section
 from .utils import get_widget_type_or_default
 from .validators import (
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 92064efe78..c2e205bf59 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -1,4 +1,5 @@
-from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+
 from .models import Task
 from .validators import TaskLockedValidator, TaskUniqueURIValidator
 
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index f515548746..8a2e3e4cbb 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -1,4 +1,5 @@
-from ..core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+
 from .models import View
 from .validators import ViewLockedValidator, ViewUniqueURIValidator
 

From 0302d6146d7d390b2b8c06ec459a3cdb2a8bdeca Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 28 May 2024 11:01:53 +0200
Subject: [PATCH 153/205] tests(style): make rdmo.core imports absolute

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/conditions/tests/test_viewset_condition_multisite.py  | 6 +++---
 rdmo/domain/tests/test_viewset_attribute_multisite.py      | 6 +++---
 rdmo/options/tests/test_viewset_options_multisite.py       | 6 +++---
 rdmo/options/tests/test_viewset_optionsets_multisite.py    | 6 +++---
 rdmo/questions/tests/test_viewset_catalog_multisite.py     | 6 +++---
 rdmo/questions/tests/test_viewset_page_multisite.py        | 6 +++---
 rdmo/questions/tests/test_viewset_question_multisite.py    | 6 +++---
 rdmo/questions/tests/test_viewset_questionset_multisite.py | 6 +++---
 rdmo/questions/tests/test_viewset_section_multisite.py     | 6 +++---
 rdmo/tasks/tests/test_viewset_task_multisite.py            | 6 +++---
 rdmo/views/tests/test_viewset_view_multisite.py            | 6 +++---
 11 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/rdmo/conditions/tests/test_viewset_condition_multisite.py b/rdmo/conditions/tests/test_viewset_condition_multisite.py
index 6d1f0da587..740485f09d 100644
--- a/rdmo/conditions/tests/test_viewset_condition_multisite.py
+++ b/rdmo/conditions/tests/test_viewset_condition_multisite.py
@@ -4,9 +4,9 @@
 
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import Condition
 from .test_viewset_condition import export_formats, urlnames
diff --git a/rdmo/domain/tests/test_viewset_attribute_multisite.py b/rdmo/domain/tests/test_viewset_attribute_multisite.py
index 39bd7a663e..f0be665e86 100644
--- a/rdmo/domain/tests/test_viewset_attribute_multisite.py
+++ b/rdmo/domain/tests/test_viewset_attribute_multisite.py
@@ -4,9 +4,9 @@
 
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import Attribute
 from .test_viewset_attribute import urlnames
diff --git a/rdmo/options/tests/test_viewset_options_multisite.py b/rdmo/options/tests/test_viewset_options_multisite.py
index 6834c6597e..a009332202 100644
--- a/rdmo/options/tests/test_viewset_options_multisite.py
+++ b/rdmo/options/tests/test_viewset_options_multisite.py
@@ -4,9 +4,9 @@
 
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import Option
 from .test_viewset_options import urlnames
diff --git a/rdmo/options/tests/test_viewset_optionsets_multisite.py b/rdmo/options/tests/test_viewset_optionsets_multisite.py
index 55bd2a30eb..2a1c2479d6 100644
--- a/rdmo/options/tests/test_viewset_optionsets_multisite.py
+++ b/rdmo/options/tests/test_viewset_optionsets_multisite.py
@@ -4,9 +4,9 @@
 
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import OptionSet
 from .test_viewset_optionsets import urlnames
diff --git a/rdmo/questions/tests/test_viewset_catalog_multisite.py b/rdmo/questions/tests/test_viewset_catalog_multisite.py
index 82bd691728..d648b2ae1a 100644
--- a/rdmo/questions/tests/test_viewset_catalog_multisite.py
+++ b/rdmo/questions/tests/test_viewset_catalog_multisite.py
@@ -5,9 +5,9 @@
 from django.contrib.sites.models import Site
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import Catalog
 from .test_viewset_catalog import export_formats, urlnames
diff --git a/rdmo/questions/tests/test_viewset_page_multisite.py b/rdmo/questions/tests/test_viewset_page_multisite.py
index 6bd56690a8..c62ade6061 100644
--- a/rdmo/questions/tests/test_viewset_page_multisite.py
+++ b/rdmo/questions/tests/test_viewset_page_multisite.py
@@ -5,9 +5,9 @@
 from django.db.models import Max
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import Page
 from .test_viewset_page import export_formats, urlnames
diff --git a/rdmo/questions/tests/test_viewset_question_multisite.py b/rdmo/questions/tests/test_viewset_question_multisite.py
index 73264498ed..3167055367 100644
--- a/rdmo/questions/tests/test_viewset_question_multisite.py
+++ b/rdmo/questions/tests/test_viewset_question_multisite.py
@@ -5,9 +5,9 @@
 from django.db.models import Max
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import Question
 from .test_viewset_question import export_formats, urlnames
diff --git a/rdmo/questions/tests/test_viewset_questionset_multisite.py b/rdmo/questions/tests/test_viewset_questionset_multisite.py
index 2a11a36c2c..5a1d00a075 100644
--- a/rdmo/questions/tests/test_viewset_questionset_multisite.py
+++ b/rdmo/questions/tests/test_viewset_questionset_multisite.py
@@ -5,9 +5,9 @@
 from django.db.models import Max
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import QuestionSet
 from .test_viewset_questionset import export_formats, urlnames
diff --git a/rdmo/questions/tests/test_viewset_section_multisite.py b/rdmo/questions/tests/test_viewset_section_multisite.py
index 203d783ef4..65a6ce1ad3 100644
--- a/rdmo/questions/tests/test_viewset_section_multisite.py
+++ b/rdmo/questions/tests/test_viewset_section_multisite.py
@@ -5,9 +5,9 @@
 from django.db.models import Max
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import Section
 from .test_viewset_section import export_formats, urlnames
diff --git a/rdmo/tasks/tests/test_viewset_task_multisite.py b/rdmo/tasks/tests/test_viewset_task_multisite.py
index 402adf9198..60e7263dff 100644
--- a/rdmo/tasks/tests/test_viewset_task_multisite.py
+++ b/rdmo/tasks/tests/test_viewset_task_multisite.py
@@ -5,9 +5,9 @@
 from django.contrib.sites.models import Site
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import Task
 from .test_viewset_task import export_formats, urlnames
diff --git a/rdmo/views/tests/test_viewset_view_multisite.py b/rdmo/views/tests/test_viewset_view_multisite.py
index bfe2c348db..1dcf1edf37 100644
--- a/rdmo/views/tests/test_viewset_view_multisite.py
+++ b/rdmo/views/tests/test_viewset_view_multisite.py
@@ -5,9 +5,9 @@
 from django.contrib.sites.models import Site
 from django.urls import reverse
 
-from rdmo.core.tests.constants import multisite_status_map as status_map
-from rdmo.core.tests.constants import multisite_users as users
-from rdmo.core.tests.utils import get_obj_perms_status_code
+from rdmo.core.tests import get_obj_perms_status_code
+from rdmo.core.tests import multisite_status_map as status_map
+from rdmo.core.tests import multisite_users as users
 
 from ..models import View
 from .test_viewset_view import export_formats, urlnames

From 48fc4d2e7c02cd7c29fa9ca634a5a0714a65a2b3 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 29 May 2024 18:34:15 +0200
Subject: [PATCH 154/205] chore(js): update styles and components for import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/ImportErrorsPanel.js | 10 ++--
 .../components/import/ImportSuccessElement.js | 11 ++++-
 .../js/components/import/common/Errors.js     | 49 +++++++++++++------
 .../components/import/common/FieldsDiffs.js   | 39 ++++++++++-----
 .../components/import/common/ImportFilters.js | 38 +++++---------
 .../js/components/import/common/ImportInfo.js | 14 +++---
 .../assets/js/components/main/Import.js       |  3 +-
 rdmo/management/assets/js/utils/getDiff.js    |  6 ---
 8 files changed, 98 insertions(+), 72 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportErrorsPanel.js b/rdmo/management/assets/js/components/import/ImportErrorsPanel.js
index 4f35381c4b..2f0911bbb1 100644
--- a/rdmo/management/assets/js/components/import/ImportErrorsPanel.js
+++ b/rdmo/management/assets/js/components/import/ImportErrorsPanel.js
@@ -5,18 +5,16 @@ import Errors from './common/Errors'
 import get from 'lodash/get'
 
 const ImportErrorsPanel = ({ config, elements, configActions }) => {
-
   const updateShowErrors = () => {
     const currentVal = get(config, 'filter.import.errors.show', false)
     configActions.updateConfig('filter.import.errors.show', !currentVal)
   }
 
   const showErrors = get(config, 'filter.import.errors.show', false)
-    const listErrors = elements.map((element, index) => {
-              return (<Errors key={index} element={element} />)
-              })
-  // const toggleImport = () => importActions.updateElement(element, {import: !element.import})
-  // const updateElement = (key, value) => importActions.updateElement(element, {[key]: value})
+  const listErrors = elements.map((element, index) => {
+            return (<Errors key={index} element={element} />)
+            })
+
   return (
     <div className="panel panel-danger">
       <div className="panel-heading"><strong>{gettext('Errors')}{' '}({elements.length}){' : '}</strong>
diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index d71be20d35..1b4c09a213 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -6,7 +6,16 @@ import { codeClass, verboseNames } from '../../constants/elements'
 import { isEmpty } from 'lodash'
 import Warnings from './common/Warnings'
 
+const prepareErrorsList = (errors) => {
+  // Filter out duplicate errors
+  const uniqueErrors = [...new Set(errors)]
+
+  return uniqueErrors.map(message => (
+        <p key={uniqueId()} className="text-danger">{message}</p>
+  ))
+}
 const ImportSuccessElement = ({ element }) => {
+  const listErrorMessages = prepareErrorsList(element.errors)
   return (
     <li className="list-group-item">
       <p>
@@ -25,7 +34,7 @@ const ImportSuccessElement = ({ element }) => {
         {'.'}
       </p>
         <Warnings element={element} showTitle={true}/>
-      {element.errors.map(message => <p key={uniqueId()} className="text-danger">{message}</p>)}
+      {listErrorMessages}
     </li>
   )
 }
diff --git a/rdmo/management/assets/js/components/import/common/Errors.js b/rdmo/management/assets/js/components/import/common/Errors.js
index 47fcd60d50..c711b846f7 100644
--- a/rdmo/management/assets/js/components/import/common/Errors.js
+++ b/rdmo/management/assets/js/components/import/common/Errors.js
@@ -1,25 +1,46 @@
 import React from 'react'
 import PropTypes from 'prop-types'
-import isEmpty from 'lodash/isEmpty'
 import uniqueId from 'lodash/uniqueId'
+import isEmpty from 'lodash/isEmpty'
 
-const Errors = ({ element }) => {
-  return !isEmpty(element.errors) && <div className="row text-danger mt-10">
-    <div className="col-sm-3 text-right">
-      {gettext('Errors')}
-    </div>
-    <div className="col-sm-9">
-      <ul className="list-unstyled">
-        {
-          element.errors.map(message => <li key={uniqueId('error-')}>{message}</li>)
-        }
-      </ul>
+// Helper function to generate error messages
+const generateErrorMessages = (messages, key) =>
+  messages.map(message => <li className="text-danger" key={key}>{message}</li>)
+
+// Helper function to prepare the list of errors
+const prepareErrorsList = (errors) => {
+  // Filter out duplicate errors
+  const uniqueErrors = [...new Set(errors)]
+
+  return uniqueErrors.map(message => (
+    <ul className="list-unstyled" key={uniqueId('error-list')}>
+      {generateErrorMessages([message], uniqueId('error-message'))}
+    </ul>
+  ))
+}
+
+const Errors = ({ element, showTitle = false }) => {
+  const listErrorMessages = prepareErrorsList(element.errors)
+
+  return !isEmpty(element.errors) && (
+    <div className="row text-danger mt-10">
+      {showTitle && (
+        <div className="col-sm-3 text-right">
+          {gettext('Errors')}
+        </div>
+      )}
+      <div className={`col-sm-${showTitle ? 9 : 12}`}>
+        <ul className="list-unstyled">
+          {listErrorMessages}
+        </ul>
+      </div>
     </div>
-  </div>
+  )
 }
 
 Errors.propTypes = {
-  element: PropTypes.object.isRequired
+  element: PropTypes.object.isRequired,
+  showTitle: PropTypes.bool,
 }
 
 export default Errors
diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
index f2ea354e47..e496144dbd 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
@@ -2,7 +2,6 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import isEmpty from 'lodash/isEmpty'
 import ReactDiffViewer from 'react-diff-viewer-continued'
-import { isUndefined } from 'lodash'
 import Warnings from './Warnings'
 import Errors from './Errors'
 
@@ -11,25 +10,39 @@ const FieldsDiffs = ({ element, field }) => {
       return null
   }
   const fieldDiffData = element.updated_and_changed[field]
-  const newVal = fieldDiffData.newValue ?? ''
-  const oldVal = fieldDiffData.oldValue ?? ''
+  const newVal = fieldDiffData.newValue.toString() ?? ''
+  const oldVal = fieldDiffData.oldValue.toString() ?? ''
   const changed = fieldDiffData.changed ?? false
-  const hideLineNumbers = fieldDiffData.hideLineNumbers ?? true
-  const splitView = fieldDiffData.splitView ?? true
-  const leftTitle = fieldDiffData.leftTitle ?? gettext('Current')
-  const rightTitle = fieldDiffData.rightTitle ?? gettext('Uploaded')
+  const splitView = false
+  const hideLineNumbers = true
+  // const leftTitle = fieldDiffData.leftTitle ?? gettext('Current')
+  // const rightTitle = fieldDiffData.rightTitle ?? gettext('Uploaded')
   const warnings = fieldDiffData.warnings ?? {}
   const errors = fieldDiffData.errors ?? []
 
-  return (changed && !isUndefined(newVal) && !isUndefined(oldVal) &&
-     <div className="col-sm-12">
+  const newStyles = {
+    variables: {
+      light: {
+        diffViewerBackground: '#fff',
+        changedBackground: '#fff',
+        gutterBackground: '#fff',
+      },
+    },
+    contentText: {
+      backgroundColor: '#fff !important',
+    },
+  }
+
+  return (changed &&
+     <div className="field-diff col-sm-12 mt-10 mb-10">
       <ReactDiffViewer
-          oldValue={oldVal.toString()}
-          newValue={newVal.toString()}
+          styles={newStyles}
+          oldValue={oldVal}
+          newValue={newVal}
           splitView={splitView}
           hideLineNumbers={hideLineNumbers}
-          leftTitle={leftTitle}
-          rightTitle={rightTitle}
+          // leftTitle={leftTitle}
+          // rightTitle={rightTitle}
           >
           </ReactDiffViewer>
       {
diff --git a/rdmo/management/assets/js/components/import/common/ImportFilters.js b/rdmo/management/assets/js/components/import/common/ImportFilters.js
index 7e3dd4deb2..bbda85fab9 100644
--- a/rdmo/management/assets/js/components/import/common/ImportFilters.js
+++ b/rdmo/management/assets/js/components/import/common/ImportFilters.js
@@ -14,8 +14,7 @@ const ImportFilters = ({ config, elements, changedElements, filteredElements, co
   const getValueFilterChanged = () => get(config, 'filter.import.elements.changed', false)
 
   return (
-    <div className="panel-body">
-    <div className="row">
+
       <div className="row">
         <div className={'col-sm-8'}>
           <FilterString value={getValueFilterString()} onChange={updateFilterString}
@@ -26,30 +25,19 @@ const ImportFilters = ({ config, elements, changedElements, filteredElements, co
                            onChange={updateFilterUriPrefix}
                            options={getUriPrefixes(elements)}/>
         </div>
-      </div>
-      <div className="row-checkbox">
-        {
-          elements.length > 0 &&
-          <div className="pull-left">
-            <div className="checkboxes">
-              <span className="mr-10">{gettext('Changed:')}</span>
-              <Checkbox label={<code
-                className="code-questions">{gettext('Filter changed')}{' ('}{changedElements.length}{') '}</code>}
-                        value={getValueFilterChanged()} onChange={updateFilterChanged}/>
-            </div>
-          </div>
-        }
-        { elements.length > 0 &&
-          <div className="pull-right">
-            <span>{gettext('Shown')}: {filteredElements.length} / {elements.length} </span>
-          </div>
-        }
 
-
-      </div>
-    </div>
-    </div>
-  )
+      {elements.length > 0 && (
+        <div className="horizontal-container">
+          <div className="checkboxes">
+            <Checkbox label={interpolate(gettext('Filter changed (%s)'), [changedElements.length])}
+                      value={getValueFilterChanged()} onChange={updateFilterChanged} />
+          </div>
+          <span className="shown-info">
+            {gettext('Shown')}: {filteredElements.length} / {elements.length}
+          </span>
+        </div>
+      )}
+  </div>)
 }
 
 ImportFilters.propTypes = {
diff --git a/rdmo/management/assets/js/components/import/common/ImportInfo.js b/rdmo/management/assets/js/components/import/common/ImportInfo.js
index 2a2a4c3ac2..4e711f76ad 100644
--- a/rdmo/management/assets/js/components/import/common/ImportInfo.js
+++ b/rdmo/management/assets/js/components/import/common/ImportInfo.js
@@ -2,9 +2,11 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import {isUndefined} from 'lodash'
 
-const renderElementLengthInfo = (label, length) => length > 0
-  && <span>{gettext(label)}: {length} </span>
-
+const renderElementLengthInfo = (label, length) => (
+  length > 0 && (
+    <span className='mr-5'>{gettext(label)}: {length} </span>
+  )
+)
 const ImportInfo = ({
                       elementsLength,
                       updatedLength,
@@ -20,9 +22,9 @@ const ImportInfo = ({
   return (
     <div className="pull-right">
       {renderElementLengthInfo('Total', elementsLength)}
-      {renderElementLengthInfo('updated', updatedLength)}
-      {changedLength > 0 && <span>{' ('}{gettext('changed')}{': '}{changedLength}{') '}</span>}
-      {renderElementLengthInfo('created', createdLength)}
+      {renderElementLengthInfo('Updated', updatedLength)}
+      {renderElementLengthInfo('Changed', changedLength)}
+      {renderElementLengthInfo('Created', createdLength)}
       {renderElementLengthInfo('Warnings', warningsLength)}
       {renderElementLengthInfo('Errors', errorsLength)}
     </div>
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 92e80c0a46..182a13760a 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -38,6 +38,7 @@ const Import = ({ config, imports, configActions, importActions }) => {
                         warningsLength={importWarnings.length} errorsLength={importErrors.length}/>
 
       </div>
+      <div className="panel-body">
         {
           elements.length > 0 &&
           <ImportFilters config={config} elements={elements}
@@ -46,7 +47,6 @@ const Import = ({ config, imports, configActions, importActions }) => {
                          configActions={configActions}
           />
         }
-
         {
         importWarnings.length > 0 &&
           <ImportWarningsPanel config={config} elements={importWarnings} importActions={importActions}
@@ -69,6 +69,7 @@ const Import = ({ config, imports, configActions, importActions }) => {
           }
         </ul>
     </div>
+  </div>
   )
 }
 
diff --git a/rdmo/management/assets/js/utils/getDiff.js b/rdmo/management/assets/js/utils/getDiff.js
index 6f671c1679..dcafde9f63 100644
--- a/rdmo/management/assets/js/utils/getDiff.js
+++ b/rdmo/management/assets/js/utils/getDiff.js
@@ -3,16 +3,12 @@ import { DiffMethod } from 'react-diff-viewer-continued'
 function getDiff(currentData, updatedData) {
   let originalValueStr = currentData || ''
   let newValueStr = updatedData || ''
-  let hideLineNumbers = true
-  let splitView = false
   let compareMethod = DiffMethod.CHARS
 
   if (Array.isArray(originalValueStr) && Array.isArray(newValueStr)) {
     // Cast array to string, joined by newline
     originalValueStr = originalValueStr.join('\n')
     newValueStr = newValueStr.join('\n')
-    hideLineNumbers = false
-    splitView = false
     compareMethod = DiffMethod.LINES
   } else {
     // Cast to string
@@ -27,8 +23,6 @@ function getDiff(currentData, updatedData) {
     oldValue: originalValueStr,
     newValue: newValueStr,
     changed: changed,
-    hideLineNumbers: hideLineNumbers,
-    splitView: splitView,
     compareMethod: compareMethod
   }
 }

From 33190de3f90c281b6d29971e1d873ff8557df4d7 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 29 May 2024 18:35:19 +0200
Subject: [PATCH 155/205] style(scss): add styles for import and field diffs

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/scss/management.scss | 39 +++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/rdmo/management/assets/scss/management.scss b/rdmo/management/assets/scss/management.scss
index 580107c1d2..a748d8421b 100644
--- a/rdmo/management/assets/scss/management.scss
+++ b/rdmo/management/assets/scss/management.scss
@@ -62,6 +62,16 @@ $icon-font-path: "bootstrap-sass/assets/fonts/bootstrap/";
                     font-weight: normal;
                 }
             }
+            .horizontal-container {
+              display: flex;
+              align-items: center;
+              justify-content: space-between;
+              padding: 6px 15px 6px;
+
+                .shown-info {
+                    margin-left: auto;
+                }
+            }
         }
     }
     .pull-right {
@@ -203,3 +213,32 @@ $icon-font-path: "bootstrap-sass/assets/fonts/bootstrap/";
         flex: 1;
     }
 }
+
+.field-diff {
+  // Target elements with classes containing "react-diff-" and "marker"
+    [class*="react-diff-"][class*="marker"] {
+        margin-top: 10px !important;
+        margin-bottom: 5px !important;
+
+        pre {
+            background-color: #fff !important;
+            font-size: 10px;
+            padding-top: 5px !important;
+            padding-bottom: 5px !important;
+        }
+    }
+
+    // Target elements with classes containing "react-diff-" and "content-text"
+    [class*="react-diff-"][class*="content-text"] {
+        font-size: 10px !important;
+        line-height: 12px !important;
+        margin-bottom: 5px !important;
+        margin-top: 5px !important;
+        padding-top: 10px !important;
+        padding-bottom: 10px !important;
+      }
+}
+
+//.list-group-item div.mt-10 div.row div.field-diff.col-sm-12.mt-10.mb-10 table.react-diff-1n5o7vh-diff-container tbody tr.react-diff-1n7ec1i-line td.react-diff-vl0irh-content.react-diff-1fxlvce-diff-removed pre.react-diff-16bq9gd-content-text
+//<span class="react-diff-42iver-word-diff">Lorem </span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">mu</span><span class="react-diff-42iver-word-diff">ps</span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">i</span><span class="react-diff-42iver-word-diff">m</span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">de</span><span class="react-diff-42iver-word-diff"> dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est.</span>
+//<pre class="react-diff-16bq9gd-content-text"><span class="react-diff-42iver-word-diff">Lorem </span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">mu</span><span class="react-diff-42iver-word-diff">ps</span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">i</span><span class="react-diff-42iver-word-diff">m</span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">de</span><span class="react-diff-42iver-word-diff"> dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est.</span></pre>

From f040226993eebf2f30d727edbca80a69ac0c75e7 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 3 Jun 2024 17:17:26 +0200
Subject: [PATCH 156/205] style(scss): update styles and selector for import
 field-diff elements

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/scss/management.scss | 43 ++++++++++-----------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/rdmo/management/assets/scss/management.scss b/rdmo/management/assets/scss/management.scss
index a748d8421b..21e7a7577e 100644
--- a/rdmo/management/assets/scss/management.scss
+++ b/rdmo/management/assets/scss/management.scss
@@ -215,30 +215,29 @@ $icon-font-path: "bootstrap-sass/assets/fonts/bootstrap/";
 }
 
 .field-diff {
-  // Target elements with classes containing "react-diff-" and "marker"
-    [class*="react-diff-"][class*="marker"] {
-        margin-top: 10px !important;
-        margin-bottom: 5px !important;
-
+    [class*="react-diff-"][class*="diff-container"] {
+      margin-top: 10px !important;
+      margin-bottom: 5px !important;
+      font-size: 11px !important;
+      line-height: 12px !important;
+      // Target elements with classes containing "react-diff-", and "marker"
+      [class*="marker"] {
+        padding: 7px 8px 7px 8px !important;
         pre {
-            background-color: #fff !important;
-            font-size: 10px;
-            padding-top: 5px !important;
-            padding-bottom: 5px !important;
+          margin: 0px !important;;
+          background-color: #fff !important;
+          padding: 2px 9px 2px 9px !important;
         }
-    }
+      }
 
-    // Target elements with classes containing "react-diff-" and "content-text"
-    [class*="react-diff-"][class*="content-text"] {
-        font-size: 10px !important;
-        line-height: 12px !important;
-        margin-bottom: 5px !important;
-        margin-top: 5px !important;
-        padding-top: 10px !important;
-        padding-bottom: 10px !important;
+      // Target elements with classes containing "react-diff-", and "content"
+      [class*="content"] {
+        padding: 7px 16px 7px 8px !important;
+        white-space: normal;
+        pre {
+          margin: 0px !important;
+          padding: 0px !important;
+        }
       }
+    }
 }
-
-//.list-group-item div.mt-10 div.row div.field-diff.col-sm-12.mt-10.mb-10 table.react-diff-1n5o7vh-diff-container tbody tr.react-diff-1n7ec1i-line td.react-diff-vl0irh-content.react-diff-1fxlvce-diff-removed pre.react-diff-16bq9gd-content-text
-//<span class="react-diff-42iver-word-diff">Lorem </span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">mu</span><span class="react-diff-42iver-word-diff">ps</span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">i</span><span class="react-diff-42iver-word-diff">m</span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">de</span><span class="react-diff-42iver-word-diff"> dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est.</span>
-//<pre class="react-diff-16bq9gd-content-text"><span class="react-diff-42iver-word-diff">Lorem </span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">mu</span><span class="react-diff-42iver-word-diff">ps</span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">i</span><span class="react-diff-42iver-word-diff">m</span><span class="react-diff-42iver-word-diff react-diff-hf3w1f-word-removed">de</span><span class="react-diff-42iver-word-diff"> dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est.</span></pre>

From bc3a77313fac9a67474e0afb4ab4bc2220bab7f5 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 3 Jun 2024 17:18:35 +0200
Subject: [PATCH 157/205] chore(js): refactor Field components for import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/common/FieldRow.js   | 29 +++++++++
 .../{FieldsDiffs.js => FieldRowDiffs.js}      |  6 +-
 .../components/import/common/FieldRowValue.js | 29 +++++++++
 .../js/components/import/common/Fields.js     | 61 +++++--------------
 .../assets/js/components/main/Import.js       |  8 +--
 5 files changed, 79 insertions(+), 54 deletions(-)
 create mode 100644 rdmo/management/assets/js/components/import/common/FieldRow.js
 rename rdmo/management/assets/js/components/import/common/{FieldsDiffs.js => FieldRowDiffs.js} (94%)
 create mode 100644 rdmo/management/assets/js/components/import/common/FieldRowValue.js

diff --git a/rdmo/management/assets/js/components/import/common/FieldRow.js b/rdmo/management/assets/js/components/import/common/FieldRow.js
new file mode 100644
index 0000000000..ee8504f8df
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/common/FieldRow.js
@@ -0,0 +1,29 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+import uniqueId from 'lodash/uniqueId'
+import FieldRowValue from './FieldRowValue'
+import FieldRowDiffs from './FieldRowDiffs'
+
+const FieldRow = ({ element, keyName, value }) => (
+  <div>
+    <div className="row" key={uniqueId()}>
+      <div className="col-sm-3 mb-5 mt-5">
+        <code className="code-import">{keyName}</code>
+      </div>
+    </div>
+    <div className="row" key={uniqueId()}>
+      <FieldRowValue value={value} />
+      {element.updated && element.changed && keyName in element.updated_and_changed && (
+        <FieldRowDiffs element={element} field={keyName} />
+      )}
+    </div>
+  </div>
+)
+
+FieldRow.propTypes = {
+  element: PropTypes.object.isRequired,
+  keyName: PropTypes.string.isRequired,
+  value: PropTypes.any.isRequired,
+}
+
+export default FieldRow
diff --git a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js b/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
similarity index 94%
rename from rdmo/management/assets/js/components/import/common/FieldsDiffs.js
rename to rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
index e496144dbd..4aa9d1d44c 100644
--- a/rdmo/management/assets/js/components/import/common/FieldsDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
@@ -5,7 +5,7 @@ import ReactDiffViewer from 'react-diff-viewer-continued'
 import Warnings from './Warnings'
 import Errors from './Errors'
 
-const FieldsDiffs = ({ element, field }) => {
+const FieldRowDiffs = ({ element, field }) => {
   if (isEmpty(element.updated_and_changed[field])) {
       return null
   }
@@ -59,9 +59,9 @@ const FieldsDiffs = ({ element, field }) => {
   )
 }
 
-FieldsDiffs.propTypes = {
+FieldRowDiffs.propTypes = {
   element: PropTypes.object.isRequired,
   field: PropTypes.string.isRequired,
 }
 
-export default FieldsDiffs
+export default FieldRowDiffs
diff --git a/rdmo/management/assets/js/components/import/common/FieldRowValue.js b/rdmo/management/assets/js/components/import/common/FieldRowValue.js
new file mode 100644
index 0000000000..e6e891c3cb
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/common/FieldRowValue.js
@@ -0,0 +1,29 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+import uniqueId from 'lodash/uniqueId'
+import isString from 'lodash/isString'
+import isUndefined from 'lodash/isUndefined'
+import truncate from 'lodash/truncate'
+import { codeClass } from '../../../constants/elements'
+
+const FieldRowValue = ({ value }) => (
+  <div className="col-sm-12">
+    {Array.isArray(value) && (
+      <ul className="list-unstyled mb-0">
+        {value.map((el) => (
+          <li key={uniqueId()}>
+            <code className={codeClass[el.type]}>{el.uri}</code>
+          </li>
+        ))}
+      </ul>
+    )}
+    {!isUndefined(value.uri) && <code className={codeClass[value.type]}>{value.uri}</code>}
+    {isString(value) && <span>{truncate(value, { length: 512 })}</span>}
+  </div>
+)
+
+FieldRowValue.propTypes = {
+  value: PropTypes.any.isRequired,
+}
+
+export default FieldRowValue
diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index b4e1762876..97a2e68ae0 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -1,14 +1,8 @@
 import React from 'react'
 import PropTypes from 'prop-types'
 import isNil from 'lodash/isNil'
-import isString from 'lodash/isString'
-import isUndefined from 'lodash/isUndefined'
-import truncate from 'lodash/truncate'
 import uniqueId from 'lodash/uniqueId'
-
-
-import { codeClass } from '../../../constants/elements'
-import FieldsDiffs from './FieldsDiffs'
+import FieldRow from './FieldRow'
 
 const excludeKeys = [
   'created',
@@ -29,48 +23,21 @@ const excludeKeys = [
   'changedFields',
 ]
 
-const Fields = ({ element }) => {
-  return (
-    <div className="mt-10">
-      {
-        Object.entries(element).sort().map(([key, value]) => {
-          if (!isNil(value) && !excludeKeys.includes(key)) {
-            return (
-              <div key={uniqueId()} className="row">
-                <div className="col-sm-3 text-right">
-                  <code className="code-import">{key}</code>
-                </div>
-                <div className="col-sm-9">
-                  {
-                    Array.isArray(value) && <ul className="list-unstyled mb-0">
-                      { value.map(el => <li key={uniqueId()}>
-                        <code className={codeClass[el.type]}>{el.uri}</code>
-                      </li>) }
-                    </ul>
-                  }
-                  {
-                    !isUndefined(value.uri) && <code className={codeClass[value.type]}>{value.uri}</code>
-                  }
-                  {
-                    isString(value) && <span>{truncate(value, {length: 512})}</span>
-                  }
-                </div>
-                {
-                  element.updated && element.changed &&
-                  key in element.updated_and_changed &&
-                  <FieldsDiffs element={element} field={key}/>
-                  }
-              </div>
-            )
-          }
-        })
-      }
-    </div>
-  )
-}
+const Fields = ({ element }) => (
+  <div className="mt-10">
+    {Object.entries(element)
+      .sort()
+      .map(([key, value]) => {
+        if (!isNil(value) && !excludeKeys.includes(key)) {
+          return <FieldRow key={uniqueId()} element={element} keyName={key} value={value} />
+        }
+        return null
+      })}
+  </div>
+)
 
 Fields.propTypes = {
-  element: PropTypes.object.isRequired
+  element: PropTypes.object.isRequired,
 }
 
 export default Fields
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 182a13760a..a5d28d5a9f 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -48,15 +48,16 @@ const Import = ({ config, imports, configActions, importActions }) => {
           />
         }
         {
-        importWarnings.length > 0 &&
+          importWarnings.length > 0 &&
           <ImportWarningsPanel config={config} elements={importWarnings} importActions={importActions}
                                configActions={configActions}/>
         }
         {
-        importErrors.length > 0 &&
+          importErrors.length > 0 &&
           <ImportErrorsPanel config={config} elements={importErrors} importActions={importActions}
-                               configActions={configActions}/>
+                             configActions={configActions}/>
         }
+      </div>
         <ul className='list-group'>
           {
             filteredElements.map((element, index) => {
@@ -68,7 +69,6 @@ const Import = ({ config, imports, configActions, importActions }) => {
             })
           }
         </ul>
-    </div>
   </div>
   )
 }

From 5cecfca105ae1d4d694c84aa19016b058d60f7c2 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 3 Jun 2024 17:19:26 +0200
Subject: [PATCH 158/205] tests(fixtures): add catalog-1.xml and add change to
 optionset-1.xml

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../updated-and-changed/catalog-1.xml         | 21 +++++++++++++++++++
 .../updated-and-changed/optionsets-1.xml      |  6 +++---
 2 files changed, 24 insertions(+), 3 deletions(-)
 create mode 100644 testing/xml/elements/updated-and-changed/catalog-1.xml

diff --git a/testing/xml/elements/updated-and-changed/catalog-1.xml b/testing/xml/elements/updated-and-changed/catalog-1.xml
new file mode 100644
index 0000000000..42c62a7a83
--- /dev/null
+++ b/testing/xml/elements/updated-and-changed/catalog-1.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rdmo xmlns:dc="http://purl.org/dc/elements/1.1/" created="2023-04-20T09:38:39.297292+02:00" version="2.0.0">
+	<catalog dc:uri="http://example.com/terms/questions/catalog">
+		<uri_prefix>http://example.com/terms</uri_prefix>
+		<uri_path>catalog</uri_path>
+		<dc:comment/>
+		<order>0</order>
+		<title lang="en">Catalog</title>
+		<help lang="en">Lorem mupsimEn dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor endunten ut labore et dolore magna aliquyam erat, sed diam voluptua. At vera eosen3n et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est.</help>
+		<title lang="de">Katalog</title>
+		<help lang="de">Lorem spimDe dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor vidididi ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero ededeos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est.</help>
+		<sections>
+			<section dc:uri="http://example.com/terms/questions/catalog/individual" order="1"/>
+			<section dc:uri="http://example.com/terms/questions/catalog/collections" order="2"/>
+			<section dc:uri="http://example.com/terms/questions/catalog/set" order="3"/>
+			<section dc:uri="http://example.com/terms/questions/catalog/conditions" order="4"/>
+			<section dc:uri="http://example.com/terms/questions/catalog/options" order="5"/>
+			<section dc:uri="http://example.com/terms/questions/catalog/blocks" order="6"/>
+		</sections>
+	</catalog>
+</rdmo>
diff --git a/testing/xml/elements/updated-and-changed/optionsets-1.xml b/testing/xml/elements/updated-and-changed/optionsets-1.xml
index e4c6ff5174..c654a06bf2 100644
--- a/testing/xml/elements/updated-and-changed/optionsets-1.xml
+++ b/testing/xml/elements/updated-and-changed/optionsets-1.xml
@@ -83,10 +83,10 @@
 		<uri_path>one_two_three_other/one</uri_path>
 		<dc:comment/>
 		<text lang="en">One - 3</text>
-		<help lang="en"/>
+		<help lang="en">Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est.</help>
 		<text lang="de">Eins - 3</text>
-		<help lang="de"/>
-		<additional_input/>
+		<help lang="de">Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est.</help>
+		<additional_input>text</additional_input>
 	</option>
 	<option dc:uri="http://example.com/terms/options/one_two_three_other/two">
 		<uri_prefix>http://example.com/terms</uri_prefix>

From 06eecfcb2a21badad0853a4ede5ad03d35bcef7e Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 4 Jun 2024 12:14:12 +0200
Subject: [PATCH 159/205] tests(e2e): fix header info assertions in import
 options

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../tests/test_frontend_import_options.py       | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
index aa5aa43179..b2d9393410 100644
--- a/rdmo/management/tests/test_frontend_import_options.py
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -17,7 +17,13 @@
 test_users = [('editor', 'editor')]
 import_xml = "./testing/xml/elements/optionsets.xml"
 import_xml_1 = "./testing/xml/elements/updated-and-changed/optionsets-1.xml"
-OPTIONS_TOTAL_COUNT = 13
+OPTIONSETS_COUNTS = {
+    "total": 13,
+    "updated": 13,
+    "changed": 5,
+    "warnings": 1
+}
+OPTIONSETS_COUNTS_HEADER_INFOS = [f"{k.capitalize()}: {v}" for k,v in OPTIONSETS_COUNTS.items()]
 
 @pytest.mark.parametrize("username, password", test_users)  # consumed by fixture
 def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> None:
@@ -36,15 +42,15 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     expect(page.get_by_text("Import from: optionsets.xml")).to_be_visible(timeout=30_000)
     ## TODO test if ImportInfo numbers are correct
     # test the components of the import-before-import staging page
-    expect(page.get_by_text(f"Created: {OPTIONS_TOTAL_COUNT}")).to_be_visible(timeout=30_000)
+    expect(page.get_by_text(f"Created: {OPTIONSETS_COUNTS['total']}")).to_be_visible(timeout=30_000)
     page.locator(".element-link").first.click()
     page.get_by_role("link", name="Unselect all").click()
     page.get_by_role("link", name="Select all", exact=True).click()
     page.get_by_role("link", name="Show all", exact=True).click()
     rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")
-    expect(rows_displayed_in_ui).to_have_count(OPTIONS_TOTAL_COUNT)
+    expect(rows_displayed_in_ui).to_have_count(OPTIONSETS_COUNTS['total'])
     # click the import button to start saving the instances to the db
-    page.get_by_role("button", name=f"Import {OPTIONS_TOTAL_COUNT} elements").click()
+    page.get_by_role("button", name=f"Import {OPTIONSETS_COUNTS['total']} elements").click()
     expect(page.get_by_role("heading", name="Import successful")).to_be_visible()
     page.screenshot(path="screenshots/management-import-optionsets-post-import.png", full_page=True)
     page.get_by_text("Created:").click()
@@ -62,7 +68,8 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     page.locator('#sidebar div.elements-sidebar form.upload-form.sidebar-form div.sidebar-form-button button.btn.btn-primary').click()  # noqa: E501
     expect(page.get_by_text("Import from: optionsets-1.xml")).to_be_visible(timeout=40_000)
     # assert changed elements
-    expect(page.get_by_text(f"Total: {OPTIONS_TOTAL_COUNT} Updated: {OPTIONS_TOTAL_COUNT} (Changed: 5) Warnings: 1")).to_be_visible(timeout=30_000)  # noqa: E501
+    for text in OPTIONSETS_COUNTS_HEADER_INFOS:
+        expect(page.locator("#main")).to_contain_text(text)
     expect(page.get_by_text("Filter changed (5)")).to_be_visible()
     page.get_by_role("link", name="Show changes").click()
     expect(page.locator(".col-sm-6 > .form-group").first).to_be_visible(timeout=30_000)

From cadd24521ef5b85b7e630ccc4094097fa1c3e858 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 4 Jun 2024 17:26:21 +0200
Subject: [PATCH 160/205] fix: #810 fix xml import of additional_input

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/xml.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index b522f488e7..5d356cbe0d 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -320,7 +320,9 @@ def convert_additional_input(elements):
     for uri, element in elements.items():
         if element['model'] == 'options.option':
             additional_input = element.get('additional_input')
-            if additional_input == 'True':
+            if additional_input in ['', 'text', 'textarea']:  # from Option.ADDITIONAL_INPUT_CHOICES
+                element['additional_input'] = additional_input
+            elif additional_input == 'True':
                 element['additional_input'] = 'text'
             else:
                 element['additional_input'] = ''

From d088df2a5c2969e905ea1f7a0ae55aa119c7f711 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 4 Jun 2024 17:28:54 +0200
Subject: [PATCH 161/205] refactor: minor changes to import related functions

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/import_helpers.py | 15 +++++++++------
 rdmo/core/imports.py        | 15 +++------------
 rdmo/management/imports.py  |  7 ++++---
 rdmo/options/imports.py     |  2 +-
 4 files changed, 17 insertions(+), 22 deletions(-)

diff --git a/rdmo/core/import_helpers.py b/rdmo/core/import_helpers.py
index 70af6a1eb1..95d3e155ad 100644
--- a/rdmo/core/import_helpers.py
+++ b/rdmo/core/import_helpers.py
@@ -21,17 +21,20 @@ class ThroughInstanceMapper:
 @dataclass(frozen=True)
 class ExtraFieldDefaultHelper:
     field_name: str
-    value: Optional[Union[str, bool, int]] = None
+    value: Union[str, bool, int, None] = None
     callback: Optional[Callable] = None
 
     def get_default(self, **kwargs):
         if self.callback is None:
             return self.value
-
-        sig = signature(self.callback)
-        _kwargs = {k:val for k,val in kwargs.items() if k in sig.parameters}
-        _value = self.callback(**_kwargs)
-        return _value
+        else:
+            return self.get_default_from_callback(self.callback, kwargs)
+    @staticmethod
+    def get_default_from_callback(callback, kwargs):
+        sig = signature(callback)
+        kwargs = {k: val for k, val in kwargs.items() if k in sig.parameters}
+        value = callback(**kwargs)
+        return value
 
 
 @dataclass(frozen=True)
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 1ddd61bd1d..94c4fe2115 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -98,7 +98,7 @@ def track_messages_on_element(element: dict, element_field: str, warning: Option
         _append_error(element, element_field, error)
 
 
-def _initialize_track_changes_element_field(element: dict, element_field: str):
+def _initialize_track_changes_element_field(element: dict, element_field: str) -> None:
     if ELEMENT_DIFF_FIELD_NAME not in element:
         element[ELEMENT_DIFF_FIELD_NAME] = {}
 
@@ -106,10 +106,6 @@ def _initialize_track_changes_element_field(element: dict, element_field: str):
         element[ELEMENT_DIFF_FIELD_NAME][element_field] = {}
 
 
-def _cast_list_of_string_to_list(list_of_strings: List[str]) -> str:
-    return "\n".join(map(str, list_of_strings))
-
-
 def track_changes_on_element(element: dict,
                              element_field: str,
                              new_value: Union[str, List[str], None] = None,
@@ -192,13 +188,11 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
             field_name=field_name
         )
         logger.info(message)
-        # [element.get('uri')]
         element['errors'].append(message)  # errors is a list
         track_messages_on_element(element, field_name, error=message)
         return
 
     foreign_uri = foreign_element['uri']
-    # breakpoint()
     model_info = model_meta.get_field_info(instance)
     foreign_model = model_info.forward_relations[field_name].related_model
     foreign_instance = None
@@ -250,15 +244,12 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
 def set_extra_field(instance, field_name, element,
                     extra_field_helper: Optional[ExtraFieldDefaultHelper] = None, original=None) -> None:
     element_value = element.get(field_name)
+    extra_value = element_value if element_value is not None else None
 
-    extra_value = None
-    if element_value is not None:
-        extra_value = element_value
-    elif extra_field_helper is not None:
+    if extra_value is None and extra_field_helper is not None:
         # default_value
         extra_value = extra_field_helper.get_default(instance=instance,
                                                      key=field_name)
-
     if extra_value is not None:
         setattr(instance, field_name, extra_value)
         # track changes
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index bffebdf3be..b9c4399339 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -69,7 +69,7 @@ def import_elements(uploaded_elements: Dict, save: bool = True, request: Optiona
         element = import_element(element=uploaded_element, save=save,
                                   uploaded_uris=uploaded_uris,
                                     request=request, current_site=current_site)
-        element['warnings'] = {k: val  for k, val in element['warnings'].items() if k not in uploaded_uris}
+        element['warnings'] = {k: val for k, val in element['warnings'].items() if k not in uploaded_uris}
         imported_elements.append(element)
     return imported_elements
 
@@ -145,10 +145,11 @@ def import_element(
         set_lang_field(instance, lang_field_name, element, original=original)
     # set foreign fields
     for foreign_field in foreign_field_names:
-        set_foreign_field(instance, foreign_field, element, uploaded_uris=uploaded_uris)
+        set_foreign_field(instance, foreign_field, element, uploaded_uris=uploaded_uris, original=original)
     # set extra fields
     for extra_field_helper in extra_field_helpers:
-        set_extra_field(instance, extra_field_helper.field_name, element, extra_field_helper=extra_field_helper)
+        set_extra_field(instance, extra_field_helper.field_name, element,
+                        extra_field_helper=extra_field_helper, original=original)
     # call the validators on the instance
     validate_instance(instance, element, *validators)
 
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index 0ba6ca5344..a17e0cf12e 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -33,7 +33,7 @@
     validators = (OptionLockedValidator, OptionUniqueURIValidator),
     lang_fields = ('text', 'help', 'view_text'),
     extra_fields = (
-        ExtraFieldDefaultHelper(field_name='additional_input'),
+        ExtraFieldDefaultHelper(field_name='additional_input', value=Option.ADDITIONAL_INPUT_NONE),
         ),
     reverse_m2m_through_instance_fields = [
         ThroughInstanceMapper(

From e9182cbb309f6e0e3a0c05a64c0a0228d56b49f1 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 4 Jun 2024 17:30:03 +0200
Subject: [PATCH 162/205] feat(import): make import sidebar links to flat list

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/sidebar/ImportSidebar.js    | 30 +++++++++----------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
index eaef0a7d46..f226d5f249 100644
--- a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
+++ b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
@@ -57,28 +57,26 @@ const ImportSidebar = ({ config, imports, importActions }) => {
               {gettext('Select all')}
             </Link>
           </li>
-            {changedElements.length > 0 &&
+          {changedElements.length > 0 &&
             <li>
-              <ul className="list-unstyled" style={{paddingLeft:'20px'}}>
-                <li>
-                  <Link onClick={() => importActions.selectChangedElements(true)}>
-                    {gettext('Select changed')}
-                  </Link>
-                </li>
-                <li>
-                  <Link onClick={() => importActions.selectChangedElements(false)}>
-                    {gettext('Unselect changed')}
-                  </Link>
-                </li>
-              </ul>
+              <Link onClick={() => importActions.selectChangedElements(true)}>
+                {gettext('Select changed')}
+              </Link>
             </li>
-            }
+          }
           <li>
             <Link onClick={() => importActions.selectElements(false)}>
-              {gettext('Unselect all')}
+              {gettext('Deselect all')}
             </Link>
           </li>
-          </ul>
+          {changedElements.length > 0 &&
+            <li>
+              <Link onClick={() => importActions.selectChangedElements(false)}>
+                {gettext('Deselect changed')}
+              </Link>
+            </li>
+          }
+        </ul>
 
           <h2>{gettext('Show')}</h2>
           <ul className="list-unstyled">

From e64c5fa7220532bba456bb20fddf194263477f83 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Tue, 4 Jun 2024 17:31:08 +0200
Subject: [PATCH 163/205] style(import): update selectors and css settings for
 import panels

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/assets/scss/management.scss | 63 +++++++++++++--------
 1 file changed, 40 insertions(+), 23 deletions(-)

diff --git a/rdmo/management/assets/scss/management.scss b/rdmo/management/assets/scss/management.scss
index 21e7a7577e..033eee3c69 100644
--- a/rdmo/management/assets/scss/management.scss
+++ b/rdmo/management/assets/scss/management.scss
@@ -216,28 +216,45 @@ $icon-font-path: "bootstrap-sass/assets/fonts/bootstrap/";
 
 .field-diff {
     [class*="react-diff-"][class*="diff-container"] {
-      margin-top: 10px !important;
-      margin-bottom: 5px !important;
-      font-size: 11px !important;
-      line-height: 12px !important;
-      // Target elements with classes containing "react-diff-", and "marker"
-      [class*="marker"] {
-        padding: 7px 8px 7px 8px !important;
-        pre {
-          margin: 0px !important;;
-          background-color: #fff !important;
-          padding: 2px 9px 2px 9px !important;
-        }
-      }
-
-      // Target elements with classes containing "react-diff-", and "content"
-      [class*="content"] {
-        padding: 7px 16px 7px 8px !important;
-        white-space: normal;
-        pre {
-          margin: 0px !important;
-          padding: 0px !important;
-        }
-      }
+        margin: 0;
+        border-collapse: separate;  // needed for border radius since this is a table
+
+        tr {
+            &:first-child {
+                [class*="marker"] {
+                    border-top-left-radius: 4px;
+                }
+                [class*="content"] {
+                    border-top-right-radius: 4px;
+                }
+            }
+            &:last-child {
+                [class*="marker"] {
+                    border-bottom-left-radius: 4px;
+                }
+                [class*="content"] {
+                    border-bottom-right-radius: 4px;
+                }
+            }
+        }
+
+        // Target elements with classes containing "react-diff-", and "marker"
+        // Target elements with classes containing "react-diff-", and "content"
+        [class*="marker"],
+        [class*="content"] {
+            padding: 8px 12px;
+            vertical-align: top;
+            pre {
+                padding: 5px 10px;
+                border-radius: 4px;
+                font-size: 13px;
+                line-height: 18px;
+                background-color: #fff;
+            }
+        }
+
+        [class*="marker"] {
+            padding-right: 0;
+        }
     }
 }

From 7d559ce3c637094598316508ea1c2ad86ee243b2 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 5 Jun 2024 10:35:09 +0200
Subject: [PATCH 164/205] style(import): update scss for import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/static/core/css/base.scss         | 3 ---
 rdmo/management/assets/scss/management.scss | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/rdmo/core/static/core/css/base.scss b/rdmo/core/static/core/css/base.scss
index 44bf835add..26e02c3eb7 100644
--- a/rdmo/core/static/core/css/base.scss
+++ b/rdmo/core/static/core/css/base.scss
@@ -193,9 +193,6 @@ metadata {
     margin-top: 0;
 }
 .sidebar h2:first-child,
-.sidebar .import-buttons {
-    margin-top: 70px;
-}
 
 /* questions overview */
 
diff --git a/rdmo/management/assets/scss/management.scss b/rdmo/management/assets/scss/management.scss
index 033eee3c69..0107bde153 100644
--- a/rdmo/management/assets/scss/management.scss
+++ b/rdmo/management/assets/scss/management.scss
@@ -66,7 +66,7 @@ $icon-font-path: "bootstrap-sass/assets/fonts/bootstrap/";
               display: flex;
               align-items: center;
               justify-content: space-between;
-              padding: 6px 15px 6px;
+              padding: 6px 15px 0px;
 
                 .shown-info {
                     margin-left: auto;

From 5122c526f020f5769f5f0f51fe8d21d98147e40d Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 5 Jun 2024 10:37:04 +0200
Subject: [PATCH 165/205] feat(import): update import sidebar, filter text and
 refactor succes component

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../components/import/ImportSuccessElement.js | 44 +++++++++++++------
 .../components/import/common/ImportFilters.js |  2 +-
 .../js/components/sidebar/ImportSidebar.js    | 28 ++++++------
 .../assets/js/hooks/useImportElements.js      |  4 +-
 4 files changed, 46 insertions(+), 32 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index 1b4c09a213..aff18345d3 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -5,35 +5,50 @@ import uniqueId from 'lodash/uniqueId'
 import { codeClass, verboseNames } from '../../constants/elements'
 import { isEmpty } from 'lodash'
 import Warnings from './common/Warnings'
+import { ShowUpdatedLink } from '../common/Links'
 
 const prepareErrorsList = (errors) => {
   // Filter out duplicate errors
   const uniqueErrors = [...new Set(errors)]
 
   return uniqueErrors.map(message => (
-        <p key={uniqueId()} className="text-danger">{message}</p>
+    <p key={uniqueId()} className="text-danger">{message}</p>
   ))
 }
-const ImportSuccessElement = ({ element }) => {
+
+const ImportSuccessElement = ({ element, importActions }) => {
   const listErrorMessages = prepareErrorsList(element.errors)
+
+  const updateShowField = () => importActions.updateElement(element, { show: !element.show })
+
   return (
     <li className="list-group-item">
-      <p>
+      <div className="pull-right">
+        <Warnings element={element} showTitle={true} />
+      </div>
+      <div>
         <strong>{verboseNames[element.model]}{' '}</strong>
         <code className={codeClass[element.model]}>{element.uri}</code>
-        {element.created && <span className="text-success">{' '}{gettext('created')}  </span> && <span className="muted element-link fa fa-plus"></span>}
-        {element.updated && <span className="text-info">{' '}{gettext('updated')} </span> && <span className="muted element-link fa fa-pencil"></span>}
-        {
-          !isEmpty(element.errors) && !(element.created || element.updated) &&
+        {element.created && (
+          <>
+            <span className="text-success">{' '}{gettext('created')} </span>
+            <span className="muted element-link fa fa-plus"></span>
+          </>
+        )}
+        {element.updated && (
+          <ShowUpdatedLink show={(element.changed && !element.created)} disabled={true} onClick={updateShowField} />
+        )}
+        {!isEmpty(element.errors) && !(element.created || element.updated) && (
           <span className="text-danger">{' '}{gettext('could not be imported')}</span>
-        }
-        {
-          !isEmpty(element.errors) && (element.created || element.updated) &&
-          <>{', '}<span className="text-danger">{gettext('but could not be added to parent element')}</span></>
-        }
+        )}
+        {!isEmpty(element.errors) && (element.created || element.updated) && (
+          <>
+            {', '}
+            <span className="text-danger">{gettext('but could not be added to parent element')}</span>
+          </>
+        )}
         {'.'}
-      </p>
-        <Warnings element={element} showTitle={true}/>
+      </div>
       {listErrorMessages}
     </li>
   )
@@ -41,6 +56,7 @@ const ImportSuccessElement = ({ element }) => {
 
 ImportSuccessElement.propTypes = {
   element: PropTypes.object.isRequired,
+  importActions: PropTypes.object.isRequired
 }
 
 export default ImportSuccessElement
diff --git a/rdmo/management/assets/js/components/import/common/ImportFilters.js b/rdmo/management/assets/js/components/import/common/ImportFilters.js
index bbda85fab9..5e545d8707 100644
--- a/rdmo/management/assets/js/components/import/common/ImportFilters.js
+++ b/rdmo/management/assets/js/components/import/common/ImportFilters.js
@@ -29,7 +29,7 @@ const ImportFilters = ({ config, elements, changedElements, filteredElements, co
       {elements.length > 0 && (
         <div className="horizontal-container">
           <div className="checkboxes">
-            <Checkbox label={interpolate(gettext('Filter changed (%s)'), [changedElements.length])}
+            <Checkbox label={interpolate(gettext('Show only created and changed (%s)'), [changedElements.length])}
                       value={getValueFilterChanged()} onChange={updateFilterChanged} />
           </div>
           <span className="shown-info">
diff --git a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
index f226d5f249..176ccb0413 100644
--- a/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
+++ b/rdmo/management/assets/js/components/sidebar/ImportSidebar.js
@@ -86,27 +86,25 @@ const ImportSidebar = ({ config, imports, importActions }) => {
             </Link>
           </li>
           {changedElements.length > 0 &&
-              <li>
-                <ul className="list-unstyled" style={{paddingLeft:'20px'}}>
-                  <li>
-                    <Link onClick={() => importActions.showChangedElements(true)}>
-                      {gettext('Show changes')}
-                    </Link>
-                  </li>
-                  <li>
-                    <Link onClick={() => importActions.showChangedElements(false)}>
-                      {gettext('Hide changes')}
-                    </Link>
-                  </li>
-                </ul>
-              </li>
+            <li>
+              <Link onClick={() => importActions.showChangedElements(true)}>
+                {gettext('Show changes')}
+              </Link>
+            </li>
           }
           <li>
             <Link onClick={() => importActions.showElements(false)}>
               {gettext('Hide all')}
             </Link>
           </li>
-        </ul>
+            {changedElements.length > 0 &&
+              <li>
+                <Link onClick={() => importActions.showChangedElements(false)}>
+                  {gettext('Hide changes')}
+                </Link>
+              </li>
+            }
+          </ul>
 
         <h2>{gettext('URI prefix')}</h2>
 
diff --git a/rdmo/management/assets/js/hooks/useImportElements.js b/rdmo/management/assets/js/hooks/useImportElements.js
index f00abc3ef3..9fa3492796 100644
--- a/rdmo/management/assets/js/hooks/useImportElements.js
+++ b/rdmo/management/assets/js/hooks/useImportElements.js
@@ -6,8 +6,8 @@ export const useImportElements = (elements) => {
     // the elements are already processed by processElementDiffs in the importsReducer
     const createdElements = elements.filter(element => element.created)
     const updatedElements = elements.filter(element => element.updated)
-    // collects elements with updated AND changed
-    const changedElements = updatedElements.filter(element => element.changed)
+    // changedElements collects elements with updated AND changed OR created
+    const changedElements = elements.filter(element => ((element.updated && element.changed) || element.created))
     const importWarnings = elements.filter(element => !isEmpty(element.warnings))
     const importErrors = elements.filter(element => !isEmpty(element.errors))
 

From e7d7eb9f80baa2862aa7604a77d9ea2cce5aa1db Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 5 Jun 2024 10:37:50 +0200
Subject: [PATCH 166/205] tests(e2e): update text selectors in import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/test_frontend_import_options.py   | 4 ++--
 rdmo/management/tests/test_frontend_import_questions.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
index b2d9393410..1907b725d1 100644
--- a/rdmo/management/tests/test_frontend_import_options.py
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -44,7 +44,7 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     # test the components of the import-before-import staging page
     expect(page.get_by_text(f"Created: {OPTIONSETS_COUNTS['total']}")).to_be_visible(timeout=30_000)
     page.locator(".element-link").first.click()
-    page.get_by_role("link", name="Unselect all").click()
+    page.get_by_role("link", name="Deselect all").click()
     page.get_by_role("link", name="Select all", exact=True).click()
     page.get_by_role("link", name="Show all", exact=True).click()
     rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")
@@ -70,7 +70,7 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     # assert changed elements
     for text in OPTIONSETS_COUNTS_HEADER_INFOS:
         expect(page.locator("#main")).to_contain_text(text)
-    expect(page.get_by_text("Filter changed (5)")).to_be_visible()
+    expect(page.get_by_text(f"Show only created and changed ({OPTIONSETS_COUNTS['changed']})")).to_be_visible()
     page.get_by_role("link", name="Show changes").click()
     expect(page.locator(".col-sm-6 > .form-group").first).to_be_visible(timeout=30_000)
     # take a screenshot of the import page
diff --git a/rdmo/management/tests/test_frontend_import_questions.py b/rdmo/management/tests/test_frontend_import_questions.py
index 368634f26f..4ff5c2c798 100644
--- a/rdmo/management/tests/test_frontend_import_questions.py
+++ b/rdmo/management/tests/test_frontend_import_questions.py
@@ -35,7 +35,7 @@ def test_import_catalogs_in_management(logged_in_user: Page) -> None:
     ## TODO test if ImportInfo numbers are correct
     # test the components of the import-before-import staging page
     page.locator(".element-link").first.click()
-    page.get_by_role("link", name="Unselect all").click()
+    page.get_by_role("link", name="Deselect all").click()
     page.get_by_role("link", name="Select all", exact=True).click()
     page.get_by_role("link", name="Show all").click()
     rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")

From 2e172fe2b570e7132131357265d1ded9cafdffd5 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 5 Jun 2024 11:30:08 +0200
Subject: [PATCH 167/205] refactor(import): improve readability and move helper
 functions to import_utils

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/import_utils.py               | 101 ++++++++++++++++
 rdmo/management/imports.py                    | 114 +++---------------
 .../tests/helpers_import_elements.py          |   4 +-
 3 files changed, 118 insertions(+), 101 deletions(-)
 create mode 100644 rdmo/management/import_utils.py

diff --git a/rdmo/management/import_utils.py b/rdmo/management/import_utils.py
new file mode 100644
index 0000000000..6d964791d2
--- /dev/null
+++ b/rdmo/management/import_utils.py
@@ -0,0 +1,101 @@
+from collections import defaultdict
+from dataclasses import asdict
+from typing import Dict
+
+from rdmo.conditions.imports import import_helper_condition
+from rdmo.core.imports import (
+    ELEMENT_DIFF_FIELD_NAME,
+    set_extra_field,
+    set_foreign_field,
+    set_lang_field,
+    set_m2m_instances,
+    set_m2m_through_instances,
+    set_reverse_m2m_through_instance,
+    track_changes_on_element,
+)
+from rdmo.domain.imports import import_helper_attribute
+from rdmo.options.imports import import_helper_option, import_helper_optionset
+from rdmo.questions.imports import (
+    import_helper_catalog,
+    import_helper_page,
+    import_helper_question,
+    import_helper_questionset,
+    import_helper_section,
+)
+from rdmo.tasks.imports import import_helper_task
+from rdmo.views.imports import import_helper_view
+
+ELEMENT_IMPORT_HELPERS = {
+    "conditions.condition": import_helper_condition,
+    "domain.attribute": import_helper_attribute,
+    "options.optionset": import_helper_optionset,
+    "options.option": import_helper_option,
+    "questions.catalog": import_helper_catalog,
+    "questions.section": import_helper_section,
+    "questions.page": import_helper_page,
+    "questions.questionset": import_helper_questionset,
+    "questions.question": import_helper_question,
+    "tasks.task": import_helper_task,
+    "views.view": import_helper_view
+}
+IMPORT_ELEMENT_INIT_DICT = {
+        'warnings': lambda: defaultdict(list),
+        'errors': list,
+        'created': bool,
+        'updated': bool,
+        ELEMENT_DIFF_FIELD_NAME: dict,
+    }
+
+
+def initialize_import_element_dict(element: Dict) -> None:
+    # initialize element dict with default values
+    for _k,_val in IMPORT_ELEMENT_INIT_DICT.items():
+        element[_k] = _val()
+
+
+def strip_uri_prefix_endswith_slash(element: dict) -> dict:
+    """Removes the trailing slash from the URI prefix if it exists."""
+    if 'uri_prefix' in element and element['uri_prefix'].endswith('/'):
+        element['uri_prefix'] = element['uri_prefix'].rstrip('/')
+    return element
+
+
+def apply_field_values(instance, element, import_helper, uploaded_uris, original) -> None:
+    """Applies the field values from the element to the instance."""
+    element = strip_uri_prefix_endswith_slash(element)
+    # start to set values on the instance
+    # set common field values from element on instance
+    for field in import_helper.common_fields:
+        value = element.get(field) or ''
+        setattr(instance, field, value)
+        if element['updated']:
+            # track changes for common fields
+            track_changes_on_element(element, field, new_value=value, original=original)
+    # set language fields
+    for field in import_helper.lang_fields:
+        set_lang_field(instance, field, element, original=original)
+    # set foreign fields
+    for field in import_helper.foreign_fields:
+        set_foreign_field(instance, field, element, uploaded_uris=uploaded_uris, original=original)
+
+    for extra_field in import_helper.extra_fields:
+        set_extra_field(instance, extra_field.field_name, element, extra_field_helper=extra_field, original=original)
+
+
+def update_related_fields(instance, element, import_helper, original, save) -> None:
+    # this part updates the related fields of the instance
+    for m2m_field in import_helper.m2m_instance_fields:
+        set_m2m_instances(instance, element, m2m_field, original=original, save=save)
+    for m2m_through_fields in import_helper.m2m_through_instance_fields:
+        set_m2m_through_instances(instance, element, **asdict(m2m_through_fields),
+                                  original=original, save=save)
+    for reverse_m2m_fields in import_helper.reverse_m2m_through_instance_fields:
+        set_reverse_m2m_through_instance(instance, element, **asdict(reverse_m2m_fields),
+                                         original=original, save=save)
+
+
+def add_current_site_to_sites_and_editor(instance, current_site, import_helper):
+    if import_helper.add_current_site_editors:
+        instance.editors.add(current_site)
+    if import_helper.add_current_site_sites:
+        instance.sites.add(current_site)
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index b9c4399339..7ac01ca99b 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -1,66 +1,29 @@
 import copy
 import logging
-from collections import defaultdict
-from dataclasses import asdict
 from typing import AbstractSet, Dict, List, Optional
 
 from django.conf import settings
 from django.contrib.sites.shortcuts import get_current_site
 from django.http import HttpRequest
 
-from rdmo.conditions.imports import import_helper_condition
 from rdmo.core.imports import (
-    ELEMENT_DIFF_FIELD_NAME,
     check_permissions,
     get_or_return_instance,
     make_import_info_msg,
-    set_extra_field,
-    set_foreign_field,
-    set_lang_field,
-    set_m2m_instances,
-    set_m2m_through_instances,
-    set_reverse_m2m_through_instance,
-    track_changes_on_element,
     validate_instance,
 )
-from rdmo.domain.imports import import_helper_attribute
-from rdmo.options.imports import import_helper_option, import_helper_optionset
-from rdmo.questions.imports import (
-    import_helper_catalog,
-    import_helper_page,
-    import_helper_question,
-    import_helper_questionset,
-    import_helper_section,
+from rdmo.management.import_utils import (
+    ELEMENT_IMPORT_HELPERS,
+    add_current_site_to_sites_and_editor,
+    apply_field_values,
+    initialize_import_element_dict,
+    strip_uri_prefix_endswith_slash,
+    update_related_fields,
 )
-from rdmo.tasks.imports import import_helper_task
-from rdmo.views.imports import import_helper_view
 
 logger = logging.getLogger(__name__)
 
 
-ELEMENT_IMPORT_HELPERS = {
-    "conditions.condition": import_helper_condition,
-    "domain.attribute": import_helper_attribute,
-    "options.optionset": import_helper_optionset,
-    "options.option": import_helper_option,
-    "questions.catalog": import_helper_catalog,
-    "questions.section": import_helper_section,
-    "questions.page": import_helper_page,
-    "questions.questionset": import_helper_questionset,
-    "questions.question": import_helper_question,
-    "tasks.task": import_helper_task,
-    "views.view": import_helper_view
-}
-
-IMPORT_ELEMENT_INIT_DICT = {
-        'warnings': lambda: defaultdict(list),
-        'errors': list,
-        'created': bool,
-        'updated': bool,
-        ELEMENT_DIFF_FIELD_NAME: dict,
-    }
-
-
 def import_elements(uploaded_elements: Dict, save: bool = True, request: Optional[HttpRequest] = None) -> List[Dict]:
     imported_elements = []
     uploaded_uris = set(uploaded_elements.keys())
@@ -73,11 +36,6 @@ def import_elements(uploaded_elements: Dict, save: bool = True, request: Optiona
         imported_elements.append(element)
     return imported_elements
 
-def _initialize_import_element_dict(element: Dict) -> None:
-    # initialize element dict with default values
-    for _k,_val in IMPORT_ELEMENT_INIT_DICT.items():
-        element[_k] = _val()
-
 
 def import_element(
         element: Optional[Dict] = None,
@@ -94,7 +52,7 @@ def import_element(
     if model_path is None:
         return element
 
-    _initialize_import_element_dict(element)
+    initialize_import_element_dict(element)
 
     user = request.user if request is not None else None
     import_helper = ELEMENT_IMPORT_HELPERS[model_path]
@@ -102,10 +60,6 @@ def import_element(
         raise ValueError(f'Invalid import helper model path: {import_helper.model_path}. Expected {model_path}.')
     model = import_helper.model
     validators = import_helper.validators
-    common_fields = import_helper.common_fields
-    lang_field_names = import_helper.lang_fields
-    foreign_field_names = import_helper.foreign_fields
-    extra_field_helpers = import_helper.extra_fields
 
     uri = element.get('uri')
     # get or create instance from uri and model_path
@@ -129,65 +83,27 @@ def import_element(
     updated = not created
     element['created'] = created
     element['updated'] = updated
-    # the dict element[ELEMENT_DIFF_FIELD_NAME] is filled by tracking changes
+    # INFO: the dict element[ELEMENT_DIFF_FIELD_NAME] is filled by calling track_changes_on_element
 
     element = strip_uri_prefix_endswith_slash(element)
     # start to set values on the instance
-    # set common field values from element on instance
-    for common_field in common_fields:
-        common_value = element.get(common_field) or ''
-        setattr(instance, common_field, common_value)
-        if updated and original:
-            # track changes for common fields
-            track_changes_on_element(element, common_field, new_value=common_value, original=original)
-    # set language fields
-    for lang_field_name in lang_field_names:
-        set_lang_field(instance, lang_field_name, element, original=original)
-    # set foreign fields
-    for foreign_field in foreign_field_names:
-        set_foreign_field(instance, foreign_field, element, uploaded_uris=uploaded_uris, original=original)
-    # set extra fields
-    for extra_field_helper in extra_field_helpers:
-        set_extra_field(instance, extra_field_helper.field_name, element,
-                        extra_field_helper=extra_field_helper, original=original)
+    apply_field_values(instance, element, import_helper, uploaded_uris, original)
+
     # call the validators on the instance
     validate_instance(instance, element, *validators)
 
     if element.get('errors'):
         # when there is an error msg, the import can be stopped and return
         return element
+
     if save:
         logger.info(msg)
         instance.save()
+
     if save or updated:
-        # this part updates the related fields of the instance
-        for m2m_field in import_helper.m2m_instance_fields:
-            set_m2m_instances(instance, element, m2m_field, original=original, save=save)
-        for m2m_through_fields in import_helper.m2m_through_instance_fields:
-            set_m2m_through_instances(instance, element, **asdict(m2m_through_fields),
-                                      original=original, save=save)
-        for reverse_m2m_fields in import_helper.reverse_m2m_through_instance_fields:
-            set_reverse_m2m_through_instance(instance, element, **asdict(reverse_m2m_fields),
-                                             original=original, save=save)
+        update_related_fields(instance, element, import_helper, original, save)
 
     if save and settings.MULTISITE:
-        if import_helper.add_current_site_editors:
-            instance.editors.add(current_site)
-        if import_helper.add_current_site_sites:
-            instance.sites.add(current_site)
+        add_current_site_to_sites_and_editor(instance, current_site, import_helper)
 
     return element
-
-
-def strip_uri_prefix_endswith_slash(element: dict) -> dict:
-    # handle URI Prefix ending with slash
-    if 'uri_prefix' not in element:
-        return element
-    if element['uri_prefix'].endswith('/'):
-        element['uri_prefix'] = element['uri_prefix'].rstrip('/')
-    return element
-
-def _set_element_diff_field_meta_info(element: dict) -> None:
-    changed_fields = {k: val for k, val in element[ELEMENT_DIFF_FIELD_NAME].items() if val['changed']}
-    element['changed'] = bool(changed_fields)
-    element['changed_fields'] = list(changed_fields.keys())
diff --git a/rdmo/management/tests/helpers_import_elements.py b/rdmo/management/tests/helpers_import_elements.py
index 53c9e6d649..0030c48153 100644
--- a/rdmo/management/tests/helpers_import_elements.py
+++ b/rdmo/management/tests/helpers_import_elements.py
@@ -3,7 +3,7 @@
 from typing import Dict, List, Optional, Tuple, Union
 
 from rdmo.core.imports import CURRENT_DATA_FIELD, ELEMENT_DIFF_FIELD_NAME, NEW_DATA_FIELD, track_changes_on_element
-from rdmo.management.imports import _initialize_import_element_dict
+from rdmo.management.import_utils import initialize_import_element_dict
 
 UPDATE_FIELD_FUNCS = {
     'comment': lambda text: f"this is a test comment {text}",
@@ -58,7 +58,7 @@ def _test_helper_change_fields_elements(elements,
     _new_elements = OrderedDict()
     for _n, (_uri, _element) in enumerate(elements.items()):
         if _n <= n - 1:
-            _initialize_import_element_dict(_element)
+            initialize_import_element_dict(_element)
             for field in fields_to_update:
                 original_value = _element[field] or ''
                 new_val = UPDATE_FIELD_FUNCS[field](_n)

From 643a5a249094762382e1c451af7c6bd15d38cbba Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 6 Jun 2024 10:22:43 +0200
Subject: [PATCH 168/205] refactor(import): revert xml import to functional
 approach and make error handling uniform

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py                          |   4 +-
 rdmo/core/xml.py                              | 203 ++++++++++--------
 rdmo/management/management/commands/import.py |  10 +-
 rdmo/management/viewsets.py                   |  16 +-
 4 files changed, 127 insertions(+), 106 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 94c4fe2115..e83eb408e0 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -66,7 +66,7 @@ def get_rdmo_model_path(target_name: str, field_name: str):
 
 def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str] = None):
     if uri is None:
-        return "%s, no uri" % verbose_name
+        return f"{verbose_name}, no uri"
     if created:
         return f"{verbose_name} created with {uri}"
     return f"{verbose_name} {uri} updated"
@@ -313,7 +313,7 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
                 through_instance = next(filter(lambda item: getattr(item, target_name).uri == target_instance.uri,
                                                through_instances))
 
-                # update order if the item if it changed
+                # update order of the item when it was changed
                 if through_instance.order != order and save:
                     through_instance.order = order
                     through_instance.save()
diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index 5d356cbe0d..885356f9f9 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -1,16 +1,16 @@
 import logging
 import re
 from collections import OrderedDict
-from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional, Tuple
+from xml.etree.ElementTree import Element as xmlElement
 
 from django.utils.translation import gettext_lazy as _
 
 import defusedxml.ElementTree as ET
 from packaging.version import Version, parse
 
-from rdmo import __version__ as VERSION
+from rdmo import __version__ as RDMO_INSTANCE_VERSION
 from rdmo.core.constants import RDMO_MODELS
 
 logger = logging.getLogger(__name__)
@@ -19,91 +19,112 @@
 ELEMENTS_USING_KEY = {RDMO_MODELS['attribute']}
 
 
-@dataclass
-class XmlToElementsParser:
-
-    file_name: str = None
-    # post init attributes
-    file: Path = None  # will be set from file_name
-    root = None
-    errors: list = field(default_factory=list)
-    parsed_elements: OrderedDict = field(default_factory=OrderedDict)
-
-    def __post_init__(self):
-        if self.file_name is None:
-            raise ValueError("File name is required.")
-        self.file = Path(self.file_name).resolve()
-        if not self.file.exists():
-            raise ValueError(f"File does not exist. {self.file}")
-
-        elements = self.parse_xml_to_elements(self.file)
-        self.parsed_elements = elements
-        self.errors.reverse()
-
-    def is_valid(self, raise_exception: bool = False) -> bool:
-        if self.errors and raise_exception:  # raise for errors
-            raise ValueError(self.errors)
-        return not bool(self.errors)
-
-    def parse_xml_to_elements(self, xml_file: Path, raise_exception:bool=False) -> Optional[OrderedDict]:
-        root = None
-        # step 2: parse xml
-        try:
-            root = read_xml_file(self.file, raise_exception=True)
-        except Exception as e:
-            self.errors.append(_('XML Parsing Error') + f': {e!s}')
-            logger.info('XML parsing error. Import failed.')
-
-        if root is None:
-            self.errors.append(_('The content of the xml file does not consist of well formed data or markup.'))
-            return
-        elif root.tag != 'rdmo':
-            self.errors.append(_('This XML does not contain RDMO content.'))
-            return
-        self.root = root
-
-        # step 2.1: validate parsed xml
-        unparsed_root_version = root.attrib.get('version') or DEFAULT_RDMO_XML_VERSION
-        root_version, rdmo_version = parse(unparsed_root_version), parse(VERSION)
-        if root_version > rdmo_version:
-            logger.info(f'Import failed version validation ({root_version} > {rdmo_version})')
-            self.errors.append(_('This RDMO XML file does not have a valid version number.'))
-            self.errors.append(f'RDMO XML Version: {root_version}')
-            return
+def resolve_file(file_name: str) -> Path:
+    file = Path(file_name).resolve()
+    if not file.exists():
+        raise ValueError(f"File does not exist: {file}")
+    return file
 
-        # step 3: create element dicts from xml
-        elements = OrderedDict()
-        try:
-            elements = flat_xml_to_elements(root)
-        except KeyError as e:
-            logger.info('Import failed with KeyError (%s)' % e)
-            self.errors.append(_('This is not a valid RDMO XML file.'))
-        except TypeError as e:
-            logger.info('Import failed with TypeError (%s)' % e)
-            self.errors.append(_('This is not a valid RDMO XML file.'))
-        except AttributeError as e:
-            logger.info('Import failed with AttributeError (%s)' % e)
-            self.errors.append(_('This is not a valid RDMO XML file.'))
-        if self.errors:
-            return
 
-        # step 3.1: validate elements for legacy versions
-        try:
-            pre_conversion_validate_missing_key_in_legacy_elements(elements, root_version)
-        except ValueError as e:
-            logger.info('Import failed with ValueError (%s)' % e)
-            self.errors.append(_('XML Parsing Error') + f': {e!s}')
-            self.errors.append(_('This is not a valid RDMO XML file.'))
-        if self.errors:
-            return
-        # step 4: convert elements from previous versions
-        elements = convert_elements(elements, root_version)
+def read_xml(file: Path) -> Tuple[Optional[xmlElement], Optional[str]]:
+    # step 2: parse xml and get the root
+    try:
+        root = ET.parse(file).getroot()
+        return root, None
+    except Exception as e:
+        return None, _('XML Parsing Error') + f': {e!s}'
+
+
+def validate_root(root: Optional[xmlElement]) -> Tuple[bool, Optional[str]]:
+    if root is None:
+        return False, _('The content of the XML file does not consist of well-formed data or markup.')
+    if root.tag != 'rdmo':
+        return False, _('This XML does not contain RDMO content.')
+    return True, None
+
+
+def validate_and_get_xml_version_from_root(root: xmlElement) -> Tuple[Optional[Version], list]:
+    unparsed_root_version = root.attrib.get('version') or DEFAULT_RDMO_XML_VERSION
+    root_version, rdmo_version = parse(unparsed_root_version), parse(RDMO_INSTANCE_VERSION)
+    if root_version > rdmo_version:
+        logger.info(f'Import failed version validation ({root_version} > {rdmo_version})')
+        errors = [
+            _('This RDMO XML file does not have a valid version number.'),
+            f'XML Version ({root_version}) is greater than RDMO instance version {rdmo_version}'
+        ]
+        return None, errors
+    return root_version, []
+
+
+def validate_legacy_elements(elements: dict, root_version: Version) -> list:
+
+    try:
+        validate_pre_conversion_for_missing_key_in_legacy_elements(elements, root_version)
+        return []
+    except ValueError as e:
+        logger.info(f'Import failed with ValueError ({e})')
+        errors = [
+            _('XML Parsing Error') + f': {e!s}',
+            _('This is not a valid RDMO XML file.')
+        ]
+        return errors
+
+
+def parse_elements(root: xmlElement) -> Tuple[Dict, Optional[str]]:
+    # step 3: create element dicts from xml
+    try:
+        elements = flat_xml_to_elements(root)
+        return elements, None
+    except (KeyError, TypeError, AttributeError) as e:
+        logger.info(f'Import failed with {type(e).__name__} ({e})')
+        return {}, _('This is not a valid RDMO XML file.')
+
+
+def parse_xml_to_elements(xml_file=None) -> Tuple[OrderedDict, list]:
+
+    file = resolve_file(xml_file)
+
+    root, read_error = read_xml(file)
+
+    errors = []
+    if read_error:
+        logger.error(read_error)
+        errors.append(read_error)
 
-        # step 5: order the elements and return
-        elements = order_elements(elements)
+    # step 2.1: validate the xml root
+    root_validation, root_validation_error = validate_root(root)
+    if root_validation is not True:
+        logger.error(f'Root element validation failed. {root_validation_error}')
+        errors.insert(0, root_validation_error)
+        return OrderedDict(), errors
 
-        logger.info(f'XML parsing of {self.file.name} success (length: {len(elements)}).')
-        return elements
+    # step 3: create element dicts from xml
+    elements, parsing_error = parse_elements(root)
+    if parsing_error is not None:
+        errors.append(parsing_error)
+        return OrderedDict(), errors
+
+    # step 3.1: validate version
+    root_version, version_errors = validate_and_get_xml_version_from_root(root)
+    if version_errors:
+        errors.extend(version_errors)
+        return OrderedDict(), errors
+
+    # step 3.1.1: validate the legacy elements
+    legacy_errors = validate_legacy_elements(elements, parse(root.attrib.get('version', DEFAULT_RDMO_XML_VERSION)))
+    if legacy_errors:
+        errors.extend(legacy_errors)
+        return OrderedDict(), errors
+
+    # step 4: convert elements from previous versions
+    elements = convert_elements(elements, parse(root.attrib.get('version', DEFAULT_RDMO_XML_VERSION)))
+
+    # step 5: order the elements and return
+    ordered_elements = order_elements(elements)
+
+    logger.info(f'XML parsing of {file.name} success (length: {len(elements)}).')
+
+    return ordered_elements, errors
 
 
 def read_xml_file(file_name, raise_exception=False):
@@ -122,7 +143,7 @@ def parse_xml_string(string):
         logger.error('Xml parsing error: ' + str(e))
 
 
-def flat_xml_to_elements(root):
+def flat_xml_to_elements(root) -> dict:
     elements = {}
     ns_map = get_ns_map(root)
     uri_attrib = get_ns_tag('dc:uri', ns_map)
@@ -206,9 +227,10 @@ def strip_ns(tag, ns_map):
 
 def convert_elements(elements, version: Version):
     if not isinstance(version, Version):
-        raise TypeError('Version should be a parsed version type. (parse(version))')
+        raise TypeError('Version should be of parsed version type.')
+
     if version < parse('2.0.0'):
-        pre_conversion_validate_missing_key_in_legacy_elements(elements, version)
+        validate_pre_conversion_for_missing_key_in_legacy_elements(elements, version)
         elements = convert_legacy_elements(elements)
 
     if version < parse('2.1.0'):
@@ -217,7 +239,7 @@ def convert_elements(elements, version: Version):
     return elements
 
 
-def pre_conversion_validate_missing_key_in_legacy_elements(elements, version: Version) -> None:
+def validate_pre_conversion_for_missing_key_in_legacy_elements(elements, version: Version) -> None:
     if version < parse('2.0.0'):
         models_in_elements = {i['model'] for i in elements.values()}
         if models_in_elements <= ELEMENTS_USING_KEY:
@@ -225,8 +247,7 @@ def pre_conversion_validate_missing_key_in_legacy_elements(elements, version: Ve
             return
         # inspect the elements for missing 'key' fields
         elements_to_inspect = filter(lambda x: x['model'] not in ELEMENTS_USING_KEY, elements.values())
-        inspected_elements_containing_key = list(filter(lambda x: 'key' in x, elements_to_inspect))
-        if not inspected_elements_containing_key:
+        if not any('key' in el for el in elements_to_inspect):
             raise ValueError(f"Missing legacy elements, elements containing 'key' were expected for this XML with version {version} and elements {models_in_elements}.")   # noqa: E501
 
 
@@ -321,7 +342,7 @@ def convert_additional_input(elements):
         if element['model'] == 'options.option':
             additional_input = element.get('additional_input')
             if additional_input in ['', 'text', 'textarea']:  # from Option.ADDITIONAL_INPUT_CHOICES
-                element['additional_input'] = additional_input
+                pass
             elif additional_input == 'True':
                 element['additional_input'] = 'text'
             else:
diff --git a/rdmo/management/management/commands/import.py b/rdmo/management/management/commands/import.py
index 73bec7cb5b..11b6d4d0ce 100644
--- a/rdmo/management/management/commands/import.py
+++ b/rdmo/management/management/commands/import.py
@@ -2,7 +2,7 @@
 
 from django.core.management.base import BaseCommand, CommandError
 
-from rdmo.core.xml import XmlToElementsParser
+from rdmo.core.xml import parse_xml_to_elements
 from rdmo.management.imports import import_elements
 
 logger = logging.getLogger(__name__)
@@ -16,14 +16,14 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
 
         try:
-            xml_parser = XmlToElementsParser(file_name=options['xmlfile'])
+            xml_parsed_elements, errors = parse_xml_to_elements(xml_file=options['xmlfile'])
         except CommandError as e:
             logger.info('Import failed with XML parsing errors.')
             raise CommandError(str(e)) from e
 
         # step 7: check if valid
-        if not xml_parser.is_valid():
+        if errors:
             logger.info('Import failed with XML validation errors.')
-            raise CommandError(" ".join(map(str, xml_parser.errors)))
+            raise CommandError(" ".join(map(str, errors)))
 
-        import_elements(xml_parser.parsed_elements)
+        import_elements(xml_parsed_elements)
diff --git a/rdmo/management/viewsets.py b/rdmo/management/viewsets.py
index 722c726c70..700134c827 100644
--- a/rdmo/management/viewsets.py
+++ b/rdmo/management/viewsets.py
@@ -12,7 +12,7 @@
 from rdmo.core.imports import handle_uploaded_file
 from rdmo.core.permissions import CanToggleElementCurrentSite
 from rdmo.core.utils import get_model_field_meta, is_truthy
-from rdmo.core.xml import XmlToElementsParser
+from rdmo.core.xml import parse_xml_to_elements
 
 from .constants import RDMO_MODEL_PATH_MAPPER
 from .imports import import_elements
@@ -42,20 +42,20 @@ def create(self, request, *args, **kwargs):
         else:
             import_tmpfile_name = handle_uploaded_file(uploaded_file)
         try:
-            # step 1.1: initialize XmlToElementsParser
+            # step 1.1: initialize parse_xml_to_elements
             # step 2-6: parse xml, validate and convert to
-            xml_parser = XmlToElementsParser(import_tmpfile_name)
+            xml_parsed_elements, errors = parse_xml_to_elements(xml_file=import_tmpfile_name)
         except ValidationError as e:
-            logger.info('Import failed with XML parsing errors.')
+            logger.info(f'Import failed with XML parsing errors. {", ".join(map(str, errors))}')
             raise ValidationError({'file': e}) from e
 
         # step 7: check if valid
-        if not xml_parser.is_valid():
-            logger.info('Import failed with XML validation errors.')
-            raise ValidationError({'file': xml_parser.errors})
+        if errors:
+            logger.info(f'Import failed with XML validation errors. {", ".join(map(str, errors))}')
+            raise ValidationError({'file': errors})
 
         # step 8: import the elements if save=True is set
-        imported_elements = import_elements(xml_parser.parsed_elements,
+        imported_elements = import_elements(xml_parsed_elements,
                                             save=is_truthy(request.POST.get('import')),
                                             request=request)
 

From faee0153ef51c447f7d60dee7de3b4deb8a06f5c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 6 Jun 2024 11:12:11 +0200
Subject: [PATCH 169/205] refactor(import): update FileNotFound error handling
 for xml import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/xml.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index 885356f9f9..24983f8cb7 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -19,11 +19,11 @@
 ELEMENTS_USING_KEY = {RDMO_MODELS['attribute']}
 
 
-def resolve_file(file_name: str) -> Path:
+def resolve_file(file_name: str) -> Tuple[Optional[Path], Optional[str]]:
     file = Path(file_name).resolve()
-    if not file.exists():
-        raise ValueError(f"File does not exist: {file}")
-    return file
+    if file.exists():
+        return file, None
+    return  None, _('This file does not exists.')
 
 
 def read_xml(file: Path) -> Tuple[Optional[xmlElement], Optional[str]]:
@@ -82,11 +82,16 @@ def parse_elements(root: xmlElement) -> Tuple[Dict, Optional[str]]:
 
 def parse_xml_to_elements(xml_file=None) -> Tuple[OrderedDict, list]:
 
-    file = resolve_file(xml_file)
+    errors = []
+
+    file, file_error = resolve_file(xml_file)
+    if file_error is not None:
+        logger.error(file_error)
+        errors.append(file_error)
+        return OrderedDict(), errors
 
     root, read_error = read_xml(file)
 
-    errors = []
     if read_error:
         logger.error(read_error)
         errors.append(read_error)

From 9f6b0b5b9ac4ce4fd5564b3f3059ddf566de217b Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 6 Jun 2024 11:14:31 +0200
Subject: [PATCH 170/205] tests(management): refactor and update import tests

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/helpers_xml.py          | 48 ++++++++++++++-----
 rdmo/management/tests/test_commands.py        | 33 +++++--------
 .../tests/test_import_conditions.py           | 12 ++---
 rdmo/management/tests/test_import_domain.py   | 12 ++---
 rdmo/management/tests/test_import_options.py  | 22 ++++-----
 .../management/tests/test_import_questions.py | 36 +++++++-------
 rdmo/management/tests/test_import_tasks.py    | 12 ++---
 rdmo/management/tests/test_import_views.py    | 12 ++---
 rdmo/management/tests/test_viewset_upload.py  |  2 +-
 9 files changed, 101 insertions(+), 88 deletions(-)

diff --git a/rdmo/management/tests/helpers_xml.py b/rdmo/management/tests/helpers_xml.py
index dc2c9ec19d..d5fd64a420 100644
--- a/rdmo/management/tests/helpers_xml.py
+++ b/rdmo/management/tests/helpers_xml.py
@@ -1,18 +1,42 @@
 
-from rdmo.core.xml import XmlToElementsParser
+from rdmo.core.xml import parse_xml_to_elements, read_xml, resolve_file
 
-xml_error_files = [
-    ('file-does-not-exist.xml', 'may not be blank'),
-    ('xml/error.xml', 'syntax error'),
-    ('xml/error-version.xml', 'RDMO XML Version: 99'),
-    ('xml/elements/legacy/catalog-error-key.xml', 'Missing legacy elements'),
-]
+xml_test_files = {
+    "xml/elements/catalogs.xml":
+        None,
+    "xml/elements/updated-and-changed/optionsets-1.xml":
+        None,
+    'file-does-not-exist.xml':
+        'This file does not exists',
+    "xml/error.xml":
+        "The content of the XML file does not consist of well-formed data or markup. XML Parsing Error: syntax error: line 1, column 0",  # noqa: E501
+    "xml/project.xml":
+        "This XML does not contain RDMO content.",
+    'xml/error-version.xml':
+        'This RDMO XML file does not have a valid version number. XML Version (99.9.9) is greater',
+    'xml/elements/legacy/catalog-error-key.xml':
+        'XML Parsing Error: Missing legacy elements',
+}
 
+xml_error_files = {k: v for k,v in xml_test_files.items() if v is not None}
+xml_error_files['file-does-not-exist.xml'] = 'This field may not be blank.'
 
-def read_xml_and_parse_to_elements(xml_file):
+def read_xml_and_parse_to_root_and_elements(file):
+    errors = []
 
-    xml_parser = XmlToElementsParser(file_name=xml_file)
-    if xml_parser.errors:
-        _msg = "\n".join(map(str, xml_parser.errors))
+    xml_file, file_error = resolve_file(file)
+    if file_error:
+        errors.append(file_error)
+
+    root, read_error = read_xml(xml_file)
+    if read_error:
+        errors.append(read_error)
+
+    xml_parsed_elements, xml_parsing_errors = parse_xml_to_elements(xml_file=xml_file)
+    if xml_parsing_errors:
+        errors.extend(xml_parsing_errors)
+
+    if errors:
+        _msg = "\n".join(map(str, xml_parsing_errors))
         raise ValueError(f"This test function should NOT raise any Exceptions. {_msg!s}")
-    return xml_parser.parsed_elements, xml_parser.root
+    return xml_parsed_elements, root
diff --git a/rdmo/management/tests/test_commands.py b/rdmo/management/tests/test_commands.py
index f410a3da08..6016a697d8 100644
--- a/rdmo/management/tests/test_commands.py
+++ b/rdmo/management/tests/test_commands.py
@@ -6,32 +6,21 @@
 from django.core.management import call_command
 from django.core.management.base import CommandError
 
+from rdmo.management.tests.helpers_xml import xml_test_files
 
-def test_import(db, settings):
-    xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
-    stdout, stderr = io.StringIO(), io.StringIO()
-
-    call_command('import', xml_file, stdout=stdout, stderr=stderr)
-
-    assert not stdout.getvalue()
-    assert not stderr.getvalue()
 
-
-def test_import_error(db, settings):
-    xml_file = Path(settings.BASE_DIR) / 'xml' / 'error.xml'
+@pytest.mark.parametrize("xml_file_path, error_message", xml_test_files.items())
+def test_import(db, settings, xml_file_path, error_message):
+    xml_file = Path(settings.BASE_DIR).joinpath(xml_file_path)
     stdout, stderr = io.StringIO(), io.StringIO()
 
-    with pytest.raises(CommandError) as e:
+    if error_message is None:
         call_command('import', xml_file, stdout=stdout, stderr=stderr)
 
-    assert str(e.value).startswith('The content of the xml file does not consist of well formed data or markup.')
-
-
-def test_import_error2(db, settings):
-    xml_file = Path(settings.BASE_DIR) / 'xml' / 'project.xml'
-    stdout, stderr = io.StringIO(), io.StringIO()
-
-    with pytest.raises(CommandError) as e:
-        call_command('import', xml_file, stdout=stdout, stderr=stderr)
+        assert not stdout.getvalue()
+        assert not stderr.getvalue()
+    else:
+        with pytest.raises(CommandError) as e:
+            call_command('import', xml_file, stdout=stdout, stderr=stderr)
 
-    assert str(e.value) == 'This XML does not contain RDMO content.'
+        assert str(e.value).startswith(error_message)
diff --git a/rdmo/management/tests/test_import_conditions.py b/rdmo/management/tests/test_import_conditions.py
index 5452f97c68..570342c3ed 100644
--- a/rdmo/management/tests/test_import_conditions.py
+++ b/rdmo/management/tests/test_import_conditions.py
@@ -10,7 +10,7 @@
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
 )
-from .helpers_xml import read_xml_and_parse_to_elements
+from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
 fields_to_be_changed = (('comment',),)
 
@@ -19,7 +19,7 @@ def test_create_conditions(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == Condition.objects.count() == 15
@@ -30,7 +30,7 @@ def test_create_conditions(db, settings):
 def test_update_conditions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 15
@@ -42,7 +42,7 @@ def test_update_conditions(db, settings):
 def test_update_conditions_with_changed_fields(db, settings, updated_fields):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     # breakpoint()
     elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=7)
     changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
@@ -61,7 +61,7 @@ def test_create_legacy_conditions(db, settings):
     Condition.objects.all().delete()
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'conditions.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == Condition.objects.count() == 15
@@ -72,7 +72,7 @@ def test_create_legacy_conditions(db, settings):
 def test_update_legacy_conditions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'conditions.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 15
diff --git a/rdmo/management/tests/test_import_domain.py b/rdmo/management/tests/test_import_domain.py
index 4f92be8bb6..b9d084863b 100644
--- a/rdmo/management/tests/test_import_domain.py
+++ b/rdmo/management/tests/test_import_domain.py
@@ -7,7 +7,7 @@
 from rdmo.management.imports import import_elements
 
 from .helpers_import_elements import _test_helper_change_fields_elements, _test_helper_filter_updated_and_changed
-from .helpers_xml import read_xml_and_parse_to_elements
+from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
 fields_to_be_changed = (('comment',),)
 
@@ -16,7 +16,7 @@ def test_create_domain(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == Attribute.objects.count() == 86
@@ -27,7 +27,7 @@ def test_create_domain(db, settings):
 def test_update_domain(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements)
@@ -39,7 +39,7 @@ def test_update_domain(db, settings):
 def test_update_attributes_with_changed_fields(db, settings, updated_fields):
     _change_count = Attribute.objects.count() / 2
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     # import initial elements from xml
     _el = import_elements(elements, save=True)
     # update the elements and call import again
@@ -62,7 +62,7 @@ def test_create_legacy_domain(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'domain.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 86
@@ -74,7 +74,7 @@ def test_create_legacy_domain(db, settings):
 def test_update_legacy_domain(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'domain.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 86
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index dd43823d1e..badae35761 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -12,7 +12,7 @@
     get_changed_elements,
 )
 from .helpers_models import delete_all_objects
-from .helpers_xml import read_xml_and_parse_to_elements
+from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
 fields_to_be_changed = (('comment',),)
 
@@ -31,7 +31,7 @@ def test_create_optionsets(db, settings):
     delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == 13
@@ -44,7 +44,7 @@ def test_create_optionsets(db, settings):
 def test_update_optionsets(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == 13
@@ -57,7 +57,7 @@ def test_update_optionsets_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 13
     # start test with fresh options in db
@@ -80,12 +80,12 @@ def test_update_optionsets_from_changed_xml(db, settings):
     delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 13
     # Act, import from xml that has changes
     xml_file_1 = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'updated-and-changed' / 'optionsets-1.xml'
-    elements_1, root_1 = read_xml_and_parse_to_elements(xml_file_1)
+    elements_1, root_1 = read_xml_and_parse_to_root_and_elements(xml_file_1)
     imported_elements_1 = import_elements(elements_1, save=False)
     assert imported_elements_1
     assert [i for i in imported_elements_1 if i[ELEMENT_DIFF_FIELD_NAME]]
@@ -126,7 +126,7 @@ def test_create_options(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == Option.objects.count() == 9
@@ -137,7 +137,7 @@ def test_create_options(db, settings):
 def test_update_options(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == 9
@@ -150,7 +150,7 @@ def test_update_options_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 9
     # start test with fresh options in db
@@ -174,7 +174,7 @@ def test_create_legacy_options(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'options.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == 12
@@ -187,7 +187,7 @@ def test_create_legacy_options(db, settings):
 def test_update_legacy_options(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'options.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(elements) == len(imported_elements) == 12
diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py
index d2ff7a0a3e..63a928cf06 100644
--- a/rdmo/management/tests/test_import_questions.py
+++ b/rdmo/management/tests/test_import_questions.py
@@ -11,7 +11,7 @@
     _test_helper_filter_updated_and_changed,
 )
 from .helpers_models import delete_all_objects
-from .helpers_xml import read_xml_and_parse_to_elements
+from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
 fields_to_be_changed = (('comment',),)
 
@@ -21,7 +21,7 @@ def test_create_catalogs(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 148
@@ -37,7 +37,7 @@ def test_create_catalogs(db, settings):
 def test_update_catalogs(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 148
@@ -50,7 +50,7 @@ def test_update_catalogs_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 148
     # start test with fresh elements in db
@@ -73,7 +73,7 @@ def test_create_sections(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 146
@@ -88,7 +88,7 @@ def test_create_sections(db, settings):
 def test_update_sections(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 146
@@ -101,7 +101,7 @@ def test_update_sections_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 146
     # start test with fresh elements in db
@@ -123,7 +123,7 @@ def test_create_pages(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 140
@@ -137,7 +137,7 @@ def test_create_pages(db, settings):
 def test_update_pages(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 140
@@ -150,7 +150,7 @@ def test_update_pages_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 140
     # start test with fresh elements in db
@@ -172,7 +172,7 @@ def test_create_questionsets(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == 10  # two questionsets appear twice in the export file
@@ -186,7 +186,7 @@ def test_create_questionsets(db, settings):
 def test_update_questionsets(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == 10  # two questionsets appear twice in the export file
@@ -200,7 +200,7 @@ def test_update_questionsets_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
     assert len(root) == 10  # two questionsets appear twice in the export file
     assert len(imported_elements) == 8
@@ -223,7 +223,7 @@ def test_create_questions(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 89
@@ -235,7 +235,7 @@ def test_create_questions(db, settings):
 def test_update_questions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 89
@@ -248,7 +248,7 @@ def test_update_questions_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 89
     # start test with fresh elements in db
@@ -270,7 +270,7 @@ def test_create_legacy_questions(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'questions.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 147
@@ -292,7 +292,7 @@ def test_create_legacy_questions(db, settings):
 def test_update_legacy_questions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'questions.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 147
diff --git a/rdmo/management/tests/test_import_tasks.py b/rdmo/management/tests/test_import_tasks.py
index 9c2a813629..3806e509b8 100644
--- a/rdmo/management/tests/test_import_tasks.py
+++ b/rdmo/management/tests/test_import_tasks.py
@@ -10,7 +10,7 @@
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
 )
-from .helpers_xml import read_xml_and_parse_to_elements
+from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
 fields_to_be_changed = (('comment',),)
 
@@ -20,7 +20,7 @@ def test_create_tasks(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'tasks.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == Task.objects.count() == 2
@@ -31,7 +31,7 @@ def test_create_tasks(db, settings):
 def test_update_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'tasks.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 2
@@ -43,7 +43,7 @@ def test_update_tasks(db, settings):
 def test_update_tasks_with_changed_fields(db, settings, updated_fields):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'tasks.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=1)
     changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
@@ -62,7 +62,7 @@ def test_create_legacy_tasks(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'tasks.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == Task.objects.count() == 2
@@ -73,7 +73,7 @@ def test_create_legacy_tasks(db, settings):
 def test_update_legacy_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'tasks.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 2
diff --git a/rdmo/management/tests/test_import_views.py b/rdmo/management/tests/test_import_views.py
index c6418499ab..8faed05bea 100644
--- a/rdmo/management/tests/test_import_views.py
+++ b/rdmo/management/tests/test_import_views.py
@@ -10,7 +10,7 @@
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
 )
-from .helpers_xml import read_xml_and_parse_to_elements
+from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
 fields_to_be_changed = (('comment',),)
 
@@ -20,7 +20,7 @@ def test_create_tasks(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'views.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == View.objects.count() == 3
@@ -31,7 +31,7 @@ def test_create_tasks(db, settings):
 def test_update_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'views.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 3
@@ -43,7 +43,7 @@ def test_update_tasks(db, settings):
 def test_update_views_with_changed_fields(db, settings, updated_fields):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'views.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=2)
     changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
@@ -62,7 +62,7 @@ def test_create_legacy_tasks(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'views.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == View.objects.count() == 3
@@ -73,7 +73,7 @@ def test_create_legacy_tasks(db, settings):
 def test_update_legacy_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'views.xml'
 
-    elements, root = read_xml_and_parse_to_elements(xml_file)
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
     imported_elements = import_elements(elements)
 
     assert len(root) == len(imported_elements) == 3
diff --git a/rdmo/management/tests/test_viewset_upload.py b/rdmo/management/tests/test_viewset_upload.py
index 72bd1fe9b9..16245cf3f8 100644
--- a/rdmo/management/tests/test_viewset_upload.py
+++ b/rdmo/management/tests/test_viewset_upload.py
@@ -110,7 +110,7 @@ def test_create_empty(db, client, username, password):
 
 
 @pytest.mark.parametrize('username,password', users)
-@pytest.mark.parametrize('xml_file_path, error_message', xml_error_files)
+@pytest.mark.parametrize('xml_file_path, error_message', xml_error_files.items())
 def test_create_error(db, client, username, password, xml_file_path, error_message):
     client.login(username=username, password=password)
 

From c1ffa8064ff8c03ed3b4cfc4faf948caffacbf97 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 7 Jun 2024 17:26:37 +0200
Subject: [PATCH 171/205] feat(import): add build_path and build_uri to import
 of Attribute

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/import_helpers.py |  1 +
 rdmo/core/imports.py        |  3 +++
 rdmo/domain/imports.py      | 12 ++++++++++--
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/rdmo/core/import_helpers.py b/rdmo/core/import_helpers.py
index 95d3e155ad..e551fdb0c5 100644
--- a/rdmo/core/import_helpers.py
+++ b/rdmo/core/import_helpers.py
@@ -23,6 +23,7 @@ class ExtraFieldDefaultHelper:
     field_name: str
     value: Union[str, bool, int, None] = None
     callback: Optional[Callable] = None
+    overwrite_in_element: bool = False
 
     def get_default(self, **kwargs):
         if self.callback is None:
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index e83eb408e0..a0fc149068 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -250,6 +250,9 @@ def set_extra_field(instance, field_name, element,
         # default_value
         extra_value = extra_field_helper.get_default(instance=instance,
                                                      key=field_name)
+        if extra_field_helper.overwrite_in_element:
+            element[field_name] = extra_value
+
     if extra_value is not None:
         setattr(instance, field_name, extra_value)
         # track changes
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 2a174a3830..919976234b 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -9,10 +9,15 @@
 logger = logging.getLogger(__name__)
 
 
-def get_default_path(instance: Optional[Attribute]=None):
+def build_attribute_path(instance: Optional[Attribute]=None):
     if instance is not None:
         return instance.build_path(instance.key, instance.parent)
 
+def build_attribute_uri(instance: Optional[Attribute]=None):
+    if instance is not None:
+        return instance.build_uri(instance.uri_prefix, instance.path)
+
+
 
 import_helper_attribute = ElementImportHelper(
     model=Attribute,
@@ -20,5 +25,8 @@ def get_default_path(instance: Optional[Attribute]=None):
     common_fields=('uri_prefix', 'key', 'comment'),
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     foreign_fields=('parent',),
-    extra_fields=[ExtraFieldDefaultHelper(field_name='path', callback=get_default_path)],
+    extra_fields=[
+        ExtraFieldDefaultHelper(field_name='path', callback=build_attribute_path, overwrite_in_element=True),
+        ExtraFieldDefaultHelper(field_name='uri', callback=build_attribute_uri, overwrite_in_element=True),
+    ]
 )

From ad146d28508fab9488eb1db5518eff0cba66044f Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 12 Jun 2024 17:40:03 +0200
Subject: [PATCH 172/205] refactor(import): rename to ExtraFieldHelper and move
 strings to ImportElementFields enum class

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/conditions/imports.py                    |   6 +-
 rdmo/core/import_helpers.py                   |  15 +--
 rdmo/core/imports.py                          | 118 +++++++++++-------
 rdmo/domain/imports.py                        |   6 +-
 rdmo/management/import_utils.py               |  20 ++-
 rdmo/management/imports.py                    |  10 +-
 .../tests/helpers_import_elements.py          |  12 +-
 .../tests/test_import_conditions.py           |   4 +-
 rdmo/management/tests/test_import_domain.py   |   4 +-
 rdmo/management/tests/test_import_options.py  |  14 +--
 .../management/tests/test_import_questions.py |  12 +-
 rdmo/management/tests/test_import_tasks.py    |   4 +-
 rdmo/management/tests/test_import_views.py    |   4 +-
 rdmo/options/imports.py                       |   8 +-
 rdmo/questions/imports.py                     |  30 ++---
 rdmo/tasks/imports.py                         |  10 +-
 rdmo/views/imports.py                         |   8 +-
 17 files changed, 155 insertions(+), 130 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index 695b72ff44..fa12fdfeb1 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -1,4 +1,4 @@
-from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldHelper
 
 from .models import Condition
 from .validators import ConditionLockedValidator, ConditionUniqueURIValidator
@@ -9,7 +9,7 @@
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
     foreign_fields=('source', 'target_option'),
     extra_fields=(
-        ExtraFieldDefaultHelper(field_name='relation', value=''),
-        ExtraFieldDefaultHelper(field_name='target_text', value=''),
+        ExtraFieldHelper(field_name='relation', value=''),
+        ExtraFieldHelper(field_name='target_text', value=''),
     ),
 )
diff --git a/rdmo/core/import_helpers.py b/rdmo/core/import_helpers.py
index e551fdb0c5..4d2be019b1 100644
--- a/rdmo/core/import_helpers.py
+++ b/rdmo/core/import_helpers.py
@@ -19,19 +19,20 @@ class ThroughInstanceMapper:
 
 
 @dataclass(frozen=True)
-class ExtraFieldDefaultHelper:
+class ExtraFieldHelper:
     field_name: str
     value: Union[str, bool, int, None] = None
     callback: Optional[Callable] = None
     overwrite_in_element: bool = False
 
-    def get_default(self, **kwargs):
-        if self.callback is None:
+    def get_value(self, **kwargs):
+        if self.value is not None:
             return self.value
-        else:
-            return self.get_default_from_callback(self.callback, kwargs)
+        elif self.callback is not None:
+            return self.get_value_from_callback(self.callback, kwargs)
+
     @staticmethod
-    def get_default_from_callback(callback, kwargs):
+    def get_value_from_callback(callback, kwargs):
         sig = signature(callback)
         kwargs = {k: val for k, val in kwargs.items() if k in sig.parameters}
         value = callback(**kwargs)
@@ -46,7 +47,7 @@ class ElementImportHelper:
     common_fields: Sequence[str] = field(default=ELEMENT_COMMON_FIELDS)
     lang_fields: Sequence[str] = field(default_factory=list)
     foreign_fields: Sequence[str] = field(default_factory=list)
-    extra_fields: Sequence[ExtraFieldDefaultHelper] = field(default_factory=list)
+    extra_fields: Sequence[ExtraFieldHelper] = field(default_factory=list)
     m2m_instance_fields: Sequence[str] = field(default_factory=list)
     m2m_through_instance_fields: Sequence[ThroughInstanceMapper] = field(default_factory=list)
     reverse_m2m_through_instance_fields: Sequence[ThroughInstanceMapper] = field(default_factory=list)
diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index a0fc149068..9f7d142d5e 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -2,6 +2,7 @@
 import tempfile
 import time
 from collections import defaultdict
+from enum import Enum
 from os.path import join as pj
 from random import randint
 from typing import List, Optional, Tuple, Union
@@ -13,14 +14,21 @@
 from rest_framework.utils import model_meta
 
 from rdmo.core.constants import RDMO_MODELS
-from rdmo.core.import_helpers import ExtraFieldDefaultHelper
+from rdmo.core.import_helpers import ExtraFieldHelper
 from rdmo.core.utils import get_languages
+from rdmo.core.validators import LockedValidator
 
 logger = logging.getLogger(__name__)
 
-ELEMENT_DIFF_FIELD_NAME = "updated_and_changed"
-NEW_DATA_FIELD = "new_data"
-CURRENT_DATA_FIELD = "current_data"
+
+class ImportElementFields(str, Enum):
+    DIFF = "updated_and_changed"
+    NEW = "new_data"
+    CURRENT = "current_data"
+    WARNINGS = "warnings"
+    ERRORS = "errors"
+    UPDATED = "updated"
+    CREATED = "created"
 
 
 def handle_uploaded_file(filedata):
@@ -73,19 +81,25 @@ def make_import_info_msg(verbose_name: str, created: bool, uri: Optional[str] =
 
 
 def _initialize_tracking_field(element: dict, element_field: str):
-    if element[ELEMENT_DIFF_FIELD_NAME].get(element_field) is None:
-        element[ELEMENT_DIFF_FIELD_NAME][element_field] = {
-            'errors': [],
-            'warnings': defaultdict(list)
+    if element[ImportElementFields.DIFF].get(element_field) is None:
+        element[ImportElementFields.DIFF][element_field] = {
+            ImportElementFields.ERRORS: [],
+            ImportElementFields.WARNINGS: defaultdict(list)
         }
+        return
+    if ImportElementFields.ERRORS not in element[ImportElementFields.DIFF][element_field]:
+        element[ImportElementFields.DIFF][element_field][ImportElementFields.ERRORS] = []
+    if ImportElementFields.WARNINGS not in element[ImportElementFields.DIFF][element_field]:
+        element[ImportElementFields.DIFF][element_field][ImportElementFields.WARNINGS] = defaultdict(list)
+
 
 
 def _append_warning(element: dict, element_field: str, warning: str):
-    element[ELEMENT_DIFF_FIELD_NAME][element_field]['warnings'][element['uri']].append(warning)
+    element[ImportElementFields.DIFF][element_field][ImportElementFields.WARNINGS][element['uri']].append(warning)
 
 
 def _append_error(element: dict, element_field: str, error: str):
-    element[ELEMENT_DIFF_FIELD_NAME][element_field]['errors'].append(error)
+    element[ImportElementFields.DIFF][element_field][ImportElementFields.ERRORS].append(error)
 
 
 def track_messages_on_element(element: dict, element_field: str, warning: Optional[str] = None,
@@ -99,11 +113,11 @@ def track_messages_on_element(element: dict, element_field: str, warning: Option
 
 
 def _initialize_track_changes_element_field(element: dict, element_field: str) -> None:
-    if ELEMENT_DIFF_FIELD_NAME not in element:
-        element[ELEMENT_DIFF_FIELD_NAME] = {}
+    if ImportElementFields.DIFF not in element:
+        element[ImportElementFields.DIFF] = {}
 
-    if element_field and element_field not in element[ELEMENT_DIFF_FIELD_NAME]:
-        element[ELEMENT_DIFF_FIELD_NAME][element_field] = {}
+    if element_field and element_field not in element[ImportElementFields.DIFF]:
+        element[ImportElementFields.DIFF][element_field] = {}
 
 
 def track_changes_on_element(element: dict,
@@ -121,8 +135,8 @@ def track_changes_on_element(element: dict,
         _get_field = element_field if instance_field is None else instance_field
         original_value = getattr(original, _get_field, '')
 
-    element[ELEMENT_DIFF_FIELD_NAME][element_field][CURRENT_DATA_FIELD] = original_value
-    element[ELEMENT_DIFF_FIELD_NAME][element_field][NEW_DATA_FIELD] = new_value
+    element[ImportElementFields.DIFF][element_field][ImportElementFields.CURRENT] = original_value
+    element[ImportElementFields.DIFF][element_field][ImportElementFields.NEW] = new_value
 
 
 def get_lang_field_values(field_name: str,
@@ -146,6 +160,13 @@ def get_lang_field_values(field_name: str,
     return ret
 
 
+def set_common_fields(instance, field_name, element, original=None):
+    value = element.get(field_name) or ''
+    setattr(instance, field_name, value)
+    # track changes for common fields
+    track_changes_on_element(element, field_name, new_value=value, original=original)
+
+
 def set_lang_field(instance, field_name, element, original=None):
     languages_field_values = get_lang_field_values(field_name, element=element)
     for lang_fields_value in languages_field_values:
@@ -188,7 +209,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
             field_name=field_name
         )
         logger.info(message)
-        element['errors'].append(message)  # errors is a list
+        element[ImportElementFields.ERRORS].append(message)  # errors is a list
         track_messages_on_element(element, field_name, error=message)
         return
 
@@ -206,7 +227,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
             instance_uri=element.get('uri')
         )
         logger.info(message)
-        element['warnings'][foreign_uri].append(message)
+        element[ImportElementFields.WARNINGS][foreign_uri].append(message)
         track_messages_on_element(element, field_name, warning=message)
     except foreign_model.MultipleObjectsReturned:
         message = '{foreign_model} {foreign_uri} for {instance_model} {instance_uri} returns multiple objects.'.format(
@@ -216,7 +237,7 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
             instance_uri=element.get('uri')
         )
         logger.info(message)
-        element['warnings'][foreign_uri].append(message)
+        element[ImportElementFields.WARNINGS][foreign_uri].append(message)
         track_messages_on_element(element, field_name, warning=message)
 
 
@@ -237,19 +258,19 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
             field_name=field_name,
         )
         logger.info(message)
-        element['errors'].append(message)
+        element[ImportElementFields.ERRORS].append(message)
         track_messages_on_element(element, field_name, error=message)
 
 
 def set_extra_field(instance, field_name, element,
-                    extra_field_helper: Optional[ExtraFieldDefaultHelper] = None, original=None) -> None:
+                    extra_field_helper: Optional[ExtraFieldHelper] = None, original=None) -> None:
     element_value = element.get(field_name)
     extra_value = element_value if element_value is not None else None
 
     if extra_value is None and extra_field_helper is not None:
         # default_value
-        extra_value = extra_field_helper.get_default(instance=instance,
-                                                     key=field_name)
+        extra_value = extra_field_helper.get_value(instance=instance,
+                                                   key=field_name)
         if extra_field_helper.overwrite_in_element:
             element[field_name] = extra_value
 
@@ -290,13 +311,13 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
     through_instances = list(getattr(instance, through_name).all())
 
     _track_changes = {}
-    _track_changes[NEW_DATA_FIELD] = []
-    _track_changes[CURRENT_DATA_FIELD] = []
+    _track_changes[ImportElementFields.NEW] = []
+    _track_changes[ImportElementFields.CURRENT] = []
 
     if original is not None:
         try:
             for _order, orig_field_instance in enumerate(getattr(original, through_name).order_by()):
-                _track_changes[CURRENT_DATA_FIELD].append({
+                _track_changes[ImportElementFields.CURRENT].append({
                     'uri': getattr(orig_field_instance, target_name).uri,
                     'order': orig_field_instance.order,
                     'model': get_rdmo_model_path(target_name, field_name)
@@ -324,7 +345,7 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
                     # remove the through_instance from the through_instances list so that it won't get removed
                     through_instances.remove(through_instance)
                 if original is not None:
-                    _track_changes[NEW_DATA_FIELD].append(target_element)
+                    _track_changes[ImportElementFields.NEW].append(target_element)
             except StopIteration:
                 # create a new item
                 if save:
@@ -334,7 +355,7 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
                         'order': order
                     }).save()
                 if original is not None:
-                    _track_changes[NEW_DATA_FIELD].append(target_element)
+                    _track_changes[ImportElementFields.NEW].append(target_element)
 
         except target_model.DoesNotExist:
             message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} does not exist.'.format(
@@ -344,7 +365,7 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
                 instance_uri=element.get('uri')
             )
             logger.info(message)
-            element['warnings'][target_uri].append(message)
+            element[ImportElementFields.WARNINGS][target_uri].append(message)
             track_messages_on_element(element, field_name, warning=message)
         except target_model.MultipleObjectsReturned:
             message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} returns multiple objects.'.format(  # noqa: E501
@@ -354,7 +375,7 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
                 instance_uri=element.get('uri')
             )
             logger.info(message)
-            element['warnings'][target_uri].append(message)
+            element[ImportElementFields.WARNINGS][target_uri].append(message)
             track_messages_on_element(element, field_name, warning=message)
     if save:
         # remove the remainders of the items list
@@ -362,15 +383,15 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
             if getattr(through_instance, target_name).uri_prefix == instance.uri_prefix:
                 through_instance.delete()
     # sort the tracked changes by order in-place
-    new_instance_data = sorted(_track_changes[NEW_DATA_FIELD], key=lambda k: k['order'])
-    original_instance_data = sorted(_track_changes[CURRENT_DATA_FIELD], key=lambda k: k['order'])
+    new_instance_data = sorted(_track_changes[ImportElementFields.NEW], key=lambda k: k['order'])
+    original_instance_data = sorted(_track_changes[ImportElementFields.CURRENT], key=lambda k: k['order'])
     track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data)
 
 
 def track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data):
     _initialize_track_changes_element_field(element, field_name)
-    element[ELEMENT_DIFF_FIELD_NAME][field_name][NEW_DATA_FIELD] = new_instance_data
-    element[ELEMENT_DIFF_FIELD_NAME][field_name][CURRENT_DATA_FIELD] = original_instance_data
+    element[ImportElementFields.DIFF][field_name][ImportElementFields.NEW] = new_instance_data
+    element[ImportElementFields.DIFF][field_name][ImportElementFields.CURRENT] = original_instance_data
     new_values = [i['uri'] for i in new_instance_data]
     original_values = [i['uri'] for i in original_instance_data]
     track_changes_on_element(element, field_name, new_value=new_values, original_value=original_values)
@@ -405,7 +426,7 @@ def set_m2m_instances(instance, element, field_name, original=None, save=None):
                 instance_uri=element.get('uri')
             )
             logger.info(message)
-            element['warnings'][foreign_uri].append(message)
+            element[ImportElementFields.WARNINGS][foreign_uri].append(message)
             track_messages_on_element(element, field_name, warning=message)
     if save:
         getattr(instance, field_name).set(foreign_instances)
@@ -431,13 +452,14 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
     through_info = model_meta.get_field_info(through_model)
     target_model = through_info.forward_relations[target_name].related_model
 
-    _track_changes = {}
-    _track_changes[NEW_DATA_FIELD] = []
-    _track_changes[CURRENT_DATA_FIELD] = []
+    _track_changes = {
+        ImportElementFields.NEW: [],
+        ImportElementFields.CURRENT: [],
+    }
     if original is not None:
         try:
             for _order, _through_instance in enumerate(getattr(original, through_name).order_by()):
-                _track_changes[CURRENT_DATA_FIELD].append({
+                _track_changes[ImportElementFields.CURRENT].append({
                     'uri': getattr(_through_instance, source_name).uri,
                     'order': _through_instance.order,
                     'model': get_rdmo_model_path(target_name, field_name),
@@ -458,7 +480,7 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
                     instance_uri=element.get('uri')
                 )
                 logger.info(message)
-                element['errors'].append(message)
+                element[ImportElementFields.ERRORS].append(message)
                 track_messages_on_element(element, field_name, error=message)
                 continue
             if save:
@@ -469,7 +491,7 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
                 through_instance.order = order
                 through_instance.save()
             if original is not None:
-                _track_changes[NEW_DATA_FIELD].append(target_element)
+                _track_changes[ImportElementFields.NEW].append(target_element)
 
         except target_model.DoesNotExist:
             message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} does not exist.'.format(
@@ -479,11 +501,11 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
                 instance_uri=element.get('uri')
             )
             logger.info(message)
-            element['warnings'][target_uri].append(message)
+            element[ImportElementFields.WARNINGS][target_uri].append(message)
             track_messages_on_element(element, field_name, warning=message)
     # sort the tracked changes by order in-place
-    new_instance_data = sorted(_track_changes[NEW_DATA_FIELD], key=lambda k: k['order'])
-    original_instance_data = sorted(_track_changes[CURRENT_DATA_FIELD], key=lambda k: k['order'])
+    new_instance_data = sorted(_track_changes[ImportElementFields.NEW], key=lambda k: k['order'])
+    original_instance_data = sorted(_track_changes[ImportElementFields.CURRENT], key=lambda k: k['order'])
     track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data)
 
 
@@ -505,11 +527,13 @@ def validate_instance(instance, element, *validators):
         )
         logger.info(message)
         _key = "FullClean"
-        element['errors'].append(message)
+        element[ImportElementFields.ERRORS].append(message)
         track_messages_on_element(element, _key, error=message)
         return
 
     for validator in validators:
+        if issubclass(validator, LockedValidator):
+            element['locked'] = False
         try:
             validator(instance=instance if instance.id else None)(vars(instance))
         except ValidationError as e:
@@ -528,8 +552,10 @@ def validate_instance(instance, element, *validators):
                 )
                 logger.info(message)
                 _key = validator.__qualname__
-                element['errors'].append(message)
+                element[ImportElementFields.ERRORS].append(message)
                 track_messages_on_element(element, _key, error=message)
+                if issubclass(validator, LockedValidator):
+                    element['locked'] = True
 
 
 def check_permissions(instance: models.Model, element_uri: str, user: models.Model) -> Optional[str]:
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index 919976234b..bb8ab1d437 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -1,7 +1,7 @@
 import logging
 from typing import Optional
 
-from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldHelper
 
 from .models import Attribute
 from .validators import AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator
@@ -26,7 +26,7 @@ def build_attribute_uri(instance: Optional[Attribute]=None):
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     foreign_fields=('parent',),
     extra_fields=[
-        ExtraFieldDefaultHelper(field_name='path', callback=build_attribute_path, overwrite_in_element=True),
-        ExtraFieldDefaultHelper(field_name='uri', callback=build_attribute_uri, overwrite_in_element=True),
+        ExtraFieldHelper(field_name='path', callback=build_attribute_path, overwrite_in_element=True),
+        ExtraFieldHelper(field_name='uri', callback=build_attribute_uri, overwrite_in_element=True),
     ]
 )
diff --git a/rdmo/management/import_utils.py b/rdmo/management/import_utils.py
index 6d964791d2..4511b7824d 100644
--- a/rdmo/management/import_utils.py
+++ b/rdmo/management/import_utils.py
@@ -4,14 +4,14 @@
 
 from rdmo.conditions.imports import import_helper_condition
 from rdmo.core.imports import (
-    ELEMENT_DIFF_FIELD_NAME,
+    ImportElementFields,
+    set_common_fields,
     set_extra_field,
     set_foreign_field,
     set_lang_field,
     set_m2m_instances,
     set_m2m_through_instances,
     set_reverse_m2m_through_instance,
-    track_changes_on_element,
 )
 from rdmo.domain.imports import import_helper_attribute
 from rdmo.options.imports import import_helper_option, import_helper_optionset
@@ -39,11 +39,11 @@
     "views.view": import_helper_view
 }
 IMPORT_ELEMENT_INIT_DICT = {
-        'warnings': lambda: defaultdict(list),
-        'errors': list,
-        'created': bool,
-        'updated': bool,
-        ELEMENT_DIFF_FIELD_NAME: dict,
+        ImportElementFields.WARNINGS: lambda: defaultdict(list),
+        ImportElementFields.ERRORS: list,
+        ImportElementFields.CREATED: bool,
+        ImportElementFields.UPDATED: bool,
+        ImportElementFields.DIFF: dict,
     }
 
 
@@ -66,11 +66,7 @@ def apply_field_values(instance, element, import_helper, uploaded_uris, original
     # start to set values on the instance
     # set common field values from element on instance
     for field in import_helper.common_fields:
-        value = element.get(field) or ''
-        setattr(instance, field, value)
-        if element['updated']:
-            # track changes for common fields
-            track_changes_on_element(element, field, new_value=value, original=original)
+        set_common_fields(instance, field, element, original=original)
     # set language fields
     for field in import_helper.lang_fields:
         set_lang_field(instance, field, element, original=original)
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index 7ac01ca99b..c517cef14c 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -29,9 +29,11 @@ def import_elements(uploaded_elements: Dict, save: bool = True, request: Optiona
     uploaded_uris = set(uploaded_elements.keys())
     current_site = get_current_site(request)
     for _uri, uploaded_element in uploaded_elements.items():
-        element = import_element(element=uploaded_element, save=save,
-                                  uploaded_uris=uploaded_uris,
-                                    request=request, current_site=current_site)
+        element = import_element(element=uploaded_element,
+                                 save=save,
+                                 uploaded_uris=uploaded_uris,
+                                 request=request,
+                                 current_site=current_site)
         element['warnings'] = {k: val for k, val in element['warnings'].items() if k not in uploaded_uris}
         imported_elements.append(element)
     return imported_elements
@@ -83,7 +85,7 @@ def import_element(
     updated = not created
     element['created'] = created
     element['updated'] = updated
-    # INFO: the dict element[ELEMENT_DIFF_FIELD_NAME] is filled by calling track_changes_on_element
+    # INFO: the dict element[FieldNames.diff.value] is filled by calling track_changes_on_element
 
     element = strip_uri_prefix_endswith_slash(element)
     # start to set values on the instance
diff --git a/rdmo/management/tests/helpers_import_elements.py b/rdmo/management/tests/helpers_import_elements.py
index 0030c48153..6ad4d8caa1 100644
--- a/rdmo/management/tests/helpers_import_elements.py
+++ b/rdmo/management/tests/helpers_import_elements.py
@@ -2,7 +2,7 @@
 from functools import partial
 from typing import Dict, List, Optional, Tuple, Union
 
-from rdmo.core.imports import CURRENT_DATA_FIELD, ELEMENT_DIFF_FIELD_NAME, NEW_DATA_FIELD, track_changes_on_element
+from rdmo.core.imports import ImportElementFields, track_changes_on_element
 from rdmo.management.import_utils import initialize_import_element_dict
 
 UPDATE_FIELD_FUNCS = {
@@ -16,12 +16,12 @@ def filter_changed_fields(element, updated_fields=None) -> bool:
     _changed = element.get('changed', False)
     if updated_fields is None:
         return _changed
-    changes = element.get(ELEMENT_DIFF_FIELD_NAME, {})
+    changes = element.get(ImportElementFields.DIFF, {})
     for field, diff in changes.items():
         if field not in updated_fields:
             continue
-        _new_value = diff.get(NEW_DATA_FIELD)
-        _current_value = diff.get(CURRENT_DATA_FIELD)
+        _new_value = diff.get(ImportElementFields.NEW)
+        _current_value = diff.get(ImportElementFields.CURRENT)
         if _new_value != _current_value:
             return True
     return _changed
@@ -31,8 +31,8 @@ def get_changed_elements(elements: List[Dict]) -> Dict[str, Dict[str,Union[bool,
     for element in elements:
 
         changed_fields = []
-        for key, diff_field in element[ELEMENT_DIFF_FIELD_NAME].items():
-            if diff_field[NEW_DATA_FIELD] != diff_field[CURRENT_DATA_FIELD]:
+        for key, diff_field in element[ImportElementFields.DIFF].items():
+            if diff_field[ImportElementFields.NEW] != diff_field[ImportElementFields.CURRENT]:
                 changed_fields += key
         if changed_fields:
             changed_elements[element['uri']] = {
diff --git a/rdmo/management/tests/test_import_conditions.py b/rdmo/management/tests/test_import_conditions.py
index 570342c3ed..bcebe7f4bd 100644
--- a/rdmo/management/tests/test_import_conditions.py
+++ b/rdmo/management/tests/test_import_conditions.py
@@ -3,7 +3,7 @@
 import pytest
 
 from rdmo.conditions.models import Condition
-from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
+from rdmo.core.imports import ImportElementFields
 from rdmo.management.imports import import_elements
 
 from .helpers_import_elements import (
@@ -54,7 +54,7 @@ def test_update_conditions_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_legacy_conditions(db, settings):
diff --git a/rdmo/management/tests/test_import_domain.py b/rdmo/management/tests/test_import_domain.py
index b9d084863b..3b3ac88595 100644
--- a/rdmo/management/tests/test_import_domain.py
+++ b/rdmo/management/tests/test_import_domain.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
+from rdmo.core.imports import ImportElementFields
 from rdmo.domain.models import Attribute
 from rdmo.management.imports import import_elements
 
@@ -54,7 +54,7 @@ def test_update_attributes_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_legacy_domain(db, settings):
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index badae35761..4e66dde484 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from rdmo.core.imports import CURRENT_DATA_FIELD, ELEMENT_DIFF_FIELD_NAME, NEW_DATA_FIELD
+from rdmo.core.imports import ImportElementFields
 from rdmo.management.imports import import_elements
 from rdmo.options.models import Option, OptionSet
 
@@ -72,7 +72,7 @@ def test_update_optionsets_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_update_optionsets_from_changed_xml(db, settings):
@@ -88,7 +88,7 @@ def test_update_optionsets_from_changed_xml(db, settings):
     elements_1, root_1 = read_xml_and_parse_to_root_and_elements(xml_file_1)
     imported_elements_1 = import_elements(elements_1, save=False)
     assert imported_elements_1
-    assert [i for i in imported_elements_1 if i[ELEMENT_DIFF_FIELD_NAME]]
+    assert [i for i in imported_elements_1 if i[ImportElementFields.DIFF]]
     warnings_elements = [i for i in imported_elements_1 if i['warnings']]
     assert len(warnings_elements) == 1
 
@@ -102,9 +102,9 @@ def test_update_optionsets_from_changed_xml(db, settings):
     # the test changes are simply the reversed order of the options
     test_optionset_changed_options = test_optionset['original']['options'][::-1]
     assert optionset_element
-    assert "options" in optionset_element[ELEMENT_DIFF_FIELD_NAME]
-    assert optionset_element[ELEMENT_DIFF_FIELD_NAME]['options'][CURRENT_DATA_FIELD] == test_optionset['original']['options']  # noqa: E501
-    assert optionset_element[ELEMENT_DIFF_FIELD_NAME]['options'][NEW_DATA_FIELD] == test_optionset_changed_options
+    assert "options" in optionset_element[ImportElementFields.DIFF]
+    assert optionset_element[ImportElementFields.DIFF]['options'][ImportElementFields.CURRENT] == test_optionset['original']['options']  # noqa: E501
+    assert optionset_element[ImportElementFields.DIFF]['options'][ImportElementFields.NEW] == test_optionset_changed_options  # noqa: E501
 
     # now save the elements_1
     _imported_elements_1_save = import_elements(elements_1, save=True)
@@ -164,7 +164,7 @@ def test_update_options_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 
diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py
index 63a928cf06..f18e1995c5 100644
--- a/rdmo/management/tests/test_import_questions.py
+++ b/rdmo/management/tests/test_import_questions.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
+from rdmo.core.imports import ImportElementFields
 from rdmo.management.imports import import_elements
 from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section
 
@@ -65,7 +65,7 @@ def test_update_catalogs_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_sections(db, settings):
@@ -115,7 +115,7 @@ def test_update_sections_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_pages(db, settings):
@@ -164,7 +164,7 @@ def test_update_pages_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_questionsets(db, settings):
@@ -215,7 +215,7 @@ def test_update_questionsets_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_questions(db, settings):
@@ -262,7 +262,7 @@ def test_update_questions_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_legacy_questions(db, settings):
diff --git a/rdmo/management/tests/test_import_tasks.py b/rdmo/management/tests/test_import_tasks.py
index 3806e509b8..0d0bbf834c 100644
--- a/rdmo/management/tests/test_import_tasks.py
+++ b/rdmo/management/tests/test_import_tasks.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
+from rdmo.core.imports import ImportElementFields
 from rdmo.management.imports import import_elements
 from rdmo.tasks.models import Task
 
@@ -54,7 +54,7 @@ def test_update_tasks_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_legacy_tasks(db, settings):
diff --git a/rdmo/management/tests/test_import_views.py b/rdmo/management/tests/test_import_views.py
index 8faed05bea..eadb55e5e6 100644
--- a/rdmo/management/tests/test_import_views.py
+++ b/rdmo/management/tests/test_import_views.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from rdmo.core.imports import ELEMENT_DIFF_FIELD_NAME
+from rdmo.core.imports import ImportElementFields
 from rdmo.management.imports import import_elements
 from rdmo.views.models import View
 
@@ -54,7 +54,7 @@ def test_update_views_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ELEMENT_DIFF_FIELD_NAME] == imported[ELEMENT_DIFF_FIELD_NAME]
+        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_legacy_tasks(db, settings):
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index a17e0cf12e..c94d0bb4f0 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -1,4 +1,4 @@
-from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper, ThroughInstanceMapper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldHelper, ThroughInstanceMapper
 
 from .models import Option, OptionSet
 from .validators import (
@@ -13,8 +13,8 @@
     model_path = "options.optionset",
     validators = (OptionSetLockedValidator, OptionSetUniqueURIValidator),
     extra_fields = (
-        ExtraFieldDefaultHelper(field_name='order'),
-        ExtraFieldDefaultHelper(field_name='provider_key', value=''),
+        ExtraFieldHelper(field_name='order'),
+        ExtraFieldHelper(field_name='provider_key', value=''),
     ),
     m2m_instance_fields = ('conditions', ),
     m2m_through_instance_fields = [
@@ -33,7 +33,7 @@
     validators = (OptionLockedValidator, OptionUniqueURIValidator),
     lang_fields = ('text', 'help', 'view_text'),
     extra_fields = (
-        ExtraFieldDefaultHelper(field_name='additional_input', value=Option.ADDITIONAL_INPUT_NONE),
+        ExtraFieldHelper(field_name='additional_input', value=Option.ADDITIONAL_INPUT_NONE),
         ),
     reverse_m2m_through_instance_fields = [
         ThroughInstanceMapper(
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index f554a55c2a..491a1dad8f 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -1,4 +1,4 @@
-from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper, ThroughInstanceMapper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldHelper, ThroughInstanceMapper
 
 from ..core.constants import VALUE_TYPE_TEXT
 from .models import Catalog, Page, Question, QuestionSet, Section
@@ -22,8 +22,8 @@
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
     lang_fields=('help', 'title'),
     extra_fields = (
-        ExtraFieldDefaultHelper(field_name='order'),
-        ExtraFieldDefaultHelper(field_name='available'),
+        ExtraFieldHelper(field_name='order'),
+        ExtraFieldHelper(field_name='available', value=True),
     ),
     m2m_through_instance_fields=[
         ThroughInstanceMapper(
@@ -60,7 +60,7 @@
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
     extra_fields = (
-        ExtraFieldDefaultHelper(field_name='is_collection'),
+        ExtraFieldHelper(field_name='is_collection'),
     ),
     m2m_instance_fields = ('conditions', ),
     m2m_through_instance_fields=[
@@ -88,16 +88,16 @@
     lang_fields=('text', 'help', 'default_text', 'verbose_name'),
     foreign_fields=('attribute', 'default_option'),
     extra_fields=(
-        ExtraFieldDefaultHelper(field_name='is_collection'),
-        ExtraFieldDefaultHelper(field_name='is_optional'),
-        ExtraFieldDefaultHelper(field_name='default_external_id', value=''),
-        ExtraFieldDefaultHelper(field_name='widget_type', callback=get_widget_type_or_default),
-        ExtraFieldDefaultHelper(field_name='value_type', value=VALUE_TYPE_TEXT),
-        ExtraFieldDefaultHelper(field_name='minimum'),
-        ExtraFieldDefaultHelper(field_name='maximum'),
-        ExtraFieldDefaultHelper(field_name='step'),
-        ExtraFieldDefaultHelper(field_name='unit', value=''),
-        ExtraFieldDefaultHelper(field_name='width'),
+        ExtraFieldHelper(field_name='is_collection'),
+        ExtraFieldHelper(field_name='is_optional'),
+        ExtraFieldHelper(field_name='default_external_id', value=''),
+        ExtraFieldHelper(field_name='widget_type', callback=get_widget_type_or_default),
+        ExtraFieldHelper(field_name='value_type', value=VALUE_TYPE_TEXT),
+        ExtraFieldHelper(field_name='minimum'),
+        ExtraFieldHelper(field_name='maximum'),
+        ExtraFieldHelper(field_name='step'),
+        ExtraFieldHelper(field_name='unit', value=''),
+        ExtraFieldHelper(field_name='width'),
     ),
     m2m_instance_fields=('conditions', 'optionsets'),
     reverse_m2m_through_instance_fields=[
@@ -118,7 +118,7 @@
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
     extra_fields=(
-        ExtraFieldDefaultHelper(field_name='is_collection'),
+        ExtraFieldHelper(field_name='is_collection'),
     ),
     m2m_instance_fields=('conditions', ),
 
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index c2e205bf59..04502a3a96 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -1,4 +1,4 @@
-from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldHelper
 
 from .models import Task
 from .validators import TaskLockedValidator, TaskUniqueURIValidator
@@ -10,10 +10,10 @@
     lang_fields=('title', 'text'),
     foreign_fields=('start_attribute', 'end_attribute'),
     extra_fields=(
-        ExtraFieldDefaultHelper(field_name='order'),
-        ExtraFieldDefaultHelper(field_name='days_before'),
-        ExtraFieldDefaultHelper(field_name='days_after'),
-        ExtraFieldDefaultHelper(field_name='available'),
+        ExtraFieldHelper(field_name='order'),
+        ExtraFieldHelper(field_name='days_before'),
+        ExtraFieldHelper(field_name='days_after'),
+        ExtraFieldHelper(field_name='available', value=True),
     ),
     m2m_instance_fields=('catalogs', 'conditions'),
 )
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 8a2e3e4cbb..45aee0ef6a 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -1,4 +1,4 @@
-from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldDefaultHelper
+from rdmo.core.import_helpers import ElementImportHelper, ExtraFieldHelper
 
 from .models import View
 from .validators import ViewLockedValidator, ViewUniqueURIValidator
@@ -9,9 +9,9 @@
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
     lang_fields=('help', 'title'),
     extra_fields=(
-        ExtraFieldDefaultHelper(field_name='order'),
-        ExtraFieldDefaultHelper(field_name='template'),
-        ExtraFieldDefaultHelper(field_name='available'),
+        ExtraFieldHelper(field_name='order'),
+        ExtraFieldHelper(field_name='template'),
+        ExtraFieldHelper(field_name='available', value=True),
     ),
     m2m_instance_fields=('catalogs',),
 )

From 9e0eb3746ee390d34cd4c1a7887588aebfc5ebef Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 13 Jun 2024 16:50:10 +0200
Subject: [PATCH 173/205] feat(import): fix order elements in core/xml

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/xml.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index 24983f8cb7..63328c85de 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -367,6 +367,10 @@ def append_element(ordered_elements, unordered_elements, uri, element):
     if element is None:
         return
 
+    has_list_or_dict = any(i for i in element.values() if isinstance(i, dict) or isinstance(i, list))
+    if has_list_or_dict and uri not in ordered_elements:
+        ordered_elements[uri] = element
+
     for element_value in element.values():
         if isinstance(element_value, dict):
             sub_uri = element_value.get('uri')

From c6128041ea71f5f9eeaa036c5af2366cc6acc2e4 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 13 Jun 2024 16:54:51 +0200
Subject: [PATCH 174/205] feat(import): prevent overwrite of comment and fix
 int, str diff for order

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 9f7d142d5e..b966e2e197 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -131,9 +131,14 @@ def track_changes_on_element(element: dict,
 
     _initialize_track_changes_element_field(element, element_field)
 
-    if original_value is None:
-        _get_field = element_field if instance_field is None else instance_field
-        original_value = getattr(original, _get_field, '')
+    if original_value is None and original is not None:
+        lookup_field = element_field if instance_field is None else instance_field
+        original_value = getattr(original, lookup_field, '')
+
+    if isinstance(new_value,str) and isinstance(original_value,int):
+        # typecasting of original value to str, for comparison '0' == 0
+        # specific edge-case, maybe generalize later
+        original_value = str(original_value)
 
     element[ImportElementFields.DIFF][element_field][ImportElementFields.CURRENT] = original_value
     element[ImportElementFields.DIFF][element_field][ImportElementFields.NEW] = new_value
@@ -161,10 +166,17 @@ def get_lang_field_values(field_name: str,
 
 
 def set_common_fields(instance, field_name, element, original=None):
-    value = element.get(field_name) or ''
-    setattr(instance, field_name, value)
+    element_value = element.get(field_name) or ''
+    if field_name == 'comment' and original is not None:
+        # prevent overwrite with an empty comment when updating an element
+        original_value = getattr(original, field_name)
+        if original_value and not element_value:
+            element_value = original_value
+            element[field_name] = element_value
+
+    setattr(instance, field_name, element_value)
     # track changes for common fields
-    track_changes_on_element(element, field_name, new_value=value, original=original)
+    track_changes_on_element(element, field_name, new_value=element_value, original=original)
 
 
 def set_lang_field(instance, field_name, element, original=None):
@@ -264,15 +276,23 @@ def set_foreign_field(instance, field_name, element, uploaded_uris=None, origina
 
 def set_extra_field(instance, field_name, element,
                     extra_field_helper: Optional[ExtraFieldHelper] = None, original=None) -> None:
-    element_value = element.get(field_name)
-    extra_value = element_value if element_value is not None else None
+
+    extra_value = None
+    if field_name in element:
+        element_value = element.get(field_name)
+        extra_value = element_value
+    else:
+        instance_value = getattr(instance, field_name)
+        element[field_name] = instance_value
+        extra_value = instance_value
 
     if extra_value is None and extra_field_helper is not None:
         # default_value
         extra_value = extra_field_helper.get_value(instance=instance,
                                                    key=field_name)
-        if extra_field_helper.overwrite_in_element:
-            element[field_name] = extra_value
+
+    if extra_field_helper.overwrite_in_element:
+        element[field_name] = extra_value
 
     if extra_value is not None:
         setattr(instance, field_name, extra_value)

From 8d982fbdef49bcbfcface48a6a28df5cff5b69e9 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 13 Jun 2024 17:06:31 +0200
Subject: [PATCH 175/205] feat(import): update import of available field

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/questions/imports.py | 2 +-
 rdmo/tasks/imports.py     | 2 +-
 rdmo/views/imports.py     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 491a1dad8f..c0b1e67306 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -23,7 +23,7 @@
     lang_fields=('help', 'title'),
     extra_fields = (
         ExtraFieldHelper(field_name='order'),
-        ExtraFieldHelper(field_name='available', value=True),
+        ExtraFieldHelper(field_name='available', overwrite_in_element=True),
     ),
     m2m_through_instance_fields=[
         ThroughInstanceMapper(
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 04502a3a96..2f7099b1e8 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -13,7 +13,7 @@
         ExtraFieldHelper(field_name='order'),
         ExtraFieldHelper(field_name='days_before'),
         ExtraFieldHelper(field_name='days_after'),
-        ExtraFieldHelper(field_name='available', value=True),
+        ExtraFieldHelper(field_name='available', overwrite_in_element=True),
     ),
     m2m_instance_fields=('catalogs', 'conditions'),
 )
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index 45aee0ef6a..bc8db9487f 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -11,7 +11,7 @@
     extra_fields=(
         ExtraFieldHelper(field_name='order'),
         ExtraFieldHelper(field_name='template'),
-        ExtraFieldHelper(field_name='available', value=True),
+        ExtraFieldHelper(field_name='available', overwrite_in_element=True),
     ),
     m2m_instance_fields=('catalogs',),
 )

From ec3db4b72e35ee854f08386c320cf117016b598a Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 13 Jun 2024 17:10:04 +0200
Subject: [PATCH 176/205] js(import): update import common components

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/common/Errors.js     |  6 +--
 .../components/import/common/FieldRowValue.js | 45 ++++++++++++-------
 .../js/components/import/common/Fields.js     |  9 ++--
 .../import/common/SelectCheckbox.js           | 22 +++++++++
 .../js/components/import/common/Warnings.js   |  6 +--
 5 files changed, 63 insertions(+), 25 deletions(-)
 create mode 100644 rdmo/management/assets/js/components/import/common/SelectCheckbox.js

diff --git a/rdmo/management/assets/js/components/import/common/Errors.js b/rdmo/management/assets/js/components/import/common/Errors.js
index c711b846f7..0a03bb7689 100644
--- a/rdmo/management/assets/js/components/import/common/Errors.js
+++ b/rdmo/management/assets/js/components/import/common/Errors.js
@@ -8,7 +8,7 @@ const generateErrorMessages = (messages, key) =>
   messages.map(message => <li className="text-danger" key={key}>{message}</li>)
 
 // Helper function to prepare the list of errors
-const prepareErrorsList = (errors) => {
+export const prepareErrorsList = (errors) => {
   // Filter out duplicate errors
   const uniqueErrors = [...new Set(errors)]
 
@@ -25,11 +25,11 @@ const Errors = ({ element, showTitle = false }) => {
   return !isEmpty(element.errors) && (
     <div className="row text-danger mt-10">
       {showTitle && (
-        <div className="col-sm-3 text-right">
+        <div className="col-sm-3 mb-5 mt-5">
           {gettext('Errors')}
         </div>
       )}
-      <div className={`col-sm-${showTitle ? 9 : 12}`}>
+      <div className="col-sm-12">
         <ul className="list-unstyled">
           {listErrorMessages}
         </ul>
diff --git a/rdmo/management/assets/js/components/import/common/FieldRowValue.js b/rdmo/management/assets/js/components/import/common/FieldRowValue.js
index e6e891c3cb..eb094236cb 100644
--- a/rdmo/management/assets/js/components/import/common/FieldRowValue.js
+++ b/rdmo/management/assets/js/components/import/common/FieldRowValue.js
@@ -5,22 +5,37 @@ import isString from 'lodash/isString'
 import isUndefined from 'lodash/isUndefined'
 import truncate from 'lodash/truncate'
 import { codeClass } from '../../../constants/elements'
+import {isNull} from 'lodash'
 
-const FieldRowValue = ({ value }) => (
-  <div className="col-sm-12">
-    {Array.isArray(value) && (
-      <ul className="list-unstyled mb-0">
-        {value.map((el) => (
-          <li key={uniqueId()}>
-            <code className={codeClass[el.type]}>{el.uri}</code>
-          </li>
-        ))}
-      </ul>
-    )}
-    {!isUndefined(value.uri) && <code className={codeClass[value.type]}>{value.uri}</code>}
-    {isString(value) && <span>{truncate(value, { length: 512 })}</span>}
-  </div>
-)
+const serializeValue = (value) => {
+  if (value === null) return ''
+  if (value === true) return 'true'
+  if (value === false) return 'false'
+  if (Array.isArray(value)) return value
+  if (isString(value)) return value
+  if (typeof value === 'number') return value.toString()
+  return value
+}
+
+
+const FieldRowValue = ({ value }) => {
+  const serializedValue = serializeValue(value)
+  return  (
+    <div className="col-sm-12">
+      {Array.isArray(value) && (
+        <ul className="list-unstyled mb-0">
+          {value.map((el) => (
+            <li key={uniqueId()}>
+              <code className={codeClass[el.type]}>{el.uri}</code>
+            </li>
+          ))}
+        </ul>
+      )}
+      {!isNull(value) && !isUndefined(value.uri) && <code className={codeClass[value.type]}>{value.uri}</code>}
+      {isString(serializedValue) && <span>{truncate(value, { length: 512 })}</span>}
+    </div>
+  )
+}
 
 FieldRowValue.propTypes = {
   value: PropTypes.any.isRequired,
diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index 97a2e68ae0..5fab7dfc94 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -5,22 +5,23 @@ import uniqueId from 'lodash/uniqueId'
 import FieldRow from './FieldRow'
 
 const excludeKeys = [
-  'created',
-  'errors',
   'import',
   'key',
   'model',
   'show',
   'type',
-  'updated',
   'uri',
   'uri_path',
   'uri_prefix',
   'valid',
+  'created',
+  'updated',
+  'errors',
   'warnings',
   'updated_and_changed',
   'changed',
   'changedFields',
+  'locked'
 ]
 
 const Fields = ({ element }) => (
@@ -28,7 +29,7 @@ const Fields = ({ element }) => (
     {Object.entries(element)
       .sort()
       .map(([key, value]) => {
-        if (!isNil(value) && !excludeKeys.includes(key)) {
+        if ((!isNil(value) || key in element.updated_and_changed) && !excludeKeys.includes(key)) {
           return <FieldRow key={uniqueId()} element={element} keyName={key} value={value} />
         }
         return null
diff --git a/rdmo/management/assets/js/components/import/common/SelectCheckbox.js b/rdmo/management/assets/js/components/import/common/SelectCheckbox.js
new file mode 100644
index 0000000000..c12b897dd6
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/common/SelectCheckbox.js
@@ -0,0 +1,22 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+import { CodeLink } from '../../common/Links'
+import { codeClass, verboseNames } from '../../../constants/elements'
+
+const SelectCheckbox = ({ element, toggleImport, updateShowField }) => (
+  <div className="checkbox">
+    <label className="mr-5">
+      <input type="checkbox" checked={element.import} onChange={toggleImport} />
+      <strong>{verboseNames[element.model]}{' '}</strong>
+    </label>
+    <CodeLink className={codeClass[element.model]} uri={element.uri} onClick={updateShowField} />
+  </div>
+)
+
+SelectCheckbox.propTypes = {
+  element: PropTypes.object.isRequired,
+  toggleImport: PropTypes.func.isRequired,
+  updateShowField: PropTypes.func.isRequired
+}
+
+export default SelectCheckbox
diff --git a/rdmo/management/assets/js/components/import/common/Warnings.js b/rdmo/management/assets/js/components/import/common/Warnings.js
index d288087b5d..e079a0b49d 100644
--- a/rdmo/management/assets/js/components/import/common/Warnings.js
+++ b/rdmo/management/assets/js/components/import/common/Warnings.js
@@ -26,11 +26,11 @@ const Warnings = ({element, showTitle = false, shouldShowURI = true}) => {
   return (
     <div className="row text-warning mt-10">
       {showTitle === true && listWarningMessages.length > 0 &&
-        <div className="col-sm-3 text-right">
+        <div className="col-sm-3 mb-5 mt-5">
           {'Warnings'}
         </div>
       }
-      <div className="col-sm-9">
+      <div className="col-sm-12">
         <ul className="list-unstyled">{listWarningMessages}</ul>
       </div>
     </div>
@@ -38,7 +38,7 @@ const Warnings = ({element, showTitle = false, shouldShowURI = true}) => {
 }
 
 
-  Warnings.propTypes = {
+Warnings.propTypes = {
     element: PropTypes.object.isRequired,
     showTitle: PropTypes.bool.isRequired,
     shouldShowURI: PropTypes.bool,

From b0bcaf700d43ed60f48fdddcaae54a4f1c5e9c94 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 14 Jun 2024 17:42:46 +0200
Subject: [PATCH 177/205] feat(import): update ordering of uploaded elements
 and refactor

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py            | 64 +++++++++++++++++---------------
 rdmo/core/xml.py                | 66 ++++++++++++++++++++++++++-------
 rdmo/management/import_utils.py | 25 -------------
 rdmo/management/imports.py      | 62 +++++++++++++++++++++++--------
 4 files changed, 133 insertions(+), 84 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index b966e2e197..39816a28bc 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -29,6 +29,7 @@ class ImportElementFields(str, Enum):
     ERRORS = "errors"
     UPDATED = "updated"
     CREATED = "created"
+    CHANGED_FIELDS = "changedFields"  # for ignored_keys when ordering at save
 
 
 def handle_uploaded_file(filedata):
@@ -102,7 +103,9 @@ def _append_error(element: dict, element_field: str, error: str):
     element[ImportElementFields.DIFF][element_field][ImportElementFields.ERRORS].append(error)
 
 
-def track_messages_on_element(element: dict, element_field: str, warning: Optional[str] = None,
+def track_messages_on_element(element: dict,
+                              element_field: str,
+                              warning: Optional[str] = None,
                               error: Optional[str] = None):
     if warning is not None:
         _initialize_tracking_field(element, element_field)
@@ -330,24 +333,26 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
     target_model = model_info.forward_relations[field_name].related_model
     through_instances = list(getattr(instance, through_name).all())
 
-    _track_changes = {}
-    _track_changes[ImportElementFields.NEW] = []
-    _track_changes[ImportElementFields.CURRENT] = []
+    new_data = []
+    current_data = []
 
+    # get the original data in correct order
     if original is not None:
         try:
-            for _order, orig_field_instance in enumerate(getattr(original, through_name).order_by()):
-                _track_changes[ImportElementFields.CURRENT].append({
+            for orig_field_instance in getattr(original, through_name).order_by():
+                current_data.append({
                     'uri': getattr(orig_field_instance, target_name).uri,
                     'order': orig_field_instance.order,
                     'model': get_rdmo_model_path(target_name, field_name)
                 })
+            current_data = sorted(current_data, key=lambda k: k['order'])
         except AttributeError:
             pass  # legacy elements miss the field_name
 
     for target_element in target_elements:
         target_uri = target_element.get('uri')
         order = target_element.get('order')
+        new_data.append(target_element)
 
         try:
             target_instance = target_model.objects.get(uri=target_uri)
@@ -364,8 +369,6 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
                 if save:
                     # remove the through_instance from the through_instances list so that it won't get removed
                     through_instances.remove(through_instance)
-                if original is not None:
-                    _track_changes[ImportElementFields.NEW].append(target_element)
             except StopIteration:
                 # create a new item
                 if save:
@@ -374,8 +377,6 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
                         target_name: target_instance,
                         'order': order
                     }).save()
-                if original is not None:
-                    _track_changes[ImportElementFields.NEW].append(target_element)
 
         except target_model.DoesNotExist:
             message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} does not exist.'.format(
@@ -386,6 +387,7 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
             )
             logger.info(message)
             element[ImportElementFields.WARNINGS][target_uri].append(message)
+            element[ImportElementFields.WARNINGS][element.get('uri')].append(message)
             track_messages_on_element(element, field_name, warning=message)
         except target_model.MultipleObjectsReturned:
             message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} returns multiple objects.'.format(  # noqa: E501
@@ -396,6 +398,7 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
             )
             logger.info(message)
             element[ImportElementFields.WARNINGS][target_uri].append(message)
+            element[ImportElementFields.WARNINGS][element.get('uri')].append(message)
             track_messages_on_element(element, field_name, warning=message)
     if save:
         # remove the remainders of the items list
@@ -403,17 +406,17 @@ def set_m2m_through_instances(instance, element, field_name=None, source_name=No
             if getattr(through_instance, target_name).uri_prefix == instance.uri_prefix:
                 through_instance.delete()
     # sort the tracked changes by order in-place
-    new_instance_data = sorted(_track_changes[ImportElementFields.NEW], key=lambda k: k['order'])
-    original_instance_data = sorted(_track_changes[ImportElementFields.CURRENT], key=lambda k: k['order'])
-    track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data)
+    new_data = sorted(new_data, key=lambda k: k['order'])
 
+    track_changes_on_m2m_through_instances(element, field_name, current_data, new_data)
 
-def track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data):
+
+def track_changes_on_m2m_through_instances(element, field_name, current_data, new_data):
     _initialize_track_changes_element_field(element, field_name)
-    element[ImportElementFields.DIFF][field_name][ImportElementFields.NEW] = new_instance_data
-    element[ImportElementFields.DIFF][field_name][ImportElementFields.CURRENT] = original_instance_data
-    new_values = [i['uri'] for i in new_instance_data]
-    original_values = [i['uri'] for i in original_instance_data]
+    element[ImportElementFields.DIFF][field_name][ImportElementFields.NEW] = new_data
+    element[ImportElementFields.DIFF][field_name][ImportElementFields.CURRENT] = current_data
+    new_values = [i['uri'] for i in new_data]
+    original_values = [i['uri'] for i in current_data]
     track_changes_on_element(element, field_name, new_value=new_values, original_value=original_values)
 
 
@@ -447,6 +450,7 @@ def set_m2m_instances(instance, element, field_name, original=None, save=None):
             )
             logger.info(message)
             element[ImportElementFields.WARNINGS][foreign_uri].append(message)
+            element[ImportElementFields.WARNINGS][element.get('uri')].append(message)
             track_messages_on_element(element, field_name, warning=message)
     if save:
         getattr(instance, field_name).set(foreign_instances)
@@ -472,24 +476,27 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
     through_info = model_meta.get_field_info(through_model)
     target_model = through_info.forward_relations[target_name].related_model
 
-    _track_changes = {
-        ImportElementFields.NEW: [],
-        ImportElementFields.CURRENT: [],
-    }
+    new_data = []
+    current_data = []
+
     if original is not None:
         try:
-            for _order, _through_instance in enumerate(getattr(original, through_name).order_by()):
-                _track_changes[ImportElementFields.CURRENT].append({
+            current_data = []
+            for _through_instance in getattr(original, through_name).order_by():
+                current_data.append({
                     'uri': getattr(_through_instance, source_name).uri,
                     'order': _through_instance.order,
                     'model': get_rdmo_model_path(target_name, field_name),
                 })
+            current_data = sorted(current_data, key=lambda k: k['order'])
         except AttributeError:
             pass  # legacy elements miss the field_name
 
     for target_element in target_elements:
         target_uri = target_element.get('uri')
         order = target_element.get('order')
+        new_data.append(target_element)
+
         try:
             target_instance = target_model.objects.get(uri=target_uri)
             if target_instance.is_locked:
@@ -510,8 +517,6 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
                 })
                 through_instance.order = order
                 through_instance.save()
-            if original is not None:
-                _track_changes[ImportElementFields.NEW].append(target_element)
 
         except target_model.DoesNotExist:
             message = '{target_model} {target_uri} for imported {instance_model} {instance_uri} does not exist.'.format(
@@ -522,11 +527,12 @@ def set_reverse_m2m_through_instance(instance, element, field_name=None, source_
             )
             logger.info(message)
             element[ImportElementFields.WARNINGS][target_uri].append(message)
+            element[ImportElementFields.WARNINGS][element.get('uri')].append(message)
             track_messages_on_element(element, field_name, warning=message)
     # sort the tracked changes by order in-place
-    new_instance_data = sorted(_track_changes[ImportElementFields.NEW], key=lambda k: k['order'])
-    original_instance_data = sorted(_track_changes[ImportElementFields.CURRENT], key=lambda k: k['order'])
-    track_changes_on_m2m_through_instances(element, field_name, original_instance_data, new_instance_data)
+    new_data = sorted(new_data, key=lambda k: k['order'])
+
+    track_changes_on_m2m_through_instances(element, field_name, current_data, new_data)
 
 
 def format_message_from_validation_error(exception: ValidationError) -> str:
diff --git a/rdmo/core/xml.py b/rdmo/core/xml.py
index 63328c85de..ee0cd0d67b 100644
--- a/rdmo/core/xml.py
+++ b/rdmo/core/xml.py
@@ -12,6 +12,7 @@
 
 from rdmo import __version__ as RDMO_INSTANCE_VERSION
 from rdmo.core.constants import RDMO_MODELS
+from rdmo.core.imports import ImportElementFields
 
 logger = logging.getLogger(__name__)
 
@@ -125,11 +126,11 @@ def parse_xml_to_elements(xml_file=None) -> Tuple[OrderedDict, list]:
     elements = convert_elements(elements, parse(root.attrib.get('version', DEFAULT_RDMO_XML_VERSION)))
 
     # step 5: order the elements and return
-    ordered_elements = order_elements(elements)
+    # ordering of elements is done in the import_elements function
 
     logger.info(f'XML parsing of {file.name} success (length: {len(elements)}).')
 
-    return ordered_elements, errors
+    return elements, errors
 
 
 def read_xml_file(file_name, raise_exception=False):
@@ -355,35 +356,72 @@ def convert_additional_input(elements):
 
     return elements
 
+def get_related_elements(element, ignored_keys=None):
+    ignored_keys = ignored_keys or list(ImportElementFields)
+    related_elements = {k: val for k, val in element.items() if
+                       k not in ignored_keys and (isinstance(val, (dict, list)))}
+    return related_elements
 
-def order_elements(elements):
+
+def sort_by_relatives(elements, descendants_first=False, ancestors_first=False):
+    ancestors, descendants = [], []
+
+    if not descendants_first and not ancestors_first:
+        return elements
+
+    for uri, element in elements.items():
+        try:
+            has_descendants = get_related_elements(element)
+        except AttributeError:
+            has_descendants = False
+        if has_descendants:
+            ancestors.append((uri, element))
+        else:
+            descendants.append((uri, element))
+    if descendants_first:
+        sort_list = descendants + ancestors
+    elif ancestors_first:
+        sort_list = ancestors + descendants
+
+    sorted_elements = OrderedDict()
+    for uri,element in sort_list:
+        sorted_elements[uri] = element
+    return sorted_elements
+
+
+def order_elements(elements, order_sets_first=False, descendants_first=False) -> OrderedDict:
     ordered_elements = OrderedDict()
+    if descendants_first:
+        elements = sort_by_relatives(elements, descendants_first=descendants_first)
     for uri, element in elements.items():
-        append_element(ordered_elements, elements, uri, element)
+        append_element(ordered_elements, elements, uri, element, order_sets_first=order_sets_first)
     return ordered_elements
 
 
-def append_element(ordered_elements, unordered_elements, uri, element):
+def append_element(ordered_elements, unordered_elements, uri, element, order_sets_first=False) -> None:
     if element is None:
         return
 
-    has_list_or_dict = any(i for i in element.values() if isinstance(i, dict) or isinstance(i, list))
-    if has_list_or_dict and uri not in ordered_elements:
-        ordered_elements[uri] = element
+    related_elements = get_related_elements(element)
+
+    if order_sets_first:
+        if related_elements and uri not in ordered_elements:
+            ordered_elements[uri] = element
 
-    for element_value in element.values():
+    for key, element_value in related_elements.items():
         if isinstance(element_value, dict):
             sub_uri = element_value.get('uri')
             sub_element = unordered_elements.get(sub_uri)
-            if sub_uri not in ordered_elements:
+            if sub_uri not in ordered_elements and sub_uri is not None:
                 append_element(ordered_elements, unordered_elements, sub_uri, sub_element)
 
         elif isinstance(element_value, list):
             for value in element_value:
-                sub_uri = value.get('uri')
-                sub_element = unordered_elements.get(sub_uri)
-                if sub_uri not in ordered_elements:
-                    append_element(ordered_elements, unordered_elements, sub_uri, sub_element)
+                if isinstance(element_value, dict):
+                    sub_uri = value.get('uri')
+                    sub_element = unordered_elements.get(sub_uri)
+                    if sub_uri not in ordered_elements and sub_uri is not None:
+                        append_element(ordered_elements, unordered_elements, sub_uri, sub_element)
 
     if uri not in ordered_elements:
         ordered_elements[uri] = element
diff --git a/rdmo/management/import_utils.py b/rdmo/management/import_utils.py
index 4511b7824d..bb792361af 100644
--- a/rdmo/management/import_utils.py
+++ b/rdmo/management/import_utils.py
@@ -2,7 +2,6 @@
 from dataclasses import asdict
 from typing import Dict
 
-from rdmo.conditions.imports import import_helper_condition
 from rdmo.core.imports import (
     ImportElementFields,
     set_common_fields,
@@ -13,31 +12,7 @@
     set_m2m_through_instances,
     set_reverse_m2m_through_instance,
 )
-from rdmo.domain.imports import import_helper_attribute
-from rdmo.options.imports import import_helper_option, import_helper_optionset
-from rdmo.questions.imports import (
-    import_helper_catalog,
-    import_helper_page,
-    import_helper_question,
-    import_helper_questionset,
-    import_helper_section,
-)
-from rdmo.tasks.imports import import_helper_task
-from rdmo.views.imports import import_helper_view
 
-ELEMENT_IMPORT_HELPERS = {
-    "conditions.condition": import_helper_condition,
-    "domain.attribute": import_helper_attribute,
-    "options.optionset": import_helper_optionset,
-    "options.option": import_helper_option,
-    "questions.catalog": import_helper_catalog,
-    "questions.section": import_helper_section,
-    "questions.page": import_helper_page,
-    "questions.questionset": import_helper_questionset,
-    "questions.question": import_helper_question,
-    "tasks.task": import_helper_task,
-    "views.view": import_helper_view
-}
 IMPORT_ELEMENT_INIT_DICT = {
         ImportElementFields.WARNINGS: lambda: defaultdict(list),
         ImportElementFields.ERRORS: list,
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index c517cef14c..c0305cc57b 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -6,28 +6,59 @@
 from django.contrib.sites.shortcuts import get_current_site
 from django.http import HttpRequest
 
+from rdmo.conditions.imports import import_helper_condition
 from rdmo.core.imports import (
     check_permissions,
     get_or_return_instance,
     make_import_info_msg,
     validate_instance,
 )
+from rdmo.core.xml import order_elements
+from rdmo.domain.imports import import_helper_attribute
 from rdmo.management.import_utils import (
-    ELEMENT_IMPORT_HELPERS,
     add_current_site_to_sites_and_editor,
     apply_field_values,
     initialize_import_element_dict,
     strip_uri_prefix_endswith_slash,
     update_related_fields,
 )
+from rdmo.options.imports import import_helper_option, import_helper_optionset
+from rdmo.questions.imports import (
+    import_helper_catalog,
+    import_helper_page,
+    import_helper_question,
+    import_helper_questionset,
+    import_helper_section,
+)
+from rdmo.tasks.imports import import_helper_task
+from rdmo.views.imports import import_helper_view
 
 logger = logging.getLogger(__name__)
 
+ELEMENT_IMPORT_HELPERS = {
+    "conditions.condition": import_helper_condition,
+    "domain.attribute": import_helper_attribute,
+    "options.optionset": import_helper_optionset,
+    "options.option": import_helper_option,
+    "questions.catalog": import_helper_catalog,
+    "questions.section": import_helper_section,
+    "questions.page": import_helper_page,
+    "questions.questionset": import_helper_questionset,
+    "questions.question": import_helper_question,
+    "tasks.task": import_helper_task,
+    "views.view": import_helper_view
+}
+
 
 def import_elements(uploaded_elements: Dict, save: bool = True, request: Optional[HttpRequest] = None) -> List[Dict]:
     imported_elements = []
+    uploaded_elements_ordering_index = {uri: n for n, uri in enumerate(uploaded_elements.keys())}
     uploaded_uris = set(uploaded_elements.keys())
     current_site = get_current_site(request)
+    if save:
+        # when saving, the descendant elements go first
+        uploaded_elements = order_elements(uploaded_elements, descendants_first=True)
+
     for _uri, uploaded_element in uploaded_elements.items():
         element = import_element(element=uploaded_element,
                                  save=save,
@@ -36,6 +67,11 @@ def import_elements(uploaded_elements: Dict, save: bool = True, request: Optiona
                                  current_site=current_site)
         element['warnings'] = {k: val for k, val in element['warnings'].items() if k not in uploaded_uris}
         imported_elements.append(element)
+
+    # sort elements back to order of uploaded elements
+    imported_elements = sorted(imported_elements,
+                               key=lambda x: uploaded_elements_ordering_index.get(x['uri'], float('inf')))
+
     return imported_elements
 
 
@@ -47,25 +83,18 @@ def import_element(
         current_site = None
     ) -> Dict:
 
-    if element is None:
+    if element is None or not isinstance(element, dict):
+        return {}
+    if 'model' not in element:
         return {}
-
-    model_path = element.get('model')
-    if model_path is None:
-        return element
 
     initialize_import_element_dict(element)
 
-    user = request.user if request is not None else None
-    import_helper = ELEMENT_IMPORT_HELPERS[model_path]
-    if import_helper.model_path != model_path:
-        raise ValueError(f'Invalid import helper model path: {import_helper.model_path}. Expected {model_path}.')
-    model = import_helper.model
-    validators = import_helper.validators
+    import_helper = ELEMENT_IMPORT_HELPERS[element['model']]
 
     uri = element.get('uri')
-    # get or create instance from uri and model_path
-    instance, created = get_or_return_instance(model, uri=uri)
+    # get or create instance from uri and model
+    instance, created = get_or_return_instance(import_helper.model, uri=uri)
 
     # keep a copy of the original
     # when the element is updated
@@ -73,9 +102,10 @@ def import_element(
     original = copy.deepcopy(instance) if not created else None
 
     # prepare a log message
-    msg = make_import_info_msg(model._meta.verbose_name, created, uri=uri)
+    msg = make_import_info_msg(import_helper.model._meta.verbose_name, created, uri=uri)
 
     # check the change or add permissions for the user on the instance
+    user = request.user if request is not None else None
     perms_error_msg = check_permissions(instance, uri, user)
     if perms_error_msg:
         # when there is an error msg, the import can be stopped and return
@@ -92,7 +122,7 @@ def import_element(
     apply_field_values(instance, element, import_helper, uploaded_uris, original)
 
     # call the validators on the instance
-    validate_instance(instance, element, *validators)
+    validate_instance(instance, element, *import_helper.validators)
 
     if element.get('errors'):
         # when there is an error msg, the import can be stopped and return

From 45e5b4501335c3db0d4c1d5a24be30438e2cb7c9 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 14 Jun 2024 17:45:18 +0200
Subject: [PATCH 178/205] refactor(import): update import helper args

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/conditions/imports.py |  1 -
 rdmo/domain/imports.py     |  1 -
 rdmo/options/imports.py    | 22 ++++++++++------------
 rdmo/questions/imports.py  |  5 -----
 rdmo/tasks/imports.py      |  1 -
 rdmo/views/imports.py      |  1 -
 6 files changed, 10 insertions(+), 21 deletions(-)

diff --git a/rdmo/conditions/imports.py b/rdmo/conditions/imports.py
index fa12fdfeb1..3a786a1506 100644
--- a/rdmo/conditions/imports.py
+++ b/rdmo/conditions/imports.py
@@ -5,7 +5,6 @@
 
 import_helper_condition = ElementImportHelper(
     model=Condition,
-    model_path="conditions.condition",
     validators=(ConditionLockedValidator, ConditionUniqueURIValidator),
     foreign_fields=('source', 'target_option'),
     extra_fields=(
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index bb8ab1d437..ef9a5daca3 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -21,7 +21,6 @@ def build_attribute_uri(instance: Optional[Attribute]=None):
 
 import_helper_attribute = ElementImportHelper(
     model=Attribute,
-    model_path="domain.attribute",
     common_fields=('uri_prefix', 'key', 'comment'),
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     foreign_fields=('parent',),
diff --git a/rdmo/options/imports.py b/rdmo/options/imports.py
index c94d0bb4f0..e39354f33a 100644
--- a/rdmo/options/imports.py
+++ b/rdmo/options/imports.py
@@ -9,15 +9,14 @@
 )
 
 import_helper_optionset = ElementImportHelper(
-    model = OptionSet,
-    model_path = "options.optionset",
-    validators = (OptionSetLockedValidator, OptionSetUniqueURIValidator),
-    extra_fields = (
+    model=OptionSet,
+    validators=(OptionSetLockedValidator, OptionSetUniqueURIValidator),
+    extra_fields=(
         ExtraFieldHelper(field_name='order'),
         ExtraFieldHelper(field_name='provider_key', value=''),
     ),
-    m2m_instance_fields = ('conditions', ),
-    m2m_through_instance_fields = [
+    m2m_instance_fields=('conditions', ),
+    m2m_through_instance_fields=[
         ThroughInstanceMapper(
             field_name='options',
             source_name='optionset',
@@ -28,14 +27,13 @@
 )
 
 import_helper_option = ElementImportHelper(
-    model = Option,
-    model_path = "options.option",
-    validators = (OptionLockedValidator, OptionUniqueURIValidator),
-    lang_fields = ('text', 'help', 'view_text'),
-    extra_fields = (
+    model=Option,
+    validators=(OptionLockedValidator, OptionUniqueURIValidator),
+    lang_fields=('text', 'help', 'view_text'),
+    extra_fields=(
         ExtraFieldHelper(field_name='additional_input', value=Option.ADDITIONAL_INPUT_NONE),
         ),
-    reverse_m2m_through_instance_fields = [
+    reverse_m2m_through_instance_fields=[
         ThroughInstanceMapper(
             field_name='optionset',
             source_name='option',
diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index c0b1e67306..973a37f598 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -18,7 +18,6 @@
 
 import_helper_catalog = ElementImportHelper(
     model = Catalog,
-    model_path="questions.catalog",
     validators=(CatalogLockedValidator, CatalogUniqueURIValidator),
     lang_fields=('help', 'title'),
     extra_fields = (
@@ -36,7 +35,6 @@
 
 import_helper_section = ElementImportHelper(
     model = Section,
-    model_path="questions.section",
     validators=(SectionLockedValidator, SectionUniqueURIValidator),
     lang_fields=('title',),
     m2m_through_instance_fields=[
@@ -55,7 +53,6 @@
 
 import_helper_page = ElementImportHelper(
     model = Page,
-    model_path="questions.page",
     validators=(PageLockedValidator, PageUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
@@ -83,7 +80,6 @@
 
 import_helper_question = ElementImportHelper(
     model=Question,
-    model_path="questions.question",
     validators=(QuestionLockedValidator, QuestionUniqueURIValidator),
     lang_fields=('text', 'help', 'default_text', 'verbose_name'),
     foreign_fields=('attribute', 'default_option'),
@@ -113,7 +109,6 @@
 )
 import_helper_questionset = ElementImportHelper(
     model = QuestionSet,
-    model_path="questions.questionset",
     validators=(QuestionSetLockedValidator, QuestionSetUniqueURIValidator),
     lang_fields=('help', 'title', 'verbose_name'),
     foreign_fields=('attribute',),
diff --git a/rdmo/tasks/imports.py b/rdmo/tasks/imports.py
index 2f7099b1e8..df5523da8b 100644
--- a/rdmo/tasks/imports.py
+++ b/rdmo/tasks/imports.py
@@ -5,7 +5,6 @@
 
 import_helper_task = ElementImportHelper(
     model=Task,
-    model_path="tasks.task",
     validators=(TaskLockedValidator, TaskUniqueURIValidator),
     lang_fields=('title', 'text'),
     foreign_fields=('start_attribute', 'end_attribute'),
diff --git a/rdmo/views/imports.py b/rdmo/views/imports.py
index bc8db9487f..89e6be4693 100644
--- a/rdmo/views/imports.py
+++ b/rdmo/views/imports.py
@@ -5,7 +5,6 @@
 
 import_helper_view = ElementImportHelper(
     model=View,
-    model_path="views.view",
     validators=(ViewLockedValidator, ViewUniqueURIValidator),
     lang_fields=('help', 'title'),
     extra_fields=(

From 9a357b8ef3e86e5ee0493082d5f1d69370f110a3 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 14 Jun 2024 17:46:54 +0200
Subject: [PATCH 179/205] tests(import): update and fix import tests

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/conftest.py             |   1 -
 .../tests/helpers_import_elements.py          |  24 +++--
 .../tests/test_frontend_import_options.py     |   3 +-
 .../tests/test_frontend_import_questions.py   |   7 +-
 .../test_frontend_management_elements.py      |   1 -
 .../tests/test_import_conditions.py           |  14 +--
 rdmo/management/tests/test_import_domain.py   |  20 ++--
 rdmo/management/tests/test_import_options.py  | 102 ++++++++++--------
 .../management/tests/test_import_questions.py |  64 ++++-------
 rdmo/management/tests/test_import_tasks.py    |  15 ++-
 rdmo/management/tests/test_import_views.py    |  15 ++-
 11 files changed, 130 insertions(+), 136 deletions(-)

diff --git a/rdmo/management/tests/conftest.py b/rdmo/management/tests/conftest.py
index 5808c00714..165588bcd4 100644
--- a/rdmo/management/tests/conftest.py
+++ b/rdmo/management/tests/conftest.py
@@ -44,7 +44,6 @@ def logout_user(page: Page):
 @pytest.fixture(scope="function")
 def logged_in_user(e2e_tests_django_db_setup, base_url_page, username:str, password: str) -> Page:
     """Log in as admin user through Django login UI, returns logged in page for e2e tests."""
-    # breakpoint()
     page = login_user(base_url_page, username, password)
     yield page
     logout_user(page)
diff --git a/rdmo/management/tests/helpers_import_elements.py b/rdmo/management/tests/helpers_import_elements.py
index 6ad4d8caa1..a1102c14db 100644
--- a/rdmo/management/tests/helpers_import_elements.py
+++ b/rdmo/management/tests/helpers_import_elements.py
@@ -4,6 +4,11 @@
 
 from rdmo.core.imports import ImportElementFields, track_changes_on_element
 from rdmo.management.import_utils import initialize_import_element_dict
+from rdmo.management.imports import import_elements
+from rdmo.management.tests.helpers_xml import read_xml_and_parse_to_root_and_elements
+
+IMPORT_ELEMENT_PANELS_LOCATOR = ".list-group > .list-group-item > .checkbox"
+IMPORT_ELEMENT_PANELS_LOCATOR_SHOWN = ".list-group > .list-group-item > .row"
 
 UPDATE_FIELD_FUNCS = {
     'comment': lambda text: f"this is a test comment {text}",
@@ -31,14 +36,13 @@ def get_changed_elements(elements: List[Dict]) -> Dict[str, Dict[str,Union[bool,
     for element in elements:
 
         changed_fields = []
-        for key, diff_field in element[ImportElementFields.DIFF].items():
-            if diff_field[ImportElementFields.NEW] != diff_field[ImportElementFields.CURRENT]:
-                changed_fields += key
+        for field, diff_field in element[ImportElementFields.DIFF].items():
+            if not (diff_field[ImportElementFields.NEW] == diff_field[ImportElementFields.CURRENT]):
+                changed_fields.append(field)
         if changed_fields:
-            changed_elements[element['uri']] = {
-                'changed': bool(changed_fields),
-                'changed_fields': changed_fields,
-            }
+            changed_elements[element['uri']] = {}
+            changed_elements[element['uri']]['changed'] = bool(changed_fields)
+            changed_elements[element['uri']]['changed_fields'] = changed_fields
     return changed_elements
 
 
@@ -66,3 +70,9 @@ def _test_helper_change_fields_elements(elements,
                 _element[field] = new_val
         _new_elements[_uri] = _element
     return _new_elements
+
+
+def parse_xml_and_import_elements(xml_file):
+    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
+    imported_elements = import_elements(elements)
+    return elements, root, imported_elements
diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
index 1907b725d1..ac76aea3a0 100644
--- a/rdmo/management/tests/test_frontend_import_options.py
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -7,6 +7,7 @@
 
 from rdmo.options.models import Option, OptionSet
 
+from .helpers_import_elements import IMPORT_ELEMENT_PANELS_LOCATOR
 from .helpers_models import delete_all_objects
 
 pytestmark = pytest.mark.e2e
@@ -47,7 +48,7 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     page.get_by_role("link", name="Deselect all").click()
     page.get_by_role("link", name="Select all", exact=True).click()
     page.get_by_role("link", name="Show all", exact=True).click()
-    rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")
+    rows_displayed_in_ui = page.locator(IMPORT_ELEMENT_PANELS_LOCATOR)
     expect(rows_displayed_in_ui).to_have_count(OPTIONSETS_COUNTS['total'])
     # click the import button to start saving the instances to the db
     page.get_by_role("button", name=f"Import {OPTIONSETS_COUNTS['total']} elements").click()
diff --git a/rdmo/management/tests/test_frontend_import_questions.py b/rdmo/management/tests/test_frontend_import_questions.py
index 4ff5c2c798..981ea7c33c 100644
--- a/rdmo/management/tests/test_frontend_import_questions.py
+++ b/rdmo/management/tests/test_frontend_import_questions.py
@@ -9,6 +9,7 @@
 from rdmo.questions.models import Page as PageModel
 from rdmo.questions.models.questionset import QuestionSet
 
+from .helpers_import_elements import IMPORT_ELEMENT_PANELS_LOCATOR_SHOWN
 from .helpers_models import delete_all_objects
 
 pytestmark = pytest.mark.e2e
@@ -38,10 +39,10 @@ def test_import_catalogs_in_management(logged_in_user: Page) -> None:
     page.get_by_role("link", name="Deselect all").click()
     page.get_by_role("link", name="Select all", exact=True).click()
     page.get_by_role("link", name="Show all").click()
-    rows_displayed_in_ui = page.locator(".list-group > .list-group-item > .row.mt-10")
-    expect(rows_displayed_in_ui).to_have_count(148)
+    rows_displayed_in_ui_show = page.locator(IMPORT_ELEMENT_PANELS_LOCATOR_SHOWN).get_by_text("URI prefix", exact=True)
+    expect(rows_displayed_in_ui_show).to_have_count(148)
     page.get_by_role("link", name="Hide all").click()
-    expect(rows_displayed_in_ui).to_have_count(0)
+    expect(rows_displayed_in_ui_show).to_have_count(0)
     page.screenshot(path="screenshots/management-import-catalogs-pre.png", full_page=True)
     # click the import button to start saving the instances to the db
     page.get_by_role("button", name="Import 148 elements").click()
diff --git a/rdmo/management/tests/test_frontend_management_elements.py b/rdmo/management/tests/test_frontend_management_elements.py
index 9eac650be9..738bb2b6ea 100644
--- a/rdmo/management/tests/test_frontend_management_elements.py
+++ b/rdmo/management/tests/test_frontend_management_elements.py
@@ -24,7 +24,6 @@
 @pytest.mark.parametrize("helper", model_helpers)
 def test_management_navigation(logged_in_user: Page, helper: ModelHelper, username: str, password: str) -> None:
     """Test that each content type is available through the navigation."""
-    # breakpoint()
     page = logged_in_user
     expect(page.get_by_role("heading", name="Management")).to_be_visible()
 
diff --git a/rdmo/management/tests/test_import_conditions.py b/rdmo/management/tests/test_import_conditions.py
index bcebe7f4bd..01a52d99b5 100644
--- a/rdmo/management/tests/test_import_conditions.py
+++ b/rdmo/management/tests/test_import_conditions.py
@@ -9,6 +9,7 @@
 from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
+    parse_xml_and_import_elements,
 )
 from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
@@ -19,8 +20,7 @@ def test_create_conditions(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == Condition.objects.count() == 15
     assert all(element['created'] is True for element in imported_elements)
@@ -30,8 +30,7 @@ def test_create_conditions(db, settings):
 def test_update_conditions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 15
     assert all(element['created'] is False for element in imported_elements)
@@ -43,7 +42,6 @@ def test_update_conditions_with_changed_fields(db, settings, updated_fields):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'conditions.xml'
 
     elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    # breakpoint()
     elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=7)
     changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
@@ -61,8 +59,7 @@ def test_create_legacy_conditions(db, settings):
     Condition.objects.all().delete()
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'conditions.xml'
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == Condition.objects.count() == 15
     assert all(element['created'] is True for element in imported_elements)
@@ -72,8 +69,7 @@ def test_create_legacy_conditions(db, settings):
 def test_update_legacy_conditions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'conditions.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 15
     assert all(element['created'] is False for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_domain.py b/rdmo/management/tests/test_import_domain.py
index 3b3ac88595..0386691313 100644
--- a/rdmo/management/tests/test_import_domain.py
+++ b/rdmo/management/tests/test_import_domain.py
@@ -6,7 +6,11 @@
 from rdmo.domain.models import Attribute
 from rdmo.management.imports import import_elements
 
-from .helpers_import_elements import _test_helper_change_fields_elements, _test_helper_filter_updated_and_changed
+from .helpers_import_elements import (
+    _test_helper_change_fields_elements,
+    _test_helper_filter_updated_and_changed,
+    parse_xml_and_import_elements,
+)
 from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
 fields_to_be_changed = (('comment',),)
@@ -16,8 +20,7 @@ def test_create_domain(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == Attribute.objects.count() == 86
     assert all(element['created'] is True for element in imported_elements)
@@ -27,8 +30,7 @@ def test_create_domain(db, settings):
 def test_update_domain(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'attributes.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements)
     assert all(element['created'] is False for element in imported_elements)
@@ -54,7 +56,7 @@ def test_update_attributes_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
+       assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_legacy_domain(db, settings):
@@ -62,8 +64,7 @@ def test_create_legacy_domain(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'domain.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 86
     assert Attribute.objects.count() == 86
@@ -74,8 +75,7 @@ def test_create_legacy_domain(db, settings):
 def test_update_legacy_domain(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'domain.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 86
     assert all(element['created'] is False for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 4e66dde484..6320689af1 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -10,6 +10,7 @@
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
     get_changed_elements,
+    parse_xml_and_import_elements,
 )
 from .helpers_models import delete_all_objects
 from .helpers_xml import read_xml_and_parse_to_root_and_elements
@@ -27,29 +28,36 @@
         },
     }
 
+
 def test_create_optionsets(db, settings):
     delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(elements) == len(imported_elements) == 13
     assert OptionSet.objects.count() == 4
     assert Option.objects.count() == 9
-    assert all(element['created'] is True for element in imported_elements)
-    assert all(element['updated'] is False for element in imported_elements)
+    assert all(element[ImportElementFields.CREATED] is True for element in imported_elements)
+    assert all(element[ImportElementFields.UPDATED] is False for element in imported_elements)
 
 
 def test_update_optionsets(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
+    # Arrange, import the optionsets.xml
+    delete_all_objects([OptionSet, Option])
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
+    assert OptionSet.objects.count() == 4
+    assert Option.objects.count() == 9
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    # Act, import the optionsets.xml again
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(elements) == len(imported_elements) == 13
-    assert all(element['created'] is False for element in imported_elements)
-    assert all(element['updated'] is True for element in imported_elements)
+    assert all(element[ImportElementFields.CREATED] is False for element in imported_elements)
+    assert all(element[ImportElementFields.UPDATED] is True for element in imported_elements)
+    assert OptionSet.objects.count() == 4
+    assert Option.objects.count() == 9
 
 
 @pytest.mark.parametrize('updated_fields', fields_to_be_changed)
@@ -57,9 +65,9 @@ def test_update_optionsets_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
     assert len(root) == len(imported_elements) == 13
+    assert OptionSet.objects.count() + Option.objects.count() == 13
     # start test with fresh options in db
     _n_change = int(Option.objects.count() / 2)
     elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=7)
@@ -67,8 +75,8 @@ def test_update_optionsets_with_changed_fields(db, settings, updated_fields):
     imported_elements = import_elements(elements)
     assert len(root) == len(imported_elements) == 13
     imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
-    assert all(element['created'] is False for element in imported_elements)
-    assert all(element['updated'] is True for element in imported_elements)
+    assert all(element[ImportElementFields.CREATED] is False for element in imported_elements)
+    assert all(element[ImportElementFields.UPDATED] is True for element in imported_elements)
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
@@ -77,30 +85,30 @@ def test_update_optionsets_with_changed_fields(db, settings, updated_fields):
 
 def test_update_optionsets_from_changed_xml(db, settings):
     # Arrange, start test with fresh options in db
+    # Arrange, import the optionsets.xml
     delete_all_objects([OptionSet, Option])
-
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'optionsets.xml'
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
-    assert len(root) == len(imported_elements) == 13
-    # Act, import from xml that has changes
+    parse_xml_and_import_elements(xml_file)
+    assert OptionSet.objects.count() + Option.objects.count() == 13
+    # Act, import from xml optionsets-1.xml that contains changes
     xml_file_1 = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'updated-and-changed' / 'optionsets-1.xml'
     elements_1, root_1 = read_xml_and_parse_to_root_and_elements(xml_file_1)
     imported_elements_1 = import_elements(elements_1, save=False)
     assert imported_elements_1
     assert [i for i in imported_elements_1 if i[ImportElementFields.DIFF]]
-    warnings_elements = [i for i in imported_elements_1 if i['warnings']]
+    warnings_elements = [i for i in imported_elements_1 if i[ImportElementFields.WARNINGS]]
     assert len(warnings_elements) == 1
 
     changed_elements = get_changed_elements(imported_elements_1)
 
     assert test_optionset['original']['uri'] in changed_elements
-    assert len([i for i in  changed_elements.values() if i]) == 5
+    assert len([i for i in changed_elements.values() if i]) == 5
 
     # change the order of the options, as in the xml
     optionset_element = next(filter(lambda x: x['uri'] == test_optionset['original']['uri'], imported_elements_1))
     # the test changes are simply the reversed order of the options
     test_optionset_changed_options = test_optionset['original']['options'][::-1]
+
     assert optionset_element
     assert "options" in optionset_element[ImportElementFields.DIFF]
     assert optionset_element[ImportElementFields.DIFF]['options'][ImportElementFields.CURRENT] == test_optionset['original']['options']  # noqa: E501
@@ -118,31 +126,34 @@ def test_update_optionsets_from_changed_xml(db, settings):
     imported_elements_2 = import_elements(elements_1, save=False)
     changed_elements_2 = get_changed_elements(imported_elements_2)
     assert len(changed_elements_2) == 0
-    assert len([i for i in imported_elements_2 if i['warnings']]) == 1
+    assert len([i for i in imported_elements_2 if i[ImportElementFields.WARNINGS]]) == 1
 
 
 def test_create_options(db, settings):
+    # Arrange
     Option.objects.all().delete()
-
+    # Act
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
-
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(elements) == len(imported_elements) == Option.objects.count() == 9
-    assert all(element['created'] is True for element in imported_elements)
-    assert all(element['updated'] is False for element in imported_elements)
+    assert all(element[ImportElementFields.CREATED] is True for element in imported_elements)
+    assert all(element[ImportElementFields.UPDATED] is False for element in imported_elements)
 
 
 def test_update_options(db, settings):
+    # Arrange
+    Option.objects.all().delete()
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
+    parse_xml_and_import_elements(xml_file)
+    assert Option.objects.count() == 9
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    # Act
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
-    assert len(root) == len(elements) == len(imported_elements) == 9
-    assert all(element['created'] is False for element in imported_elements)
-    assert all(element['updated'] is True for element in imported_elements)
+    assert len(root) == len(elements) == len(imported_elements) == Option.objects.count() == 9
+    assert all(element[ImportElementFields.CREATED] is False for element in imported_elements)
+    assert all(element[ImportElementFields.UPDATED] is True for element in imported_elements)
 
 
 @pytest.mark.parametrize('updated_fields', fields_to_be_changed)
@@ -150,8 +161,7 @@ def test_update_options_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'options.xml'
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
     assert len(root) == len(imported_elements) == 9
     # start test with fresh options in db
     elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=4)
@@ -159,37 +169,39 @@ def test_update_options_with_changed_fields(db, settings, updated_fields):
     imported_elements = import_elements(elements)
     imported_and_changed = _test_helper_filter_updated_and_changed(imported_elements, updated_fields=updated_fields)
     assert len(root) == len(imported_elements) == 9
-    assert all(element['created'] is False for element in imported_elements)
-    assert all(element['updated'] is True for element in imported_elements)
+    assert all(element[ImportElementFields.CREATED] is False for element in imported_elements)
+    assert all(element[ImportElementFields.UPDATED] is True for element in imported_elements)
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
         assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
-
-
 def test_create_legacy_options(db, settings):
     delete_all_objects([OptionSet, Option])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'options.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(elements) == len(imported_elements) == 12
     assert OptionSet.objects.count() == 4
     assert Option.objects.count() == 8
-    assert all(element['created'] is True for element in imported_elements)
-    assert all(element['updated'] is False for element in imported_elements)
+    assert all(element[ImportElementFields.CREATED] is True for element in imported_elements)
+    assert all(element[ImportElementFields.UPDATED] is False for element in imported_elements)
 
 
 def test_update_legacy_options(db, settings):
+    delete_all_objects([OptionSet, Option])
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'options.xml'
+    parse_xml_and_import_elements(xml_file)
+    assert OptionSet.objects.count() == 4
+    assert Option.objects.count() == 8
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(elements) == len(imported_elements) == 12
-    assert all(element['created'] is False for element in imported_elements)
-    assert all(element['updated'] is True for element in imported_elements)
+    assert OptionSet.objects.count() == 4
+    assert Option.objects.count() == 8
+    assert all(element[ImportElementFields.CREATED] is False for element in imported_elements)
+    assert all(element[ImportElementFields.UPDATED] is True for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py
index f18e1995c5..0f2083ad39 100644
--- a/rdmo/management/tests/test_import_questions.py
+++ b/rdmo/management/tests/test_import_questions.py
@@ -9,9 +9,9 @@
 from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
+    parse_xml_and_import_elements,
 )
 from .helpers_models import delete_all_objects
-from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
 fields_to_be_changed = (('comment',),)
 
@@ -21,8 +21,7 @@ def test_create_catalogs(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 148
     assert Catalog.objects.count() == 2
@@ -37,8 +36,7 @@ def test_create_catalogs(db, settings):
 def test_update_catalogs(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 148
     assert all(element['created'] is False for element in imported_elements)
@@ -50,11 +48,9 @@ def test_update_catalogs_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'catalogs.xml'
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
     assert len(root) == len(imported_elements) == 148
     # start test with fresh elements in db
-    # breakpoint()
     elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=75)
     changed_elements = _test_helper_filter_updated_and_changed(elements.values(), updated_fields=updated_fields)
     imported_elements = import_elements(elements)
@@ -65,7 +61,7 @@ def test_update_catalogs_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
+       assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_sections(db, settings):
@@ -73,8 +69,7 @@ def test_create_sections(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 146
     assert Section.objects.count() == 6
@@ -88,8 +83,7 @@ def test_create_sections(db, settings):
 def test_update_sections(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 146
     assert all(element['created'] is False for element in imported_elements)
@@ -101,8 +95,7 @@ def test_update_sections_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'sections.xml'
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
     assert len(root) == len(imported_elements) == 146
     # start test with fresh elements in db
 
@@ -115,7 +108,7 @@ def test_update_sections_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
+       assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_pages(db, settings):
@@ -123,8 +116,7 @@ def test_create_pages(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 140
     assert Page.objects.count() == 48
@@ -137,8 +129,7 @@ def test_create_pages(db, settings):
 def test_update_pages(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 140
     assert all(element['created'] is False for element in imported_elements)
@@ -150,8 +141,7 @@ def test_update_pages_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'pages.xml'
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
     assert len(root) == len(imported_elements) == 140
     # start test with fresh elements in db
     elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=75)
@@ -164,7 +154,7 @@ def test_update_pages_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
+       assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_questionsets(db, settings):
@@ -172,8 +162,7 @@ def test_create_questionsets(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == 10  # two questionsets appear twice in the export file
     assert len(imported_elements) == 8
@@ -186,8 +175,7 @@ def test_create_questionsets(db, settings):
 def test_update_questionsets(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == 10  # two questionsets appear twice in the export file
     assert len(imported_elements) == 8
@@ -200,8 +188,7 @@ def test_update_questionsets_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questionsets.xml'
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
     assert len(root) == 10  # two questionsets appear twice in the export file
     assert len(imported_elements) == 8
     # start test with fresh elements in db
@@ -215,7 +202,7 @@ def test_update_questionsets_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
+       assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_questions(db, settings):
@@ -223,8 +210,7 @@ def test_create_questions(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 89
     assert Question.objects.count() == 89
@@ -235,8 +221,7 @@ def test_create_questions(db, settings):
 def test_update_questions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 89
     assert all(element['created'] is False for element in imported_elements)
@@ -248,8 +233,7 @@ def test_update_questions_with_changed_fields(db, settings, updated_fields):
     delete_all_objects([Catalog, Section, Page, QuestionSet, Question])
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'questions.xml'
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
     assert len(root) == len(imported_elements) == 89
     # start test with fresh elements in db
     elements = _test_helper_change_fields_elements(elements, fields_to_update=updated_fields, n=45)
@@ -262,7 +246,7 @@ def test_update_questions_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
+       assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_legacy_questions(db, settings):
@@ -270,8 +254,7 @@ def test_create_legacy_questions(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'questions.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 147
     assert Catalog.objects.count() == 1
@@ -292,8 +275,7 @@ def test_create_legacy_questions(db, settings):
 def test_update_legacy_questions(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'questions.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 147
     assert all(element['created'] is False for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_tasks.py b/rdmo/management/tests/test_import_tasks.py
index 0d0bbf834c..bd88542bcb 100644
--- a/rdmo/management/tests/test_import_tasks.py
+++ b/rdmo/management/tests/test_import_tasks.py
@@ -9,6 +9,7 @@
 from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
+    parse_xml_and_import_elements,
 )
 from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
@@ -20,8 +21,7 @@ def test_create_tasks(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'tasks.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == Task.objects.count() == 2
     assert all(element['created'] is True for element in imported_elements)
@@ -31,8 +31,7 @@ def test_create_tasks(db, settings):
 def test_update_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'tasks.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 2
     assert all(element['created'] is False for element in imported_elements)
@@ -54,7 +53,7 @@ def test_update_tasks_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
+       assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_legacy_tasks(db, settings):
@@ -62,8 +61,7 @@ def test_create_legacy_tasks(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'tasks.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == Task.objects.count() == 2
     assert all(element['created'] is True for element in imported_elements)
@@ -73,8 +71,7 @@ def test_create_legacy_tasks(db, settings):
 def test_update_legacy_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'tasks.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 2
     assert all(element['created'] is False for element in imported_elements)
diff --git a/rdmo/management/tests/test_import_views.py b/rdmo/management/tests/test_import_views.py
index eadb55e5e6..7fe08f1378 100644
--- a/rdmo/management/tests/test_import_views.py
+++ b/rdmo/management/tests/test_import_views.py
@@ -9,6 +9,7 @@
 from .helpers_import_elements import (
     _test_helper_change_fields_elements,
     _test_helper_filter_updated_and_changed,
+    parse_xml_and_import_elements,
 )
 from .helpers_xml import read_xml_and_parse_to_root_and_elements
 
@@ -20,8 +21,7 @@ def test_create_tasks(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'views.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == View.objects.count() == 3
     assert all(element['created'] is True for element in imported_elements)
@@ -31,8 +31,7 @@ def test_create_tasks(db, settings):
 def test_update_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'views.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 3
     assert all(element['created'] is False for element in imported_elements)
@@ -54,7 +53,7 @@ def test_update_views_with_changed_fields(db, settings, updated_fields):
     assert len(imported_and_changed) == len(changed_elements)
     # compare two ordered lists with "updated_and_changed" dicts
     for test, imported in zip(changed_elements, imported_and_changed):
-        assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
+       assert test[ImportElementFields.DIFF] == imported[ImportElementFields.DIFF]
 
 
 def test_create_legacy_tasks(db, settings):
@@ -62,8 +61,7 @@ def test_create_legacy_tasks(db, settings):
 
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'views.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == View.objects.count() == 3
     assert all(element['created'] is True for element in imported_elements)
@@ -73,8 +71,7 @@ def test_create_legacy_tasks(db, settings):
 def test_update_legacy_tasks(db, settings):
     xml_file = Path(settings.BASE_DIR) / 'xml' / 'elements' / 'legacy' / 'views.xml'
 
-    elements, root = read_xml_and_parse_to_root_and_elements(xml_file)
-    imported_elements = import_elements(elements)
+    elements, root, imported_elements = parse_xml_and_import_elements(xml_file)
 
     assert len(root) == len(imported_elements) == 3
     assert all(element['created'] is False for element in imported_elements)

From 41c5a3b5327ac90a10c14bbab68fa3ac62d972cc Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 14 Jun 2024 17:50:08 +0200
Subject: [PATCH 180/205] js(import): update and refactor import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/common/Links.js      | 39 +++---------
 .../js/components/import/ImportElement.js     | 61 ++++++++++++++-----
 .../components/import/ImportSuccessElement.js | 19 ++----
 .../js/components/import/common/FieldRow.js   | 29 +++++----
 .../components/import/common/FieldRowDiffs.js |  2 -
 .../components/import/common/FieldRowValue.js | 16 +----
 .../js/components/import/common/Fields.js     | 40 ++++++++----
 .../assets/js/components/main/Import.js       |  2 +-
 .../assets/js/reducers/importsReducer.js      |  4 +-
 .../assets/js/utils/importFilters.js          |  2 +-
 10 files changed, 110 insertions(+), 104 deletions(-)

diff --git a/rdmo/management/assets/js/components/common/Links.js b/rdmo/management/assets/js/components/common/Links.js
index 1ef7e777ee..1d30db1809 100644
--- a/rdmo/management/assets/js/components/common/Links.js
+++ b/rdmo/management/assets/js/components/common/Links.js
@@ -18,14 +18,15 @@ NestedLink.propTypes = {
   show: PropTypes.bool
 }
 
-const EditLink = ({ href, title, onClick }) => {
-  return <Link href={href} className="element-link fa fa-pencil" title={title} onClick={onClick} />
+const EditLink = ({ href, title, onClick, disabled= false }) => {
+  return <Link href={href} className="element-link fa fa-pencil" title={title} onClick={onClick} disabled={disabled} />
 }
 
 EditLink.propTypes = {
   href: PropTypes.string.isRequired,
   title: PropTypes.string.isRequired,
-  onClick: PropTypes.func.isRequired
+  onClick: PropTypes.func.isRequired,
+  disabled: PropTypes.bool
 }
 
 const CopyLink = ({ href, title, onClick }) => {
@@ -217,44 +218,22 @@ CodeLink.propTypes = {
   order: PropTypes.number
 }
 
-const ErrorLink = ({ show, onClick }) => {
-  return (
-    show &&
-    <Link className="element-link fa fa-warning text-danger" onClick={onClick} />
-  )
+const ErrorLink = ({ onClick }) => {
+  return <Link className="element-link fa fa-warning text-danger" onClick={onClick} />
 }
 
 ErrorLink.propTypes = {
-  show: PropTypes.bool,
   onClick: PropTypes.func.isRequired
 }
 
-const WarningLink = ({ show= false, onClick }) => {
-  return (
-    show &&
-    <Link className="element-link fa fa-warning text-warning" onClick={onClick} />
-  )
+const WarningLink = ({ onClick }) => {
+  return <Link className="element-link fa fa-warning text-warning" onClick={onClick} />
 }
 
 WarningLink.propTypes = {
-  show: PropTypes.bool,
   onClick: PropTypes.func.isRequired
 }
 
-const ShowUpdatedLink = ({ show= false, disabled= false, onClick }) => {
-  return (
-    show &&
-    <Link  className='element-link fa fa-pencil' onClick={onClick} disabled={disabled}/>
-  )
-}
-
-ShowUpdatedLink.propTypes = {
-  show: PropTypes.bool,
-  disabled: PropTypes.bool,
-  onClick: PropTypes.func.isRequired
-}
-
-
 const ShowLink = ({ show = false, onClick }) => {
   const title = show ? gettext('Hide') : gettext('Show')
   const className = classNames({
@@ -272,4 +251,4 @@ ShowLink.propTypes = {
 }
 
 export { EditLink, CopyLink, AddLink, AvailableLink, ToggleCurrentSiteLink, LockedLink, ShowElementsLink,
-         NestedLink, ExportLink, ExtendLink, CodeLink, ErrorLink, WarningLink, ShowUpdatedLink, ShowLink }
+         NestedLink, ExportLink, ExtendLink, CodeLink, ErrorLink, WarningLink, ShowLink }
diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index 63cad75348..4b6fcb72e2 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -1,41 +1,72 @@
 import React from 'react'
 import PropTypes from 'prop-types'
 
-import { CodeLink, WarningLink, ErrorLink, ShowLink, ShowUpdatedLink } from '../common/Links'
+import {isEmpty} from 'lodash'
 
+import {
+  WarningLink,
+  ErrorLink,
+  ShowLink,
+  AvailableLink,
+  LockedLink, AddLink, EditLink
+} from '../common/Links'
+import SelectCheckbox from './common/SelectCheckbox'
 import Errors from './common/Errors'
+
+import Warnings from './common/Warnings'
 import Fields from './common/Fields'
 import Form from './common/Form'
 
-import { codeClass, verboseNames } from '../../constants/elements'
-import { isEmpty } from 'lodash'
 
 const ImportElement = ({ config, element, importActions }) => {
   const updateShowField = () => importActions.updateElement(element, {show: !element.show})
   const toggleImport = () => importActions.updateElement(element, {import: !element.import})
   const updateElement = (key, value) => importActions.updateElement(element, {[key]: value})
+  const toggleAvailable = () => importActions.updateElement(element, {available: !element.available})
 
   return (
     <li className="list-group-item">
       <div className="pull-right">
-        <WarningLink show={!isEmpty(element.warnings)} onClick={updateShowField} />
-        <ErrorLink show={!isEmpty(element.errors)} onClick={updateShowField} />
-        <ShowUpdatedLink show={(element.changed && !element.created)} disabled={true} onClick={updateShowField} />
-        <ShowLink show={element.show} onClick={updateShowField} />
+        {
+          (isEmpty(element.errors) && ('available' in element)) &&
+           <AvailableLink available={element.available}
+                          locked={element.locked} onClick={toggleAvailable}
+                          title={element.available ? gettext('Make unavailable')
+                                         : gettext('Make available')}/>
+        }
+        {
+          !isEmpty(element.warnings) &&
+          <WarningLink onClick={updateShowField} />
+        }
+        {
+          !isEmpty(element.errors) &&
+          <ErrorLink onClick={updateShowField} />
+        }
+        {
+          (element.changed && element.updated) &&
+          <EditLink href='' title={gettext('Updated and changed')} disabled={true} onClick={updateShowField} />
+        }
+        {
+          element.created &&
+          <AddLink title={gettext('Created')} disabled={true} onClick={updateShowField} />
+        }
+        {
+          (element.updated && element.locked) &&
+          <LockedLink title={gettext('Locked')}
+                      locked={element.locked} onClick={updateShowField} disabled={true} />
 
+        }
+        <ShowLink show={element.show} onClick={updateShowField} />
       </div>
-      <div className="checkbox">
-        <label className="mr-5">
-          <input type="checkbox" checked={element.import} onChange={toggleImport} />
-          <strong>{verboseNames[element.model]}{' '}</strong>
-        </label>
-        <CodeLink className={codeClass[element.model]} uri={element.uri} onClick={updateShowField} />
-      </div>
+
+      <SelectCheckbox element={element} toggleImport={toggleImport} updateShowField={updateShowField} />
+
       {
         element.show && <>
           <Form config={config} element={element} updateElement={updateElement} />
           <Fields element={element} />
-          <Errors element={element} />
+          <Errors element={element} showTitle={true}/>
+          <Warnings element={element} showTitle={true} shouldShowURI={false} />
         </>
       }
     </li>
diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index aff18345d3..375de66025 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -1,20 +1,13 @@
 import React from 'react'
 import PropTypes from 'prop-types'
-import uniqueId from 'lodash/uniqueId'
 
 import { codeClass, verboseNames } from '../../constants/elements'
 import { isEmpty } from 'lodash'
 import Warnings from './common/Warnings'
-import { ShowUpdatedLink } from '../common/Links'
+import {prepareErrorsList } from './common/Errors'
+import {EditLink} from '../common/Links'
 
-const prepareErrorsList = (errors) => {
-  // Filter out duplicate errors
-  const uniqueErrors = [...new Set(errors)]
 
-  return uniqueErrors.map(message => (
-    <p key={uniqueId()} className="text-danger">{message}</p>
-  ))
-}
 
 const ImportSuccessElement = ({ element, importActions }) => {
   const listErrorMessages = prepareErrorsList(element.errors)
@@ -23,9 +16,6 @@ const ImportSuccessElement = ({ element, importActions }) => {
 
   return (
     <li className="list-group-item">
-      <div className="pull-right">
-        <Warnings element={element} showTitle={true} />
-      </div>
       <div>
         <strong>{verboseNames[element.model]}{' '}</strong>
         <code className={codeClass[element.model]}>{element.uri}</code>
@@ -35,8 +25,8 @@ const ImportSuccessElement = ({ element, importActions }) => {
             <span className="muted element-link fa fa-plus"></span>
           </>
         )}
-        {element.updated && (
-          <ShowUpdatedLink show={(element.changed && !element.created)} disabled={true} onClick={updateShowField} />
+        {element.updated && element.changed && (
+          <EditLink href={''} title={gettext('Updated')} disabled={true} onClick={updateShowField} />
         )}
         {!isEmpty(element.errors) && !(element.created || element.updated) && (
           <span className="text-danger">{' '}{gettext('could not be imported')}</span>
@@ -49,6 +39,7 @@ const ImportSuccessElement = ({ element, importActions }) => {
         )}
         {'.'}
       </div>
+      <Warnings element={element} showTitle={true} shouldShowURI={false} />
       {listErrorMessages}
     </li>
   )
diff --git a/rdmo/management/assets/js/components/import/common/FieldRow.js b/rdmo/management/assets/js/components/import/common/FieldRow.js
index ee8504f8df..33a9229763 100644
--- a/rdmo/management/assets/js/components/import/common/FieldRow.js
+++ b/rdmo/management/assets/js/components/import/common/FieldRow.js
@@ -4,21 +4,24 @@ import uniqueId from 'lodash/uniqueId'
 import FieldRowValue from './FieldRowValue'
 import FieldRowDiffs from './FieldRowDiffs'
 
-const FieldRow = ({ element, keyName, value }) => (
-  <div>
-    <div className="row" key={uniqueId()}>
-      <div className="col-sm-3 mb-5 mt-5">
-        <code className="code-import">{keyName}</code>
+const FieldRow = ({ element, keyName, value }) => {
+
+  return (
+    <div>
+      <div className="row" key={uniqueId()}>
+        <div className="col-sm-3 mb-5 mt-5">
+          <code className="code-import">{keyName}</code>
+        </div>
+      </div>
+      <div className="row" key={uniqueId()}>
+        <FieldRowValue value={value} />
+        {element.updated && element.changed && keyName in element.updated_and_changed && (
+          <FieldRowDiffs element={element} field={keyName} />
+        )}
       </div>
     </div>
-    <div className="row" key={uniqueId()}>
-      <FieldRowValue value={value} />
-      {element.updated && element.changed && keyName in element.updated_and_changed && (
-        <FieldRowDiffs element={element} field={keyName} />
-      )}
-    </div>
-  </div>
-)
+  )
+}
 
 FieldRow.propTypes = {
   element: PropTypes.object.isRequired,
diff --git a/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js b/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
index 4aa9d1d44c..a11bd97c70 100644
--- a/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
@@ -15,8 +15,6 @@ const FieldRowDiffs = ({ element, field }) => {
   const changed = fieldDiffData.changed ?? false
   const splitView = false
   const hideLineNumbers = true
-  // const leftTitle = fieldDiffData.leftTitle ?? gettext('Current')
-  // const rightTitle = fieldDiffData.rightTitle ?? gettext('Uploaded')
   const warnings = fieldDiffData.warnings ?? {}
   const errors = fieldDiffData.errors ?? []
 
diff --git a/rdmo/management/assets/js/components/import/common/FieldRowValue.js b/rdmo/management/assets/js/components/import/common/FieldRowValue.js
index eb094236cb..55715eb1c6 100644
--- a/rdmo/management/assets/js/components/import/common/FieldRowValue.js
+++ b/rdmo/management/assets/js/components/import/common/FieldRowValue.js
@@ -4,22 +4,12 @@ import uniqueId from 'lodash/uniqueId'
 import isString from 'lodash/isString'
 import isUndefined from 'lodash/isUndefined'
 import truncate from 'lodash/truncate'
-import { codeClass } from '../../../constants/elements'
+import {codeClass} from '../../../constants/elements'
 import {isNull} from 'lodash'
 
-const serializeValue = (value) => {
-  if (value === null) return ''
-  if (value === true) return 'true'
-  if (value === false) return 'false'
-  if (Array.isArray(value)) return value
-  if (isString(value)) return value
-  if (typeof value === 'number') return value.toString()
-  return value
-}
-
 
 const FieldRowValue = ({ value }) => {
-  const serializedValue = serializeValue(value)
+
   return  (
     <div className="col-sm-12">
       {Array.isArray(value) && (
@@ -32,7 +22,7 @@ const FieldRowValue = ({ value }) => {
         </ul>
       )}
       {!isNull(value) && !isUndefined(value.uri) && <code className={codeClass[value.type]}>{value.uri}</code>}
-      {isString(serializedValue) && <span>{truncate(value, { length: 512 })}</span>}
+      {isString(value) && <span>{truncate(value, { length: 512 })}</span>}
     </div>
   )
 }
diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index 5fab7dfc94..03376b5c84 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -1,8 +1,8 @@
 import React from 'react'
 import PropTypes from 'prop-types'
-import isNil from 'lodash/isNil'
 import uniqueId from 'lodash/uniqueId'
 import FieldRow from './FieldRow'
+import isString from 'lodash/isString'
 
 const excludeKeys = [
   'import',
@@ -24,18 +24,32 @@ const excludeKeys = [
   'locked'
 ]
 
-const Fields = ({ element }) => (
-  <div className="mt-10">
-    {Object.entries(element)
-      .sort()
-      .map(([key, value]) => {
-        if ((!isNil(value) || key in element.updated_and_changed) && !excludeKeys.includes(key)) {
-          return <FieldRow key={uniqueId()} element={element} keyName={key} value={value} />
-        }
-        return null
-      })}
-  </div>
-)
+
+export const serializeValue = (value) => {
+  if (value === null) return ''
+  if (value === true) return 'true'
+  if (value === false) return 'false'
+  if (Array.isArray(value)) return value
+  if (isString(value)) return value
+  if (typeof value === 'number') return value.toString()
+  return value
+}
+
+const Fields = ({ element }) => {
+
+  return (
+    <div className="mt-10">
+      {Object.entries(element)
+        .sort()
+        .map(([key, value]) => {
+          if (!excludeKeys.includes(key)) {
+            return <FieldRow key={uniqueId()} element={element} keyName={key} value={serializeValue(value)} />
+          }
+          return null
+        })}
+    </div>
+  )
+}
 
 Fields.propTypes = {
   element: PropTypes.object.isRequired,
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index a5d28d5a9f..48725031f8 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -62,7 +62,7 @@ const Import = ({ config, imports, configActions, importActions }) => {
           {
             filteredElements.map((element, index) => {
               if (success) {
-                return <ImportSuccessElement key={index} element={element}/>
+                return <ImportSuccessElement key={index} element={element} importActions={importActions}/>
               } else {
                 return <ImportElement key={index} config={config} element={element} importActions={importActions}/>
               }
diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index 68f4a02e66..ffb2621db6 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -59,7 +59,7 @@ export default function importsReducer(state = initialState, action) {
       })}
     case 'import/selectChangedElements':
       return {...state, elements: state.elements.map(element => {
-        if (element.changed && !element.created ) {
+        if (element.changed || element.created ) {
           return {...element, import: action.value}
         }
         else if (action.value) {return {...element, import: !action.value}}
@@ -72,7 +72,7 @@ export default function importsReducer(state = initialState, action) {
       })}
     case 'import/showChangedElements':
       return {...state, elements: state.elements.map(element => {
-        if (element.changed && !element.created ) {
+        if (element.changed || element.created ) {
           return {...element, show: action.value}
         }
         else if (action.value) {return {...element, show: !action.value}}
diff --git a/rdmo/management/assets/js/utils/importFilters.js b/rdmo/management/assets/js/utils/importFilters.js
index b97c5ce829..a5fc35562d 100644
--- a/rdmo/management/assets/js/utils/importFilters.js
+++ b/rdmo/management/assets/js/utils/importFilters.js
@@ -2,7 +2,7 @@ import { useMemo } from 'react'
 import { filterUriPrefix, filterSearch} from './filter'
 
 const filterChanged = (selectFilterChanged, element) => {
-  return element.changed
+  return element.changed || element.created
 }
 
 function filterElementsByChanged(elements, selectFilterChanged) {

From 62100821ece082280f77af1a5674f39e1459c003 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 17 Jun 2024 09:37:39 +0200
Subject: [PATCH 181/205] tests(multisite): update imports for multisite tests

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/conditions/tests/test_viewset_condition_multisite.py  | 6 +++---
 rdmo/domain/tests/test_viewset_attribute_multisite.py      | 6 +++---
 rdmo/options/tests/test_viewset_options_multisite.py       | 6 +++---
 rdmo/options/tests/test_viewset_optionsets_multisite.py    | 6 +++---
 rdmo/questions/tests/test_viewset_catalog_multisite.py     | 6 +++---
 rdmo/questions/tests/test_viewset_page_multisite.py        | 6 +++---
 rdmo/questions/tests/test_viewset_question_multisite.py    | 6 +++---
 rdmo/questions/tests/test_viewset_questionset_multisite.py | 6 +++---
 rdmo/questions/tests/test_viewset_section_multisite.py     | 6 +++---
 rdmo/tasks/tests/test_viewset_task_multisite.py            | 6 +++---
 rdmo/views/tests/test_viewset_view_multisite.py            | 6 +++---
 11 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/rdmo/conditions/tests/test_viewset_condition_multisite.py b/rdmo/conditions/tests/test_viewset_condition_multisite.py
index 740485f09d..6d1f0da587 100644
--- a/rdmo/conditions/tests/test_viewset_condition_multisite.py
+++ b/rdmo/conditions/tests/test_viewset_condition_multisite.py
@@ -4,9 +4,9 @@
 
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import Condition
 from .test_viewset_condition import export_formats, urlnames
diff --git a/rdmo/domain/tests/test_viewset_attribute_multisite.py b/rdmo/domain/tests/test_viewset_attribute_multisite.py
index f0be665e86..39bd7a663e 100644
--- a/rdmo/domain/tests/test_viewset_attribute_multisite.py
+++ b/rdmo/domain/tests/test_viewset_attribute_multisite.py
@@ -4,9 +4,9 @@
 
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import Attribute
 from .test_viewset_attribute import urlnames
diff --git a/rdmo/options/tests/test_viewset_options_multisite.py b/rdmo/options/tests/test_viewset_options_multisite.py
index a009332202..6834c6597e 100644
--- a/rdmo/options/tests/test_viewset_options_multisite.py
+++ b/rdmo/options/tests/test_viewset_options_multisite.py
@@ -4,9 +4,9 @@
 
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import Option
 from .test_viewset_options import urlnames
diff --git a/rdmo/options/tests/test_viewset_optionsets_multisite.py b/rdmo/options/tests/test_viewset_optionsets_multisite.py
index 2a1c2479d6..55bd2a30eb 100644
--- a/rdmo/options/tests/test_viewset_optionsets_multisite.py
+++ b/rdmo/options/tests/test_viewset_optionsets_multisite.py
@@ -4,9 +4,9 @@
 
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import OptionSet
 from .test_viewset_optionsets import urlnames
diff --git a/rdmo/questions/tests/test_viewset_catalog_multisite.py b/rdmo/questions/tests/test_viewset_catalog_multisite.py
index d648b2ae1a..82bd691728 100644
--- a/rdmo/questions/tests/test_viewset_catalog_multisite.py
+++ b/rdmo/questions/tests/test_viewset_catalog_multisite.py
@@ -5,9 +5,9 @@
 from django.contrib.sites.models import Site
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import Catalog
 from .test_viewset_catalog import export_formats, urlnames
diff --git a/rdmo/questions/tests/test_viewset_page_multisite.py b/rdmo/questions/tests/test_viewset_page_multisite.py
index c62ade6061..6bd56690a8 100644
--- a/rdmo/questions/tests/test_viewset_page_multisite.py
+++ b/rdmo/questions/tests/test_viewset_page_multisite.py
@@ -5,9 +5,9 @@
 from django.db.models import Max
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import Page
 from .test_viewset_page import export_formats, urlnames
diff --git a/rdmo/questions/tests/test_viewset_question_multisite.py b/rdmo/questions/tests/test_viewset_question_multisite.py
index 3167055367..73264498ed 100644
--- a/rdmo/questions/tests/test_viewset_question_multisite.py
+++ b/rdmo/questions/tests/test_viewset_question_multisite.py
@@ -5,9 +5,9 @@
 from django.db.models import Max
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import Question
 from .test_viewset_question import export_formats, urlnames
diff --git a/rdmo/questions/tests/test_viewset_questionset_multisite.py b/rdmo/questions/tests/test_viewset_questionset_multisite.py
index 5a1d00a075..2a11a36c2c 100644
--- a/rdmo/questions/tests/test_viewset_questionset_multisite.py
+++ b/rdmo/questions/tests/test_viewset_questionset_multisite.py
@@ -5,9 +5,9 @@
 from django.db.models import Max
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import QuestionSet
 from .test_viewset_questionset import export_formats, urlnames
diff --git a/rdmo/questions/tests/test_viewset_section_multisite.py b/rdmo/questions/tests/test_viewset_section_multisite.py
index 65a6ce1ad3..203d783ef4 100644
--- a/rdmo/questions/tests/test_viewset_section_multisite.py
+++ b/rdmo/questions/tests/test_viewset_section_multisite.py
@@ -5,9 +5,9 @@
 from django.db.models import Max
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import Section
 from .test_viewset_section import export_formats, urlnames
diff --git a/rdmo/tasks/tests/test_viewset_task_multisite.py b/rdmo/tasks/tests/test_viewset_task_multisite.py
index 60e7263dff..402adf9198 100644
--- a/rdmo/tasks/tests/test_viewset_task_multisite.py
+++ b/rdmo/tasks/tests/test_viewset_task_multisite.py
@@ -5,9 +5,9 @@
 from django.contrib.sites.models import Site
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import Task
 from .test_viewset_task import export_formats, urlnames
diff --git a/rdmo/views/tests/test_viewset_view_multisite.py b/rdmo/views/tests/test_viewset_view_multisite.py
index 1dcf1edf37..bfe2c348db 100644
--- a/rdmo/views/tests/test_viewset_view_multisite.py
+++ b/rdmo/views/tests/test_viewset_view_multisite.py
@@ -5,9 +5,9 @@
 from django.contrib.sites.models import Site
 from django.urls import reverse
 
-from rdmo.core.tests import get_obj_perms_status_code
-from rdmo.core.tests import multisite_status_map as status_map
-from rdmo.core.tests import multisite_users as users
+from rdmo.core.tests.constants import multisite_status_map as status_map
+from rdmo.core.tests.constants import multisite_users as users
+from rdmo.core.tests.utils import get_obj_perms_status_code
 
 from ..models import View
 from .test_viewset_view import export_formats, urlnames

From da739548597106927fcb674b8eba33f296594333 Mon Sep 17 00:00:00 2001
From: Jochen Klar <mail@jochenklar.de>
Date: Mon, 24 Jun 2024 13:42:07 +0200
Subject: [PATCH 182/205] Adjust CSS in new import interface

---
 .../assets/js/components/import/common/ImportInfo.js      | 3 ++-
 rdmo/management/assets/scss/management.scss               | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/ImportInfo.js b/rdmo/management/assets/js/components/import/common/ImportInfo.js
index 4e711f76ad..d1fbf9722b 100644
--- a/rdmo/management/assets/js/components/import/common/ImportInfo.js
+++ b/rdmo/management/assets/js/components/import/common/ImportInfo.js
@@ -4,9 +4,10 @@ import {isUndefined} from 'lodash'
 
 const renderElementLengthInfo = (label, length) => (
   length > 0 && (
-    <span className='mr-5'>{gettext(label)}: {length} </span>
+    <span className='ml-5'>{gettext(label)}: {length} </span>
   )
 )
+
 const ImportInfo = ({
                       elementsLength,
                       updatedLength,
diff --git a/rdmo/management/assets/scss/management.scss b/rdmo/management/assets/scss/management.scss
index 0107bde153..54b947b27c 100644
--- a/rdmo/management/assets/scss/management.scss
+++ b/rdmo/management/assets/scss/management.scss
@@ -63,10 +63,10 @@ $icon-font-path: "bootstrap-sass/assets/fonts/bootstrap/";
                 }
             }
             .horizontal-container {
-              display: flex;
-              align-items: center;
-              justify-content: space-between;
-              padding: 6px 15px 0px;
+                display: flex;
+                align-items: center;
+                justify-content: space-between;
+                padding: 10px 15px 0px;
 
                 .shown-info {
                     margin-left: auto;

From 5712e2cd7106c386f6f80aba67cfe035416c1773 Mon Sep 17 00:00:00 2001
From: Jochen Klar <mail@jochenklar.de>
Date: Mon, 24 Jun 2024 14:32:05 +0200
Subject: [PATCH 183/205] Adjust CSS for diff feature in new import interface

---
 rdmo/management/assets/scss/management.scss | 24 ++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/rdmo/management/assets/scss/management.scss b/rdmo/management/assets/scss/management.scss
index 54b947b27c..121c135bcd 100644
--- a/rdmo/management/assets/scss/management.scss
+++ b/rdmo/management/assets/scss/management.scss
@@ -215,24 +215,38 @@ $icon-font-path: "bootstrap-sass/assets/fonts/bootstrap/";
 }
 
 .field-diff {
-    [class*="react-diff-"][class*="diff-container"] {
+    table[class*="react-diff-"][class*="diff-container"] {
         margin: 0;
         border-collapse: separate;  // needed for border radius since this is a table
 
         tr {
+            td[class*="marker"] {
+                border-left: 1px solid #ccc;
+            }
+            td[class*="content"] {
+                border-right: 1px solid #ccc;
+            }
             &:first-child {
-                [class*="marker"] {
+                td[class*="marker"] {
+                    border-top: 1px solid #ccc;
+                    border-left: 1px solid #ccc;
                     border-top-left-radius: 4px;
                 }
-                [class*="content"] {
+                td[class*="content"] {
+                    border-top: 1px solid #ccc;
+                    border-right: 1px solid #ccc;
                     border-top-right-radius: 4px;
                 }
             }
             &:last-child {
-                [class*="marker"] {
+                td[class*="marker"] {
+                    border-bottom: 1px solid #ccc;
+                    border-left: 1px solid #ccc;
                     border-bottom-left-radius: 4px;
                 }
-                [class*="content"] {
+                td[class*="content"] {
+                    border-bottom: 1px solid #ccc;
+                    border-right: 1px solid #ccc;
                     border-bottom-right-radius: 4px;
                 }
             }

From 9b62bda30064d6048fe8af3765315d9e18e6618c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 26 Jun 2024 09:48:10 +0200
Subject: [PATCH 184/205] refactor(js, import): remove extra ul from warnings

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../management/assets/js/components/import/ImportElement.js | 2 +-
 .../assets/js/components/import/ImportSuccessElement.js     | 6 +++---
 .../assets/js/components/import/ImportWarningsPanel.js      | 2 +-
 .../assets/js/components/import/common/FieldRowDiffs.js     | 5 +++--
 .../assets/js/components/import/common/Warnings.js          | 4 ++--
 5 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index 4b6fcb72e2..0986bce997 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -65,7 +65,7 @@ const ImportElement = ({ config, element, importActions }) => {
         element.show && <>
           <Form config={config} element={element} updateElement={updateElement} />
           <Fields element={element} />
-          <Errors element={element} showTitle={true}/>
+          <Errors element={element} showTitle={true} />
           <Warnings element={element} showTitle={true} shouldShowURI={false} />
         </>
       }
diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index 375de66025..47550e911c 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -1,11 +1,11 @@
 import React from 'react'
 import PropTypes from 'prop-types'
+import { isEmpty } from 'lodash'
 
 import { codeClass, verboseNames } from '../../constants/elements'
-import { isEmpty } from 'lodash'
 import Warnings from './common/Warnings'
-import {prepareErrorsList } from './common/Errors'
-import {EditLink} from '../common/Links'
+import { prepareErrorsList } from './common/Errors'
+import { EditLink } from '../common/Links'
 
 
 
diff --git a/rdmo/management/assets/js/components/import/ImportWarningsPanel.js b/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
index a36c95d54c..ade012dcfb 100644
--- a/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
+++ b/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
@@ -26,7 +26,7 @@ const ImportWarningsPanel = ({ config, elements, configActions }) => {
       </div>
       <div className="panel-body">
         { showWarnings &&
-        <ul className="list-group">{listWarnings}</ul>
+          listWarnings
         }
       </div>
     </div>
diff --git a/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js b/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
index a11bd97c70..c79a2ba883 100644
--- a/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
@@ -2,6 +2,7 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import isEmpty from 'lodash/isEmpty'
 import ReactDiffViewer from 'react-diff-viewer-continued'
+
 import Warnings from './Warnings'
 import Errors from './Errors'
 
@@ -45,12 +46,12 @@ const FieldRowDiffs = ({ element, field }) => {
           </ReactDiffViewer>
       {
         !isEmpty(warnings) && <>
-        <Warnings element={fieldDiffData} showTitle={true} shouldShowURI={false}/>
+          <Warnings element={fieldDiffData} showTitle={true} shouldShowURI={false}/>
         </>
       }
       {
         !isEmpty(errors) && <>
-        <Errors element={fieldDiffData} />
+          <Errors element={fieldDiffData} />
         </>
       }
       </div>
diff --git a/rdmo/management/assets/js/components/import/common/Warnings.js b/rdmo/management/assets/js/components/import/common/Warnings.js
index e079a0b49d..5d365caadf 100644
--- a/rdmo/management/assets/js/components/import/common/Warnings.js
+++ b/rdmo/management/assets/js/components/import/common/Warnings.js
@@ -1,7 +1,7 @@
 import React from 'react'
 import PropTypes from 'prop-types'
-// import isEmpty from 'lodash/isEmpty'
 import uniqueId from 'lodash/uniqueId'
+
 import {codeClass} from '../../../constants/elements'
 
 
@@ -31,7 +31,7 @@ const Warnings = ({element, showTitle = false, shouldShowURI = true}) => {
         </div>
       }
       <div className="col-sm-12">
-        <ul className="list-unstyled">{listWarningMessages}</ul>
+        {listWarningMessages}
       </div>
     </div>
   )

From 8533368dc041fb12555b1cc1b675ba702552fd84 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 26 Jun 2024 11:04:09 +0200
Subject: [PATCH 185/205] refactor(import): update import command error comment

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/management/commands/import.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rdmo/management/management/commands/import.py b/rdmo/management/management/commands/import.py
index 11b6d4d0ce..e145118ef3 100644
--- a/rdmo/management/management/commands/import.py
+++ b/rdmo/management/management/commands/import.py
@@ -21,7 +21,7 @@ def handle(self, *args, **options):
             logger.info('Import failed with XML parsing errors.')
             raise CommandError(str(e)) from e
 
-        # step 7: check if valid
+        # raise exception when xml parsing returned any errors
         if errors:
             logger.info('Import failed with XML validation errors.')
             raise CommandError(" ".join(map(str, errors)))

From 5a26e8dc674b5412cc0f658ee1eb9f8cd0bec8f6 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 26 Jun 2024 17:14:06 +0200
Subject: [PATCH 186/205] feat(import): update style and fix display bugs for
 import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../js/components/import/ImportElement.js     | 15 ++-----
 .../js/components/import/ImportErrorsPanel.js | 12 ++---
 .../components/import/ImportWarningsPanel.js  | 11 ++---
 .../js/components/import/common/Errors.js     | 24 +++++-----
 .../components/import/common/FieldRowValue.js | 20 ++++++---
 .../js/components/import/common/Fields.js     |  5 ++-
 ...ectCheckbox.js => ImportSelectCheckbox.js} | 16 +++++--
 .../js/components/import/common/Warnings.js   | 44 +++++++++++--------
 8 files changed, 84 insertions(+), 63 deletions(-)
 rename rdmo/management/assets/js/components/import/common/{SelectCheckbox.js => ImportSelectCheckbox.js} (56%)

diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index 0986bce997..ca5cdb37d5 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -8,9 +8,9 @@ import {
   ErrorLink,
   ShowLink,
   AvailableLink,
-  LockedLink, AddLink, EditLink
+  LockedLink,
 } from '../common/Links'
-import SelectCheckbox from './common/SelectCheckbox'
+import ImportSelectCheckbox from './common/ImportSelectCheckbox'
 import Errors from './common/Errors'
 
 import Warnings from './common/Warnings'
@@ -26,6 +26,7 @@ const ImportElement = ({ config, element, importActions }) => {
 
   return (
     <li className="list-group-item">
+
       <div className="pull-right">
         {
           (isEmpty(element.errors) && ('available' in element)) &&
@@ -42,14 +43,6 @@ const ImportElement = ({ config, element, importActions }) => {
           !isEmpty(element.errors) &&
           <ErrorLink onClick={updateShowField} />
         }
-        {
-          (element.changed && element.updated) &&
-          <EditLink href='' title={gettext('Updated and changed')} disabled={true} onClick={updateShowField} />
-        }
-        {
-          element.created &&
-          <AddLink title={gettext('Created')} disabled={true} onClick={updateShowField} />
-        }
         {
           (element.updated && element.locked) &&
           <LockedLink title={gettext('Locked')}
@@ -59,7 +52,7 @@ const ImportElement = ({ config, element, importActions }) => {
         <ShowLink show={element.show} onClick={updateShowField} />
       </div>
 
-      <SelectCheckbox element={element} toggleImport={toggleImport} updateShowField={updateShowField} />
+      <ImportSelectCheckbox element={element} toggleImport={toggleImport} updateShowField={updateShowField} />
 
       {
         element.show && <>
diff --git a/rdmo/management/assets/js/components/import/ImportErrorsPanel.js b/rdmo/management/assets/js/components/import/ImportErrorsPanel.js
index 2f0911bbb1..fa6d683a0c 100644
--- a/rdmo/management/assets/js/components/import/ImportErrorsPanel.js
+++ b/rdmo/management/assets/js/components/import/ImportErrorsPanel.js
@@ -12,23 +12,23 @@ const ImportErrorsPanel = ({ config, elements, configActions }) => {
 
   const showErrors = get(config, 'filter.import.errors.show', false)
   const listErrors = elements.map((element, index) => {
-            return (<Errors key={index} element={element} />)
-            })
-
+    return (<Errors key={index} element={element}/>)
+  })
+  const errorsHeadingText = <strong onClick={updateShowErrors}>{gettext('Errors')}{' '}({elements.length}){' : '}</strong>
   return (
     <div className="panel panel-danger">
-      <div className="panel-heading"><strong>{gettext('Errors')}{' '}({elements.length}){' : '}</strong>
+      <div className="panel-heading">{errorsHeadingText}
         <div className="pull-right">
           <ShowLink show={showErrors} onClick={updateShowErrors}/>
         </div>
       </div>
       <div className="panel-body">
-        { showErrors &&
+        {showErrors &&
           <ul className="list-group">{listErrors}</ul>
         }
       </div>
     </div>
-   )
+  )
 }
 
 ImportErrorsPanel.propTypes = {
diff --git a/rdmo/management/assets/js/components/import/ImportWarningsPanel.js b/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
index ade012dcfb..27ab425b4a 100644
--- a/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
+++ b/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
@@ -15,22 +15,23 @@ const ImportWarningsPanel = ({ config, elements, configActions }) => {
   }
   const showWarnings = get(config, 'filter.import.warnings.show', false)
   const listWarnings = elements.map((element, index) => {
-              return (<Warnings key={index} element={element} showWarningTitle={true} showTitle={false} />)
-              })
+    return (<Warnings key={index} element={element} showWarningTitle={true} showTitle={false}/>)
+  })
+  const warningsHeadingText = <strong onClick={updateShowWarnings}>{gettext('Warnings')}{' '}({elements.length}){': '}</strong>
   return (
     <div className="panel panel-warning">
-      <div className="panel-heading"><strong onClick={updateShowWarnings}>{gettext('Warnings')}{' '}({elements.length}){': '}</strong>
+      <div className="panel-heading">{warningsHeadingText}
         <div className="pull-right">
           <ShowLink show={showWarnings} onClick={updateShowWarnings}/>
         </div>
       </div>
       <div className="panel-body">
-        { showWarnings &&
+        {showWarnings &&
           listWarnings
         }
       </div>
     </div>
-   )
+  )
 }
 
 ImportWarningsPanel.propTypes = {
diff --git a/rdmo/management/assets/js/components/import/common/Errors.js b/rdmo/management/assets/js/components/import/common/Errors.js
index 0a03bb7689..c072051766 100644
--- a/rdmo/management/assets/js/components/import/common/Errors.js
+++ b/rdmo/management/assets/js/components/import/common/Errors.js
@@ -1,11 +1,10 @@
 import React from 'react'
 import PropTypes from 'prop-types'
 import uniqueId from 'lodash/uniqueId'
-import isEmpty from 'lodash/isEmpty'
 
 // Helper function to generate error messages
-const generateErrorMessages = (messages, key) =>
-  messages.map(message => <li className="text-danger" key={key}>{message}</li>)
+const generateErrorMessageListItems = (messages, key) =>
+  messages.map(message => <li className="text-danger error-item pb-5" key={key}>{message}</li>)
 
 // Helper function to prepare the list of errors
 export const prepareErrorsList = (errors) => {
@@ -13,26 +12,25 @@ export const prepareErrorsList = (errors) => {
   const uniqueErrors = [...new Set(errors)]
 
   return uniqueErrors.map(message => (
-    <ul className="list-unstyled" key={uniqueId('error-list')}>
-      {generateErrorMessages([message], uniqueId('error-message'))}
+    <ul className="list-unstyled error-list" key={uniqueId('error-list')}>
+      {generateErrorMessageListItems([message], uniqueId('error-message'))}
     </ul>
   ))
 }
 
 const Errors = ({ element, showTitle = false }) => {
-  const listErrorMessages = prepareErrorsList(element.errors)
+  const errorMessagesList = prepareErrorsList(element.errors)
+  const show = element.errors.length > 0
 
-  return !isEmpty(element.errors) && (
-    <div className="row text-danger mt-10">
+  return show && (
+    <div className="row text-danger">
       {showTitle && (
-        <div className="col-sm-3 mb-5 mt-5">
+        <div className="col-sm-12 mb-5 mt-5">
           {gettext('Errors')}
         </div>
       )}
-      <div className="col-sm-12">
-        <ul className="list-unstyled">
-          {listErrorMessages}
-        </ul>
+      <div className="col-sm-12 mb-5 mt-5">
+          {errorMessagesList}
       </div>
     </div>
   )
diff --git a/rdmo/management/assets/js/components/import/common/FieldRowValue.js b/rdmo/management/assets/js/components/import/common/FieldRowValue.js
index 55715eb1c6..22cedefc3c 100644
--- a/rdmo/management/assets/js/components/import/common/FieldRowValue.js
+++ b/rdmo/management/assets/js/components/import/common/FieldRowValue.js
@@ -2,27 +2,35 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import uniqueId from 'lodash/uniqueId'
 import isString from 'lodash/isString'
+import isPlainObject from 'lodash/isPlainObject'
 import isUndefined from 'lodash/isUndefined'
 import truncate from 'lodash/truncate'
 import {codeClass} from '../../../constants/elements'
-import {isNull} from 'lodash'
 
 
 const FieldRowValue = ({ value }) => {
 
   return  (
     <div className="col-sm-12">
-      {Array.isArray(value) && (
+      {
+        Array.isArray(value) && (
         <ul className="list-unstyled mb-0">
           {value.map((el) => (
             <li key={uniqueId()}>
-              <code className={codeClass[el.type]}>{el.uri}</code>
+              <code className={codeClass[el.model]}>{el.uri}</code>
             </li>
           ))}
         </ul>
-      )}
-      {!isNull(value) && !isUndefined(value.uri) && <code className={codeClass[value.type]}>{value.uri}</code>}
-      {isString(value) && <span>{truncate(value, { length: 512 })}</span>}
+      )
+      }
+      {
+        isPlainObject(value) && !isUndefined(value.uri) && !isUndefined(value.model) &&
+        <code className={codeClass[value.model]}>{value.uri}</code>
+      }
+      {
+        isString(value) &&
+        <span>{truncate(value, { length: 512 })}</span>
+      }
     </div>
   )
 }
diff --git a/rdmo/management/assets/js/components/import/common/Fields.js b/rdmo/management/assets/js/components/import/common/Fields.js
index 03376b5c84..f49217bebd 100644
--- a/rdmo/management/assets/js/components/import/common/Fields.js
+++ b/rdmo/management/assets/js/components/import/common/Fields.js
@@ -43,7 +43,10 @@ const Fields = ({ element }) => {
         .sort()
         .map(([key, value]) => {
           if (!excludeKeys.includes(key)) {
-            return <FieldRow key={uniqueId()} element={element} keyName={key} value={serializeValue(value)} />
+            const serializedValue = serializeValue(value)
+            if (serializedValue !== '' || (element.changedFields?.includes(key))) {
+              return <FieldRow key={uniqueId()} element={element} keyName={key} value={serializedValue} />
+            }
           }
           return null
         })}
diff --git a/rdmo/management/assets/js/components/import/common/SelectCheckbox.js b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
similarity index 56%
rename from rdmo/management/assets/js/components/import/common/SelectCheckbox.js
rename to rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
index c12b897dd6..bc18ecef1c 100644
--- a/rdmo/management/assets/js/components/import/common/SelectCheckbox.js
+++ b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
@@ -3,20 +3,30 @@ import PropTypes from 'prop-types'
 import { CodeLink } from '../../common/Links'
 import { codeClass, verboseNames } from '../../../constants/elements'
 
-const SelectCheckbox = ({ element, toggleImport, updateShowField }) => (
+const ImportSelectCheckbox = ({ element, toggleImport, updateShowField }) => (
   <div className="checkbox">
     <label className="mr-5">
       <input type="checkbox" checked={element.import} onChange={toggleImport} />
       <strong>{verboseNames[element.model]}{' '}</strong>
     </label>
     <CodeLink className={codeClass[element.model]} uri={element.uri} onClick={updateShowField} />
+   {
+      (element.changed && element.updated) &&
+        <span className="label label-success ml-5" onClick={updateShowField}>
+          {gettext('changed')}</span>
+    }
+    {
+      element.created &&
+        <span className="label label-success ml-5" onClick={updateShowField}>
+          {gettext('created')}</span>
+    }
   </div>
 )
 
-SelectCheckbox.propTypes = {
+ImportSelectCheckbox.propTypes = {
   element: PropTypes.object.isRequired,
   toggleImport: PropTypes.func.isRequired,
   updateShowField: PropTypes.func.isRequired
 }
 
-export default SelectCheckbox
+export default ImportSelectCheckbox
diff --git a/rdmo/management/assets/js/components/import/common/Warnings.js b/rdmo/management/assets/js/components/import/common/Warnings.js
index 5d365caadf..f4df4dff64 100644
--- a/rdmo/management/assets/js/components/import/common/Warnings.js
+++ b/rdmo/management/assets/js/components/import/common/Warnings.js
@@ -1,37 +1,45 @@
 import React from 'react'
 import PropTypes from 'prop-types'
 import uniqueId from 'lodash/uniqueId'
-
-import {codeClass} from '../../../constants/elements'
+import {codeClass, verboseNames} from '../../../constants/elements'
 
 
 const Warnings = ({element, showTitle = false, shouldShowURI = true}) => {
-  const generateWarningMessagesForUri = (messages, key) =>
-    messages.map(message => <li className="text-warning" key={key}>{message}</li>)
+  const warningListItems = (messages, uri) =>
+    messages.map(message => (
+      <li className="warning-item pb-5" key={uniqueId('warning-uri-message')}>
+        {shouldShowURI && (
+          <>
+            <strong>{verboseNames[element.model]}{' '}</strong>
+            <code className={codeClass[element.model]}>{uri}</code>
+            <br/>
+          </>
+        )}
+        <div className="text-warning">
+           {message}
+        </div>
+      </li>
+    ))
 
   const prepareWarningsList = (warningsObj) =>
     Object.entries(warningsObj).map(([uri, messages]) => (
-      <ul className="list-unstyled" key={uniqueId('warning-uri-list')}>
-        {shouldShowURI &&
-          <li key={uniqueId('warning-uri-')}>
-            <code className={codeClass['domain.attribute']}>{uri}</code>
-          </li>
-        }
-        {generateWarningMessagesForUri(messages, uniqueId('warning-uri-message'))}
+      <ul className="list-unstyled warning-list" key={uniqueId('warning-uri-list')}>
+        {warningListItems(messages, uri)}
       </ul>
     ))
 
-  const listWarningMessages = prepareWarningsList(element.warnings)
+  const warningsMessagesList = prepareWarningsList(element.warnings)
+  const show =  warningsMessagesList.length > 0
 
-  return (
-    <div className="row text-warning mt-10">
-      {showTitle === true && listWarningMessages.length > 0 &&
-        <div className="col-sm-3 mb-5 mt-5">
+  return show && (
+    <div className="row ">
+      {showTitle === true &&
+        <div className="col-sm-12 mb-5 mt-5 text-warning">
           {'Warnings'}
         </div>
       }
-      <div className="col-sm-12">
-        {listWarningMessages}
+      <div className="col-sm-12 mb-5 mt-5">
+        {warningsMessagesList}
       </div>
     </div>
   )

From 0b84b0551c96d38aa77df72e65bf830250c916b3 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 28 Jun 2024 13:46:47 +0200
Subject: [PATCH 187/205] style(import): remove row and col- from divs in
 Warnings and Errors

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../management/assets/js/components/import/common/Errors.js | 6 +++---
 .../assets/js/components/import/common/Warnings.js          | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/Errors.js b/rdmo/management/assets/js/components/import/common/Errors.js
index c072051766..4484b68efe 100644
--- a/rdmo/management/assets/js/components/import/common/Errors.js
+++ b/rdmo/management/assets/js/components/import/common/Errors.js
@@ -23,13 +23,13 @@ const Errors = ({ element, showTitle = false }) => {
   const show = element.errors.length > 0
 
   return show && (
-    <div className="row text-danger">
+    <div>
       {showTitle && (
-        <div className="col-sm-12 mb-5 mt-5">
+        <div className="col-sm-12 mb-5 mt-5 text-danger">
           {gettext('Errors')}
         </div>
       )}
-      <div className="col-sm-12 mb-5 mt-5">
+      <div className="mb-5 mt-5">
           {errorMessagesList}
       </div>
     </div>
diff --git a/rdmo/management/assets/js/components/import/common/Warnings.js b/rdmo/management/assets/js/components/import/common/Warnings.js
index f4df4dff64..d9d45132ce 100644
--- a/rdmo/management/assets/js/components/import/common/Warnings.js
+++ b/rdmo/management/assets/js/components/import/common/Warnings.js
@@ -32,13 +32,13 @@ const Warnings = ({element, showTitle = false, shouldShowURI = true}) => {
   const show =  warningsMessagesList.length > 0
 
   return show && (
-    <div className="row ">
+    <div>
       {showTitle === true &&
-        <div className="col-sm-12 mb-5 mt-5 text-warning">
+        <div className="mb-5 mt-5 text-warning">
           {'Warnings'}
         </div>
       }
-      <div className="col-sm-12 mb-5 mt-5">
+      <div className="mb-5 mt-5">
         {warningsMessagesList}
       </div>
     </div>

From dae5872072661aa1921c4de3208d6af19b6c754c Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 28 Jun 2024 17:47:42 +0200
Subject: [PATCH 188/205] style,refactor(import): display errors and warnings
 in list-groups

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../import/ImportAggregatedErrorsPanel.js     | 55 +++++++++++++++++
 .../import/ImportAggregatedWarningsPanel.js   | 55 +++++++++++++++++
 .../js/components/import/ImportElement.js     |  4 +-
 .../js/components/import/ImportErrorsPanel.js | 40 ------------
 .../components/import/ImportSuccessElement.js |  9 ++-
 .../components/import/ImportWarningsPanel.js  | 43 -------------
 .../js/components/import/common/Errors.js     | 41 +++----------
 .../import/common/ErrorsListGroup.js          | 31 ++++++++++
 .../components/import/common/FieldRowDiffs.js |  2 +-
 .../js/components/import/common/Warnings.js   | 61 ++++++-------------
 .../import/common/WarningsListGroup.js        | 44 +++++++++++++
 .../assets/js/components/main/Import.js       | 20 +++---
 12 files changed, 231 insertions(+), 174 deletions(-)
 create mode 100644 rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js
 create mode 100644 rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportErrorsPanel.js
 delete mode 100644 rdmo/management/assets/js/components/import/ImportWarningsPanel.js
 create mode 100644 rdmo/management/assets/js/components/import/common/ErrorsListGroup.js
 create mode 100644 rdmo/management/assets/js/components/import/common/WarningsListGroup.js

diff --git a/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js b/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js
new file mode 100644
index 0000000000..f8385aec9c
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js
@@ -0,0 +1,55 @@
+// ImportAggregatedErrorsPanel.js
+import React from 'react'
+import PropTypes from 'prop-types'
+import { ShowLink } from '../common/Links'
+import { generateErrorMessageListItems } from './common/ErrorsListGroup'
+import get from 'lodash/get'
+
+// Function to aggregate unique errors from elements
+const aggregateUniqueErrors = (elements) => {
+  const allErrors = elements.reduce((acc, element) => {
+    return acc.concat(element.errors)
+  }, [])
+
+  // Filter out duplicate errors
+  const uniqueErrors = [...new Set(allErrors)]
+
+  return uniqueErrors
+}
+
+const ImportAggregatedErrorsPanel = ({ config, elements, configActions }) => {
+  const updateShowErrors = () => {
+    const currentVal = get(config, 'filter.import.errors.show', false)
+    configActions.updateConfig('filter.import.errors.show', !currentVal)
+  }
+
+  const showErrors = get(config, 'filter.import.errors.show', false)
+
+  // Aggregate all unique errors into a single flat array
+  const uniqueErrors = aggregateUniqueErrors(elements)
+
+  const errorsHeadingText = <strong onClick={updateShowErrors}>{gettext('Errors')} ({elements.length}) :</strong>
+
+  return (
+    <div className="panel panel-danger mt-5">
+      <div className="panel-heading">{errorsHeadingText}
+        <div className="pull-right">
+          <ShowLink show={showErrors} onClick={updateShowErrors}/>
+        </div>
+      </div>
+      {showErrors && (
+        <ul className="list-group mb-5 pb-5 pt-5 pl-5 pr-5">
+          {generateErrorMessageListItems(uniqueErrors)}
+        </ul>
+      )}
+    </div>
+  )
+}
+
+ImportAggregatedErrorsPanel.propTypes = {
+  config: PropTypes.object.isRequired,
+  elements: PropTypes.array.isRequired,
+  configActions: PropTypes.object.isRequired
+}
+
+export default ImportAggregatedErrorsPanel
diff --git a/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js b/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
new file mode 100644
index 0000000000..0a92e21715
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
@@ -0,0 +1,55 @@
+// ImportAggregatedWarningsPanel.js
+import React from 'react'
+import PropTypes from 'prop-types'
+import { ShowLink } from '../common/Links'
+import { generateWarningListItems } from './common/WarningsListGroup'
+import get from 'lodash/get'
+
+// Function to aggregate warnings from elements
+const aggregateWarnings = (elements) => {
+  return elements.reduce((acc, element) => {
+    Object.entries(element.warnings).forEach(([uri, messages]) => {
+      acc.push({ elementWarnings: { [uri]: messages }, elementModel: element.model })
+    })
+    return acc
+  }, [])
+}
+
+const ImportAggregatedWarningsPanel = ({ config, elements, configActions }) => {
+  const updateShowWarnings = () => {
+    const currentVal = get(config, 'filter.import.warnings.show', false)
+    configActions.updateConfig('filter.import.warnings.show', !currentVal)
+  }
+
+  const showWarnings = get(config, 'filter.import.warnings.show', false)
+
+  // Aggregate all warnings into a single list
+  const aggregatedWarnings = aggregateWarnings(elements)
+
+  const warningsHeadingText = <strong onClick={updateShowWarnings}>{gettext('Warnings')} ({elements.length}):</strong>
+
+  return (
+    <div className="panel panel-warning mt-5">
+      <div className="panel-heading">{warningsHeadingText}
+        <div className="pull-right">
+          <ShowLink show={showWarnings} onClick={updateShowWarnings} />
+        </div>
+      </div>
+      {showWarnings && (
+        <ul className="list-group mb-5 pb-5 pt-5 pl-5 pr-5">
+          {aggregatedWarnings.map(({ elementWarnings, elementModel }) =>
+            generateWarningListItems(elementWarnings, elementModel)
+          )}
+        </ul>
+      )}
+    </div>
+  )
+}
+
+ImportAggregatedWarningsPanel.propTypes = {
+  config: PropTypes.object.isRequired,
+  elements: PropTypes.array.isRequired,
+  configActions: PropTypes.object.isRequired
+}
+
+export default ImportAggregatedWarningsPanel
diff --git a/rdmo/management/assets/js/components/import/ImportElement.js b/rdmo/management/assets/js/components/import/ImportElement.js
index ca5cdb37d5..64b7887d58 100644
--- a/rdmo/management/assets/js/components/import/ImportElement.js
+++ b/rdmo/management/assets/js/components/import/ImportElement.js
@@ -58,8 +58,8 @@ const ImportElement = ({ config, element, importActions }) => {
         element.show && <>
           <Form config={config} element={element} updateElement={updateElement} />
           <Fields element={element} />
-          <Errors element={element} showTitle={true} />
-          <Warnings element={element} showTitle={true} shouldShowURI={false} />
+          <Errors elementErrors={element.errors} />
+          <Warnings elementWarnings={element.warnings} elementModel={element.model} showTitle={true} shouldShowURI={false} />
         </>
       }
     </li>
diff --git a/rdmo/management/assets/js/components/import/ImportErrorsPanel.js b/rdmo/management/assets/js/components/import/ImportErrorsPanel.js
deleted file mode 100644
index fa6d683a0c..0000000000
--- a/rdmo/management/assets/js/components/import/ImportErrorsPanel.js
+++ /dev/null
@@ -1,40 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-import {ShowLink} from '../common/Links'
-import Errors from './common/Errors'
-import get from 'lodash/get'
-
-const ImportErrorsPanel = ({ config, elements, configActions }) => {
-  const updateShowErrors = () => {
-    const currentVal = get(config, 'filter.import.errors.show', false)
-    configActions.updateConfig('filter.import.errors.show', !currentVal)
-  }
-
-  const showErrors = get(config, 'filter.import.errors.show', false)
-  const listErrors = elements.map((element, index) => {
-    return (<Errors key={index} element={element}/>)
-  })
-  const errorsHeadingText = <strong onClick={updateShowErrors}>{gettext('Errors')}{' '}({elements.length}){' : '}</strong>
-  return (
-    <div className="panel panel-danger">
-      <div className="panel-heading">{errorsHeadingText}
-        <div className="pull-right">
-          <ShowLink show={showErrors} onClick={updateShowErrors}/>
-        </div>
-      </div>
-      <div className="panel-body">
-        {showErrors &&
-          <ul className="list-group">{listErrors}</ul>
-        }
-      </div>
-    </div>
-  )
-}
-
-ImportErrorsPanel.propTypes = {
-  config: PropTypes.object.isRequired,
-  elements: PropTypes.array.isRequired,
-  configActions: PropTypes.object.isRequired
-}
-
-export default ImportErrorsPanel
diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index 47550e911c..fbe817d75d 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -4,19 +4,18 @@ import { isEmpty } from 'lodash'
 
 import { codeClass, verboseNames } from '../../constants/elements'
 import Warnings from './common/Warnings'
-import { prepareErrorsList } from './common/Errors'
+import Errors from './common/Errors'
 import { EditLink } from '../common/Links'
 
 
 
 const ImportSuccessElement = ({ element, importActions }) => {
-  const listErrorMessages = prepareErrorsList(element.errors)
 
   const updateShowField = () => importActions.updateElement(element, { show: !element.show })
 
   return (
     <li className="list-group-item">
-      <div>
+      <div className="mb-5">
         <strong>{verboseNames[element.model]}{' '}</strong>
         <code className={codeClass[element.model]}>{element.uri}</code>
         {element.created && (
@@ -39,8 +38,8 @@ const ImportSuccessElement = ({ element, importActions }) => {
         )}
         {'.'}
       </div>
-      <Warnings element={element} showTitle={true} shouldShowURI={false} />
-      {listErrorMessages}
+      <Errors elementErrors={element.errors} />
+      <Warnings elementWarnings={element.warnings} elementModel={element.model} showTitle={true} shouldShowURI={false} />
     </li>
   )
 }
diff --git a/rdmo/management/assets/js/components/import/ImportWarningsPanel.js b/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
deleted file mode 100644
index 27ab425b4a..0000000000
--- a/rdmo/management/assets/js/components/import/ImportWarningsPanel.js
+++ /dev/null
@@ -1,43 +0,0 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-
-import {ShowLink} from '../common/Links'
-
-import Warnings from './common/Warnings'
-
-import get from 'lodash/get'
-
-const ImportWarningsPanel = ({ config, elements, configActions }) => {
-
-  const updateShowWarnings = () => {
-    const currentVal = get(config, 'filter.import.warnings.show', false)
-    configActions.updateConfig('filter.import.warnings.show', !currentVal)
-  }
-  const showWarnings = get(config, 'filter.import.warnings.show', false)
-  const listWarnings = elements.map((element, index) => {
-    return (<Warnings key={index} element={element} showWarningTitle={true} showTitle={false}/>)
-  })
-  const warningsHeadingText = <strong onClick={updateShowWarnings}>{gettext('Warnings')}{' '}({elements.length}){': '}</strong>
-  return (
-    <div className="panel panel-warning">
-      <div className="panel-heading">{warningsHeadingText}
-        <div className="pull-right">
-          <ShowLink show={showWarnings} onClick={updateShowWarnings}/>
-        </div>
-      </div>
-      <div className="panel-body">
-        {showWarnings &&
-          listWarnings
-        }
-      </div>
-    </div>
-  )
-}
-
-ImportWarningsPanel.propTypes = {
-  config: PropTypes.object.isRequired,
-  elements: PropTypes.array.isRequired,
-  configActions: PropTypes.object.isRequired
-}
-
-export default ImportWarningsPanel
diff --git a/rdmo/management/assets/js/components/import/common/Errors.js b/rdmo/management/assets/js/components/import/common/Errors.js
index 4484b68efe..f05b2757af 100644
--- a/rdmo/management/assets/js/components/import/common/Errors.js
+++ b/rdmo/management/assets/js/components/import/common/Errors.js
@@ -1,44 +1,23 @@
+// Errors.js
 import React from 'react'
 import PropTypes from 'prop-types'
-import uniqueId from 'lodash/uniqueId'
+import ErrorsListGroup from './ErrorsListGroup'
+import isUndefined from 'lodash/isUndefined'
 
-// Helper function to generate error messages
-const generateErrorMessageListItems = (messages, key) =>
-  messages.map(message => <li className="text-danger error-item pb-5" key={key}>{message}</li>)
-
-// Helper function to prepare the list of errors
-export const prepareErrorsList = (errors) => {
-  // Filter out duplicate errors
-  const uniqueErrors = [...new Set(errors)]
-
-  return uniqueErrors.map(message => (
-    <ul className="list-unstyled error-list" key={uniqueId('error-list')}>
-      {generateErrorMessageListItems([message], uniqueId('error-message'))}
-    </ul>
-  ))
-}
-
-const Errors = ({ element, showTitle = false }) => {
-  const errorMessagesList = prepareErrorsList(element.errors)
-  const show = element.errors.length > 0
+const Errors = ({ elementErrors }) => {
+  const show = !isUndefined(elementErrors) &&  elementErrors.length > 0
+  const errorsHeadingText = <strong>{gettext('Errors')}</strong>
 
   return show && (
-    <div>
-      {showTitle && (
-        <div className="col-sm-12 mb-5 mt-5 text-danger">
-          {gettext('Errors')}
-        </div>
-      )}
-      <div className="mb-5 mt-5">
-          {errorMessagesList}
-      </div>
+    <div className="panel panel-danger">
+      <div className="panel-heading">{errorsHeadingText}</div>
+      <ErrorsListGroup elementErrors={elementErrors} />
     </div>
   )
 }
 
 Errors.propTypes = {
-  element: PropTypes.object.isRequired,
-  showTitle: PropTypes.bool,
+  elementErrors: PropTypes.array.isRequired,
 }
 
 export default Errors
diff --git a/rdmo/management/assets/js/components/import/common/ErrorsListGroup.js b/rdmo/management/assets/js/components/import/common/ErrorsListGroup.js
new file mode 100644
index 0000000000..c6f209938b
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/common/ErrorsListGroup.js
@@ -0,0 +1,31 @@
+// ErrorsListGroup.js
+import React from 'react'
+import PropTypes from 'prop-types'
+import uniqueId from 'lodash/uniqueId'
+
+// Helper function to generate error messages
+export const generateErrorMessageListItems = (messages) =>
+  messages.map(message => (
+    <li className="list-group-item text-danger" key={uniqueId('error-message')}>
+      <div className="text-danger">
+        {message}
+      </div>
+    </li>
+  ))
+
+const ErrorsListGroup = ({ elementErrors }) => {
+  // Filter out duplicate elementErrors
+  const uniqueErrors = [...new Set(elementErrors)]
+
+  return (
+    <ul className="list-group">
+      {generateErrorMessageListItems(uniqueErrors)}
+    </ul>
+  )
+}
+
+ErrorsListGroup.propTypes = {
+  elementErrors: PropTypes.array.isRequired
+}
+
+export default ErrorsListGroup
diff --git a/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js b/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
index c79a2ba883..86d2e97230 100644
--- a/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
+++ b/rdmo/management/assets/js/components/import/common/FieldRowDiffs.js
@@ -46,7 +46,7 @@ const FieldRowDiffs = ({ element, field }) => {
           </ReactDiffViewer>
       {
         !isEmpty(warnings) && <>
-          <Warnings element={fieldDiffData} showTitle={true} shouldShowURI={false}/>
+          <Warnings elementWarnings={fieldDiffData.warnings} elementModel={element.model} showTitle={true} shouldShowURI={false}/>
         </>
       }
       {
diff --git a/rdmo/management/assets/js/components/import/common/Warnings.js b/rdmo/management/assets/js/components/import/common/Warnings.js
index d9d45132ce..1e96a2bac5 100644
--- a/rdmo/management/assets/js/components/import/common/Warnings.js
+++ b/rdmo/management/assets/js/components/import/common/Warnings.js
@@ -1,55 +1,28 @@
 import React from 'react'
 import PropTypes from 'prop-types'
-import uniqueId from 'lodash/uniqueId'
-import {codeClass, verboseNames} from '../../../constants/elements'
+import WarningsListGroup from './WarningsListGroup'
+import isUndefined from 'lodash/isUndefined'
 
-
-const Warnings = ({element, showTitle = false, shouldShowURI = true}) => {
-  const warningListItems = (messages, uri) =>
-    messages.map(message => (
-      <li className="warning-item pb-5" key={uniqueId('warning-uri-message')}>
-        {shouldShowURI && (
-          <>
-            <strong>{verboseNames[element.model]}{' '}</strong>
-            <code className={codeClass[element.model]}>{uri}</code>
-            <br/>
-          </>
-        )}
-        <div className="text-warning">
-           {message}
-        </div>
-      </li>
-    ))
-
-  const prepareWarningsList = (warningsObj) =>
-    Object.entries(warningsObj).map(([uri, messages]) => (
-      <ul className="list-unstyled warning-list" key={uniqueId('warning-uri-list')}>
-        {warningListItems(messages, uri)}
-      </ul>
-    ))
-
-  const warningsMessagesList = prepareWarningsList(element.warnings)
-  const show =  warningsMessagesList.length > 0
+const Warnings = ({elementWarnings, elementModel, shouldShowURI = true}) => {
+  const show = !isUndefined(elementWarnings) && Object.keys(elementWarnings).length > 0
+  const warningsHeadingText = <strong>{gettext('Warnings')}</strong>
 
   return show && (
-    <div>
-      {showTitle === true &&
-        <div className="mb-5 mt-5 text-warning">
-          {'Warnings'}
-        </div>
-      }
-      <div className="mb-5 mt-5">
-        {warningsMessagesList}
-      </div>
+    <div className="panel panel-warning">
+      <div className="panel-heading">{warningsHeadingText}</div>
+      <WarningsListGroup
+        elementWarnings={elementWarnings}
+        elementModel={elementModel}
+        shouldShowURI={shouldShowURI}
+      />
     </div>
   )
 }
 
-
 Warnings.propTypes = {
-    element: PropTypes.object.isRequired,
-    showTitle: PropTypes.bool.isRequired,
-    shouldShowURI: PropTypes.bool,
-  }
+  elementWarnings: PropTypes.object.isRequired,
+  elementModel: PropTypes.string.isRequired,
+  shouldShowURI: PropTypes.bool
+}
 
-  export default Warnings
+export default Warnings
diff --git a/rdmo/management/assets/js/components/import/common/WarningsListGroup.js b/rdmo/management/assets/js/components/import/common/WarningsListGroup.js
new file mode 100644
index 0000000000..00c6e3d99f
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/common/WarningsListGroup.js
@@ -0,0 +1,44 @@
+// WarningsListGroup.js
+import React from 'react'
+import PropTypes from 'prop-types'
+import uniqueId from 'lodash/uniqueId'
+import { codeClass, verboseNames } from '../../../constants/elements'
+
+// Helper function to generate warning messages
+export const generateWarningListItems = (elementWarnings, elementModel, shouldShowURI = true) =>
+  Object.entries(elementWarnings).flatMap(([uri, messages]) =>
+    messages.map(message => (
+      <li className="list-group-item" key={uniqueId('warning-uri-message')}>
+        {shouldShowURI && (
+          <>
+            <strong>{verboseNames[elementModel]}{' '}</strong>
+            <code className={codeClass[elementModel]}>{uri}</code>
+            <br />
+          </>
+        )}
+        <div className="text-warning">
+          {message}
+        </div>
+      </li>
+    ))
+  )
+
+const WarningsListGroup = ({ elementWarnings, elementModel, shouldShowURI }) => {
+  return (
+    <ul className="list-group">
+      {generateWarningListItems(elementWarnings, elementModel, shouldShowURI)}
+    </ul>
+  )
+}
+
+WarningsListGroup.propTypes = {
+  elementWarnings: PropTypes.object.isRequired,
+  elementModel: PropTypes.string.isRequired,
+  shouldShowURI: PropTypes.bool
+}
+
+WarningsListGroup.defaultProps = {
+  shouldShowURI: true,
+}
+
+export default WarningsListGroup
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index 48725031f8..cd9ff46f48 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -4,8 +4,8 @@ import get from 'lodash/get'
 
 import ImportElement from '../import/ImportElement'
 import ImportSuccessElement from '../import/ImportSuccessElement'
-import ImportWarningsPanel from '../import/ImportWarningsPanel'
-import ImportErrorsPanel from '../import/ImportErrorsPanel'
+import ImportAggregatedWarningsPanel from '../import/ImportAggregatedWarningsPanel'
+import ImportAggregatedErrorsPanel from '../import/ImportAggregatedErrorsPanel'
 import ImportInfo from '../import/common/ImportInfo'
 import ImportFilters from '../import/common/ImportFilters'
 import useFilteredElements from '../../utils/importFilters'
@@ -49,22 +49,26 @@ const Import = ({ config, imports, configActions, importActions }) => {
         }
         {
           importWarnings.length > 0 &&
-          <ImportWarningsPanel config={config} elements={importWarnings} importActions={importActions}
-                               configActions={configActions}/>
+          <ImportAggregatedWarningsPanel config={config} elements={importWarnings}
+                                         importActions={importActions}
+                                         configActions={configActions}/>
         }
         {
           importErrors.length > 0 &&
-          <ImportErrorsPanel config={config} elements={importErrors} importActions={importActions}
-                             configActions={configActions}/>
+          <ImportAggregatedErrorsPanel config={config} elements={importErrors}
+                                       importActions={importActions}
+                                       configActions={configActions}/>
         }
       </div>
         <ul className='list-group'>
           {
             filteredElements.map((element, index) => {
               if (success) {
-                return <ImportSuccessElement key={index} element={element} importActions={importActions}/>
+                return <ImportSuccessElement key={index} element={element}
+                                             importActions={importActions}/>
               } else {
-                return <ImportElement key={index} config={config} element={element} importActions={importActions}/>
+                return <ImportElement key={index} element={element} config={config}
+                                      importActions={importActions}/>
               }
             })
           }

From 29609cdcb5d0e36af4421c21804eb80fcdb3a08e Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 28 Jun 2024 17:49:25 +0200
Subject: [PATCH 189/205] tests(import): add one condition that raises a
 warning to optionsets-1 fixture

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/tests/test_frontend_import_options.py     | 2 +-
 rdmo/management/tests/test_import_options.py              | 4 ++--
 testing/xml/elements/updated-and-changed/optionsets-1.xml | 4 +++-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
index ac76aea3a0..3333635de8 100644
--- a/rdmo/management/tests/test_frontend_import_options.py
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -22,7 +22,7 @@
     "total": 13,
     "updated": 13,
     "changed": 5,
-    "warnings": 1
+    "warnings": 2
 }
 OPTIONSETS_COUNTS_HEADER_INFOS = [f"{k.capitalize()}: {v}" for k,v in OPTIONSETS_COUNTS.items()]
 
diff --git a/rdmo/management/tests/test_import_options.py b/rdmo/management/tests/test_import_options.py
index 6320689af1..d1f54b156b 100644
--- a/rdmo/management/tests/test_import_options.py
+++ b/rdmo/management/tests/test_import_options.py
@@ -97,7 +97,7 @@ def test_update_optionsets_from_changed_xml(db, settings):
     assert imported_elements_1
     assert [i for i in imported_elements_1 if i[ImportElementFields.DIFF]]
     warnings_elements = [i for i in imported_elements_1 if i[ImportElementFields.WARNINGS]]
-    assert len(warnings_elements) == 1
+    assert len(warnings_elements) == 2
 
     changed_elements = get_changed_elements(imported_elements_1)
 
@@ -126,7 +126,7 @@ def test_update_optionsets_from_changed_xml(db, settings):
     imported_elements_2 = import_elements(elements_1, save=False)
     changed_elements_2 = get_changed_elements(imported_elements_2)
     assert len(changed_elements_2) == 0
-    assert len([i for i in imported_elements_2 if i[ImportElementFields.WARNINGS]]) == 1
+    assert len([i for i in imported_elements_2 if i[ImportElementFields.WARNINGS]]) == 2
 
 
 def test_create_options(db, settings):
diff --git a/testing/xml/elements/updated-and-changed/optionsets-1.xml b/testing/xml/elements/updated-and-changed/optionsets-1.xml
index c654a06bf2..c072d471da 100644
--- a/testing/xml/elements/updated-and-changed/optionsets-1.xml
+++ b/testing/xml/elements/updated-and-changed/optionsets-1.xml
@@ -32,7 +32,9 @@
 			<option dc:uri="http://example.com/terms/options/one_two_three/two" order="2"/>
 			<option dc:uri="http://example.com/terms/options/one_two_three/one" order="3"/>
 		</options>
-		<conditions/>
+		<conditions>
+			<condition dc:uri="http://example.com/terms/conditions/does_not_exist"/>
+		</conditions>
 	</optionset>
 	<option dc:uri="http://example.com/terms/options/one_two_three/one">
 		<uri_prefix>http://example.com/terms</uri_prefix>

From 167cf1c0cc278aaa3b1ba47f9a02b319d75da5d6 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Fri, 28 Jun 2024 18:18:32 +0200
Subject: [PATCH 190/205] feat(import): use label-info for changed

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/import/common/ImportSelectCheckbox.js  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
index bc18ecef1c..2bec30f946 100644
--- a/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
+++ b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
@@ -12,7 +12,7 @@ const ImportSelectCheckbox = ({ element, toggleImport, updateShowField }) => (
     <CodeLink className={codeClass[element.model]} uri={element.uri} onClick={updateShowField} />
    {
       (element.changed && element.updated) &&
-        <span className="label label-success ml-5" onClick={updateShowField}>
+        <span className="label label-info ml-5" onClick={updateShowField}>
           {gettext('changed')}</span>
     }
     {

From 2b626f162d6e81f1ec74db54e05616d754f096a8 Mon Sep 17 00:00:00 2001
From: Jochen Klar <mail@jochenklar.de>
Date: Mon, 1 Jul 2024 10:33:40 +0200
Subject: [PATCH 191/205] Adjust warning panel in import

---
 .../js/components/import/ImportAggregatedWarningsPanel.js  | 7 ++++---
 .../assets/js/components/import/common/Warnings.js         | 2 +-
 rdmo/management/assets/scss/management.scss                | 4 ++++
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js b/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
index 0a92e21715..6a7b6856b4 100644
--- a/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
+++ b/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
@@ -29,10 +29,11 @@ const ImportAggregatedWarningsPanel = ({ config, elements, configActions }) => {
   const warningsHeadingText = <strong onClick={updateShowWarnings}>{gettext('Warnings')} ({elements.length}):</strong>
 
   return (
-    <div className="panel panel-warning mt-5">
-      <div className="panel-heading">{warningsHeadingText}
+    <div className="panel panel-warning mt-10">
+      <div className="panel-heading" onClick={updateShowWarnings}>
+        {warningsHeadingText}
         <div className="pull-right">
-          <ShowLink show={showWarnings} onClick={updateShowWarnings} />
+          <ShowLink show={showWarnings} onClick={() => {}}/>
         </div>
       </div>
       {showWarnings && (
diff --git a/rdmo/management/assets/js/components/import/common/Warnings.js b/rdmo/management/assets/js/components/import/common/Warnings.js
index 1e96a2bac5..2ecfc75ab1 100644
--- a/rdmo/management/assets/js/components/import/common/Warnings.js
+++ b/rdmo/management/assets/js/components/import/common/Warnings.js
@@ -8,7 +8,7 @@ const Warnings = ({elementWarnings, elementModel, shouldShowURI = true}) => {
   const warningsHeadingText = <strong>{gettext('Warnings')}</strong>
 
   return show && (
-    <div className="panel panel-warning">
+    <div className="panel panel-warning mt-10 mb-0">
       <div className="panel-heading">{warningsHeadingText}</div>
       <WarningsListGroup
         elementWarnings={elementWarnings}
diff --git a/rdmo/management/assets/scss/management.scss b/rdmo/management/assets/scss/management.scss
index 121c135bcd..a2665c58b7 100644
--- a/rdmo/management/assets/scss/management.scss
+++ b/rdmo/management/assets/scss/management.scss
@@ -73,6 +73,10 @@ $icon-font-path: "bootstrap-sass/assets/fonts/bootstrap/";
                 }
             }
         }
+
+        &.panel-warning {
+            cursor: pointer;
+        }
     }
     .pull-right {
         .element-link,

From f82d5894d84f585d2d433beea8e80aabc8e434fa Mon Sep 17 00:00:00 2001
From: Jochen Klar <mail@jochenklar.de>
Date: Mon, 1 Jul 2024 10:34:01 +0200
Subject: [PATCH 192/205] Fix ImportSuccessElement component

---
 .../components/import/ImportSuccessElement.js   | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index fbe817d75d..b755733c71 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -5,9 +5,6 @@ import { isEmpty } from 'lodash'
 import { codeClass, verboseNames } from '../../constants/elements'
 import Warnings from './common/Warnings'
 import Errors from './common/Errors'
-import { EditLink } from '../common/Links'
-
-
 
 const ImportSuccessElement = ({ element, importActions }) => {
 
@@ -18,14 +15,13 @@ const ImportSuccessElement = ({ element, importActions }) => {
       <div className="mb-5">
         <strong>{verboseNames[element.model]}{' '}</strong>
         <code className={codeClass[element.model]}>{element.uri}</code>
-        {element.created && (
-          <>
-            <span className="text-success">{' '}{gettext('created')} </span>
-            <span className="muted element-link fa fa-plus"></span>
-          </>
-        )}
         {element.updated && element.changed && (
-          <EditLink href={''} title={gettext('Updated')} disabled={true} onClick={updateShowField} />
+        <span className="label label-info ml-5" onClick={updateShowField}>
+          {gettext('changed')}</span>
+        )}
+        {element.created && (
+          <span className="label label-success ml-5" onClick={updateShowField}>
+            {gettext('created')}</span>
         )}
         {!isEmpty(element.errors) && !(element.created || element.updated) && (
           <span className="text-danger">{' '}{gettext('could not be imported')}</span>
@@ -36,7 +32,6 @@ const ImportSuccessElement = ({ element, importActions }) => {
             <span className="text-danger">{gettext('but could not be added to parent element')}</span>
           </>
         )}
-        {'.'}
       </div>
       <Errors elementErrors={element.errors} />
       <Warnings elementWarnings={element.warnings} elementModel={element.model} showTitle={true} shouldShowURI={false} />

From bb57561e7312e1590ceea3570223dfc0e46eb331 Mon Sep 17 00:00:00 2001
From: Jochen Klar <mail@jochenklar.de>
Date: Mon, 1 Jul 2024 10:35:10 +0200
Subject: [PATCH 193/205] Fix whitespaces

---
 .../js/components/import/common/Form.js       |  1 -
 .../components/import/common/ImportFilters.js | 24 +++++++++----------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/Form.js b/rdmo/management/assets/js/components/import/common/Form.js
index a34892cc14..fa872b4efa 100644
--- a/rdmo/management/assets/js/components/import/common/Form.js
+++ b/rdmo/management/assets/js/components/import/common/Form.js
@@ -7,7 +7,6 @@ import UriPath from './UriPath'
 import UriPrefix from './UriPrefix'
 
 const Form = ({ config, element, updateElement }) => {
-
   return (
     <div className="row mt-10">
       <div className="col-sm-6">
diff --git a/rdmo/management/assets/js/components/import/common/ImportFilters.js b/rdmo/management/assets/js/components/import/common/ImportFilters.js
index 5e545d8707..0f9731a7df 100644
--- a/rdmo/management/assets/js/components/import/common/ImportFilters.js
+++ b/rdmo/management/assets/js/components/import/common/ImportFilters.js
@@ -14,17 +14,16 @@ const ImportFilters = ({ config, elements, changedElements, filteredElements, co
   const getValueFilterChanged = () => get(config, 'filter.import.elements.changed', false)
 
   return (
-
-      <div className="row">
-        <div className={'col-sm-8'}>
-          <FilterString value={getValueFilterString()} onChange={updateFilterString}
-                        placeholder={gettext('Filter uri')}/>
-        </div>
-        <div className="col-sm-4">
-          <FilterUriPrefix value={getValueFilterUirPrefix()}
-                           onChange={updateFilterUriPrefix}
-                           options={getUriPrefixes(elements)}/>
-        </div>
+    <div className="row">
+      <div className={'col-sm-8'}>
+        <FilterString value={getValueFilterString()} onChange={updateFilterString}
+                      placeholder={gettext('Filter uri')}/>
+      </div>
+      <div className="col-sm-4">
+        <FilterUriPrefix value={getValueFilterUirPrefix()}
+                         onChange={updateFilterUriPrefix}
+                         options={getUriPrefixes(elements)}/>
+      </div>
 
       {elements.length > 0 && (
         <div className="horizontal-container">
@@ -37,7 +36,8 @@ const ImportFilters = ({ config, elements, changedElements, filteredElements, co
           </span>
         </div>
       )}
-  </div>)
+    </div>
+  )
 }
 
 ImportFilters.propTypes = {

From f20109f84c2af97956e3a3d225b90a0145837901 Mon Sep 17 00:00:00 2001
From: Jochen Klar <mail@jochenklar.de>
Date: Mon, 1 Jul 2024 10:59:15 +0200
Subject: [PATCH 194/205] Adjust error panel im import

---
 .../js/components/import/ImportAggregatedErrorsPanel.js    | 7 ++++---
 .../js/components/import/ImportAggregatedWarningsPanel.js  | 2 +-
 .../assets/js/components/import/common/Errors.js           | 2 +-
 rdmo/management/assets/scss/management.scss                | 3 ++-
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js b/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js
index f8385aec9c..b40021c1a5 100644
--- a/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js
+++ b/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js
@@ -31,10 +31,11 @@ const ImportAggregatedErrorsPanel = ({ config, elements, configActions }) => {
   const errorsHeadingText = <strong onClick={updateShowErrors}>{gettext('Errors')} ({elements.length}) :</strong>
 
   return (
-    <div className="panel panel-danger mt-5">
-      <div className="panel-heading">{errorsHeadingText}
+    <div className="panel panel-danger panel-import-errors mt-10">
+      <div className="panel-heading" onClick={updateShowErrors}>
+        {errorsHeadingText}
         <div className="pull-right">
-          <ShowLink show={showErrors} onClick={updateShowErrors}/>
+          <ShowLink show={showErrors} onClick={() => {}}/>
         </div>
       </div>
       {showErrors && (
diff --git a/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js b/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
index 6a7b6856b4..4a10ca2241 100644
--- a/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
+++ b/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
@@ -29,7 +29,7 @@ const ImportAggregatedWarningsPanel = ({ config, elements, configActions }) => {
   const warningsHeadingText = <strong onClick={updateShowWarnings}>{gettext('Warnings')} ({elements.length}):</strong>
 
   return (
-    <div className="panel panel-warning mt-10">
+    <div className="panel panel-warning panel-import-warnings mt-10">
       <div className="panel-heading" onClick={updateShowWarnings}>
         {warningsHeadingText}
         <div className="pull-right">
diff --git a/rdmo/management/assets/js/components/import/common/Errors.js b/rdmo/management/assets/js/components/import/common/Errors.js
index f05b2757af..35af668236 100644
--- a/rdmo/management/assets/js/components/import/common/Errors.js
+++ b/rdmo/management/assets/js/components/import/common/Errors.js
@@ -9,7 +9,7 @@ const Errors = ({ elementErrors }) => {
   const errorsHeadingText = <strong>{gettext('Errors')}</strong>
 
   return show && (
-    <div className="panel panel-danger">
+    <div className="panel panel-danger mt-10 mb-0">
       <div className="panel-heading">{errorsHeadingText}</div>
       <ErrorsListGroup elementErrors={elementErrors} />
     </div>
diff --git a/rdmo/management/assets/scss/management.scss b/rdmo/management/assets/scss/management.scss
index a2665c58b7..0efdb1a78a 100644
--- a/rdmo/management/assets/scss/management.scss
+++ b/rdmo/management/assets/scss/management.scss
@@ -74,7 +74,8 @@ $icon-font-path: "bootstrap-sass/assets/fonts/bootstrap/";
             }
         }
 
-        &.panel-warning {
+        &.panel-import-warnings,
+        &.panel-import-errors {
             cursor: pointer;
         }
     }

From b13b44303b5d9a04b680bc1700b2eaaa597a1cf4 Mon Sep 17 00:00:00 2001
From: Jochen Klar <mail@jochenklar.de>
Date: Mon, 1 Jul 2024 11:03:20 +0200
Subject: [PATCH 195/205] Change ShowLink to chevron

---
 rdmo/core/assets/scss/variables.scss                 |  1 +
 rdmo/management/assets/js/components/common/Links.js |  4 ++--
 rdmo/management/assets/scss/management.scss          | 11 +++++++++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/rdmo/core/assets/scss/variables.scss b/rdmo/core/assets/scss/variables.scss
index 1af6a6b451..c93eca130b 100644
--- a/rdmo/core/assets/scss/variables.scss
+++ b/rdmo/core/assets/scss/variables.scss
@@ -25,6 +25,7 @@ $footer-link-color: #999;
 $footer-background-color: #001;
 
 $danger-color: #a94442;
+$warning-color: #8a6d3b;
 $success-color: #5cb85c;
 $modal-border-color: #e5e5e5;
 
diff --git a/rdmo/management/assets/js/components/common/Links.js b/rdmo/management/assets/js/components/common/Links.js
index 1d30db1809..c9dc7e11ca 100644
--- a/rdmo/management/assets/js/components/common/Links.js
+++ b/rdmo/management/assets/js/components/common/Links.js
@@ -238,8 +238,8 @@ const ShowLink = ({ show = false, onClick }) => {
   const title = show ? gettext('Hide') : gettext('Show')
   const className = classNames({
     'element-link fa': true,
-    'fa-eye-slash': !show,
-    'fa-eye': show
+    'fa-chevron-down': !show,
+    'fa-chevron-up': show
   })
 
   return <Link className={className} title={title} onClick={onClick} />
diff --git a/rdmo/management/assets/scss/management.scss b/rdmo/management/assets/scss/management.scss
index 0efdb1a78a..03042cdd1c 100644
--- a/rdmo/management/assets/scss/management.scss
+++ b/rdmo/management/assets/scss/management.scss
@@ -78,6 +78,17 @@ $icon-font-path: "bootstrap-sass/assets/fonts/bootstrap/";
         &.panel-import-errors {
             cursor: pointer;
         }
+
+        &.panel-import-warnings {
+            .element-link {
+                color: $warning-color;
+            }
+        }
+        &.panel-import-errors {
+            .element-link {
+                color: $danger-color;
+            }
+        }
     }
     .pull-right {
         .element-link,

From a384f8d47643a4cb05d286ecee5774feab69d4a5 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 1 Jul 2024 15:01:48 +0200
Subject: [PATCH 196/205] refactor(js,import): add components for import labels

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/common/Labels.js     | 21 +++++++++++++
 .../import/ImportAggregatedErrorsPanel.js     |  2 +-
 .../import/ImportAggregatedWarningsPanel.js   |  2 +-
 .../components/import/ImportSuccessElement.js | 22 +++++++------
 .../components/import/common/ImportFilters.js |  5 ++-
 .../components/import/common/ImportLabels.js  | 31 +++++++++++++++++++
 .../import/common/ImportSelectCheckbox.js     | 15 +++------
 .../assets/js/components/main/Import.js       | 10 ++----
 8 files changed, 76 insertions(+), 32 deletions(-)
 create mode 100644 rdmo/management/assets/js/components/common/Labels.js
 create mode 100644 rdmo/management/assets/js/components/import/common/ImportLabels.js

diff --git a/rdmo/management/assets/js/components/common/Labels.js b/rdmo/management/assets/js/components/common/Labels.js
new file mode 100644
index 0000000000..f0d3e3729e
--- /dev/null
+++ b/rdmo/management/assets/js/components/common/Labels.js
@@ -0,0 +1,21 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+
+const Label = ({ text, type, onClick, show = true, className = '' }) => {
+  const labelClass = `label label-${type} ${className}`
+  return show && (
+    <span className={labelClass} onClick={onClick}>
+      {text}
+    </span>
+  )
+}
+
+Label.propTypes = {
+  text: PropTypes.string.isRequired,
+  type: PropTypes.string.isRequired,
+  onClick: PropTypes.func,
+  show: PropTypes.bool,
+  className: PropTypes.string,
+}
+
+export default Label
diff --git a/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js b/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js
index b40021c1a5..8f7fd000e6 100644
--- a/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js
+++ b/rdmo/management/assets/js/components/import/ImportAggregatedErrorsPanel.js
@@ -30,7 +30,7 @@ const ImportAggregatedErrorsPanel = ({ config, elements, configActions }) => {
 
   const errorsHeadingText = <strong onClick={updateShowErrors}>{gettext('Errors')} ({elements.length}) :</strong>
 
-  return (
+  return ( uniqueErrors.length > 0 &&
     <div className="panel panel-danger panel-import-errors mt-10">
       <div className="panel-heading" onClick={updateShowErrors}>
         {errorsHeadingText}
diff --git a/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js b/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
index 4a10ca2241..12d7dc6b58 100644
--- a/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
+++ b/rdmo/management/assets/js/components/import/ImportAggregatedWarningsPanel.js
@@ -28,7 +28,7 @@ const ImportAggregatedWarningsPanel = ({ config, elements, configActions }) => {
 
   const warningsHeadingText = <strong onClick={updateShowWarnings}>{gettext('Warnings')} ({elements.length}):</strong>
 
-  return (
+  return ( aggregatedWarnings.length > 0 &&
     <div className="panel panel-warning panel-import-warnings mt-10">
       <div className="panel-heading" onClick={updateShowWarnings}>
         {warningsHeadingText}
diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index b755733c71..dbcd38955e 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -5,6 +5,9 @@ import { isEmpty } from 'lodash'
 import { codeClass, verboseNames } from '../../constants/elements'
 import Warnings from './common/Warnings'
 import Errors from './common/Errors'
+import {ChangedLabel, CreatedLabel} from './common/ImportLabels'
+
+
 
 const ImportSuccessElement = ({ element, importActions }) => {
 
@@ -15,23 +18,22 @@ const ImportSuccessElement = ({ element, importActions }) => {
       <div className="mb-5">
         <strong>{verboseNames[element.model]}{' '}</strong>
         <code className={codeClass[element.model]}>{element.uri}</code>
-        {element.updated && element.changed && (
-        <span className="label label-info ml-5" onClick={updateShowField}>
-          {gettext('changed')}</span>
-        )}
-        {element.created && (
-          <span className="label label-success ml-5" onClick={updateShowField}>
-            {gettext('created')}</span>
-        )}
-        {!isEmpty(element.errors) && !(element.created || element.updated) && (
+        <CreatedLabel onClick={updateShowField} show={element.created} />
+
+        <ChangedLabel onClick={updateShowField} show={(element.changed && element.updated)} />
+
+        {
+          !isEmpty(element.errors) && !(element.created || element.updated) && (
           <span className="text-danger">{' '}{gettext('could not be imported')}</span>
         )}
-        {!isEmpty(element.errors) && (element.created || element.updated) && (
+        {
+          !isEmpty(element.errors) && (element.created || element.updated) && (
           <>
             {', '}
             <span className="text-danger">{gettext('but could not be added to parent element')}</span>
           </>
         )}
+        {'.'}
       </div>
       <Errors elementErrors={element.errors} />
       <Warnings elementWarnings={element.warnings} elementModel={element.model} showTitle={true} shouldShowURI={false} />
diff --git a/rdmo/management/assets/js/components/import/common/ImportFilters.js b/rdmo/management/assets/js/components/import/common/ImportFilters.js
index 0f9731a7df..cfb7a23e73 100644
--- a/rdmo/management/assets/js/components/import/common/ImportFilters.js
+++ b/rdmo/management/assets/js/components/import/common/ImportFilters.js
@@ -13,7 +13,7 @@ const ImportFilters = ({ config, elements, changedElements, filteredElements, co
   const updateFilterChanged = (value) => configActions.updateConfig('filter.import.elements.changed', value)
   const getValueFilterChanged = () => get(config, 'filter.import.elements.changed', false)
 
-  return (
+  return ( elements.length > 0 &&
     <div className="row">
       <div className={'col-sm-8'}>
         <FilterString value={getValueFilterString()} onChange={updateFilterString}
@@ -36,8 +36,7 @@ const ImportFilters = ({ config, elements, changedElements, filteredElements, co
           </span>
         </div>
       )}
-    </div>
-  )
+  </div>)
 }
 
 ImportFilters.propTypes = {
diff --git a/rdmo/management/assets/js/components/import/common/ImportLabels.js b/rdmo/management/assets/js/components/import/common/ImportLabels.js
new file mode 100644
index 0000000000..283d939d6e
--- /dev/null
+++ b/rdmo/management/assets/js/components/import/common/ImportLabels.js
@@ -0,0 +1,31 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+import Label from 'rdmo/management/assets/js/components/common/Labels'
+
+const ChangedLabel = ({ onClick, show }) => {
+  return <Label text={gettext('changed')}
+                type="info"
+                onClick={onClick}
+                className={'ml-5'}
+                show={show} />
+}
+
+ChangedLabel.propTypes = {
+  onClick: PropTypes.func,
+  show: PropTypes.bool,
+}
+
+const CreatedLabel = ({ onClick, show }) => {
+    return <Label text={gettext('created')}
+                type="success"
+                onClick={onClick}
+                className={'ml-5'}
+                show={show} />
+}
+
+CreatedLabel.propTypes = {
+  onClick: PropTypes.func,
+  show: PropTypes.bool,
+}
+
+export { ChangedLabel, CreatedLabel }
diff --git a/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
index 2bec30f946..4dc037c0c3 100644
--- a/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
+++ b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
@@ -2,6 +2,7 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import { CodeLink } from '../../common/Links'
 import { codeClass, verboseNames } from '../../../constants/elements'
+import {ChangedLabel, CreatedLabel} from './ImportLabels'
 
 const ImportSelectCheckbox = ({ element, toggleImport, updateShowField }) => (
   <div className="checkbox">
@@ -10,16 +11,10 @@ const ImportSelectCheckbox = ({ element, toggleImport, updateShowField }) => (
       <strong>{verboseNames[element.model]}{' '}</strong>
     </label>
     <CodeLink className={codeClass[element.model]} uri={element.uri} onClick={updateShowField} />
-   {
-      (element.changed && element.updated) &&
-        <span className="label label-info ml-5" onClick={updateShowField}>
-          {gettext('changed')}</span>
-    }
-    {
-      element.created &&
-        <span className="label label-success ml-5" onClick={updateShowField}>
-          {gettext('created')}</span>
-    }
+
+    <ChangedLabel onClick={updateShowField} show={(element.changed && element.updated)} />
+
+    <CreatedLabel onClick={updateShowField} show={element.created} />
   </div>
 )
 
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index cd9ff46f48..bd600cfc49 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -40,24 +40,20 @@ const Import = ({ config, imports, configActions, importActions }) => {
       </div>
       <div className="panel-body">
         {
-          elements.length > 0 &&
           <ImportFilters config={config} elements={elements}
                          changedElements={changedElements}
                          filteredElements={filteredElements}
-                         configActions={configActions}
-          />
+                         configActions={configActions} />
         }
         {
-          importWarnings.length > 0 &&
           <ImportAggregatedWarningsPanel config={config} elements={importWarnings}
                                          importActions={importActions}
-                                         configActions={configActions}/>
+                                         configActions={configActions} />
         }
         {
-          importErrors.length > 0 &&
           <ImportAggregatedErrorsPanel config={config} elements={importErrors}
                                        importActions={importActions}
-                                       configActions={configActions}/>
+                                       configActions={configActions} />
         }
       </div>
         <ul className='list-group'>

From 2766b0bbb2da0895f1d252c9e70a130ee7a51143 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 1 Jul 2024 18:42:40 +0200
Subject: [PATCH 197/205] feat(js,import): fix updating key for attributes at
 import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../components/import/common/FieldRowValue.js |  5 ++---
 .../assets/js/reducers/importsReducer.js      |  5 ++++-
 rdmo/management/assets/js/utils/elements.js   | 22 ++++++++++++++++++-
 3 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/FieldRowValue.js b/rdmo/management/assets/js/components/import/common/FieldRowValue.js
index 22cedefc3c..7d633fd153 100644
--- a/rdmo/management/assets/js/components/import/common/FieldRowValue.js
+++ b/rdmo/management/assets/js/components/import/common/FieldRowValue.js
@@ -9,7 +9,6 @@ import {codeClass} from '../../../constants/elements'
 
 
 const FieldRowValue = ({ value }) => {
-
   return  (
     <div className="col-sm-12">
       {
@@ -24,8 +23,8 @@ const FieldRowValue = ({ value }) => {
       )
       }
       {
-        isPlainObject(value) && !isUndefined(value.uri) && !isUndefined(value.model) &&
-        <code className={codeClass[value.model]}>{value.uri}</code>
+        isPlainObject(value) && !isUndefined(value.uri) &&
+        <code className={codeClass[value.model || 'domain.attribute']}>{value.uri}</code>
       }
       {
         isString(value) &&
diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index ffb2621db6..e6a0f32b1b 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -2,7 +2,7 @@ import isArray from 'lodash/isArray'
 import isNil from 'lodash/isNil'
 import isUndefined from 'lodash/isUndefined'
 
-import { buildUri } from '../utils/elements'
+import { buildUri, buildPath } from '../utils/elements'
 import processElementDiffs from '../utils/processElementDiffs'
 
 
@@ -48,6 +48,9 @@ export default function importsReducer(state = initialState, action) {
       if (index > -1) {
         const elements = [...state.elements]
         elements[index] = {...elements[index], ...action.values}
+        if (elements[index].model === 'domain.attribute') {
+          elements[index].path = buildPath(elements[index].key, elements[index].parent ? elements[index].parent.uri : null, elements)
+        }
         elements[index].uri = buildUri(elements[index])
         return {...state, elements}
       } else {
diff --git a/rdmo/management/assets/js/utils/elements.js b/rdmo/management/assets/js/utils/elements.js
index 97ca6434ed..f56f86df04 100644
--- a/rdmo/management/assets/js/utils/elements.js
+++ b/rdmo/management/assets/js/utils/elements.js
@@ -174,4 +174,24 @@ const buildUri = (element) => {
   return uri
 }
 
-export { compareElements, updateElement, resetElement, canMoveElement, moveElement, findDescendants, buildUri }
+const buildPath = (key, parentUri, elements) => {
+
+  let path = key
+  if (parentUri) {
+    const parentElement = elements.find(element => element.uri === parentUri)
+    if (parentElement) {
+      const parentPath = parentElement.path
+      path = parentPath ? `${parentPath}/${key}` : key
+    } else if (parentUri.includes('/domain/')) {
+      // If parentElement is not found, construct the path using parentUri directly
+      const parentPath = parentUri.split('/domain/')[1]
+      path = parentPath ? `${parentPath}/${key}` : key
+    }
+  }
+
+  return path
+}
+
+
+
+export { compareElements, updateElement, resetElement, canMoveElement, moveElement, findDescendants, buildUri, buildPath }

From f2418784d2efe883c5b48f1663bd463c7def81bb Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 1 Jul 2024 18:49:39 +0200
Subject: [PATCH 198/205] fix(domain,import): use value from helper when needed

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/core/imports.py   | 15 +++++++++------
 rdmo/domain/imports.py |  4 ++--
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/rdmo/core/imports.py b/rdmo/core/imports.py
index 39816a28bc..2a2a165e40 100644
--- a/rdmo/core/imports.py
+++ b/rdmo/core/imports.py
@@ -282,20 +282,23 @@ def set_extra_field(instance, field_name, element,
 
     extra_value = None
     if field_name in element:
-        element_value = element.get(field_name)
-        extra_value = element_value
+        extra_value = element.get(field_name)
     else:
+        # get the default field value from the instance
         instance_value = getattr(instance, field_name)
         element[field_name] = instance_value
         extra_value = instance_value
 
-    if extra_value is None and extra_field_helper is not None:
+    if extra_field_helper is not None:
         # default_value
-        extra_value = extra_field_helper.get_value(instance=instance,
+        extra_value_from_helper = extra_field_helper.get_value(instance=instance,
                                                    key=field_name)
+        # overwrite None or '' values by the get_value from the helper
+        if extra_value is None or extra_value == '':
+            extra_value = extra_value_from_helper
 
-    if extra_field_helper.overwrite_in_element:
-        element[field_name] = extra_value
+        if extra_field_helper.overwrite_in_element:
+            element[field_name] = extra_value
 
     if extra_value is not None:
         setattr(instance, field_name, extra_value)
diff --git a/rdmo/domain/imports.py b/rdmo/domain/imports.py
index ef9a5daca3..cf4aefed15 100644
--- a/rdmo/domain/imports.py
+++ b/rdmo/domain/imports.py
@@ -24,8 +24,8 @@ def build_attribute_uri(instance: Optional[Attribute]=None):
     common_fields=('uri_prefix', 'key', 'comment'),
     validators=(AttributeLockedValidator, AttributeParentValidator, AttributeUniqueURIValidator),
     foreign_fields=('parent',),
-    extra_fields=[
+    extra_fields=(
         ExtraFieldHelper(field_name='path', callback=build_attribute_path, overwrite_in_element=True),
         ExtraFieldHelper(field_name='uri', callback=build_attribute_uri, overwrite_in_element=True),
-    ]
+    )
 )

From b5a035361f7402953285d521d583cd8b5006ab79 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 3 Jul 2024 11:05:13 +0200
Subject: [PATCH 199/205] feat(js, import): change label texts during import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../components/import/ImportSuccessElement.js | 20 +++++++++++--------
 .../components/import/common/ImportFilters.js | 14 +++++++++----
 .../components/import/common/ImportLabels.js  | 10 ++++++----
 .../import/common/ImportSelectCheckbox.js     | 13 +++++++-----
 .../assets/js/components/main/Import.js       |  3 ++-
 5 files changed, 38 insertions(+), 22 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index dbcd38955e..ed4f3c38d1 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -12,15 +12,18 @@ import {ChangedLabel, CreatedLabel} from './common/ImportLabels'
 const ImportSuccessElement = ({ element, importActions }) => {
 
   const updateShowField = () => importActions.updateElement(element, { show: !element.show })
+  const changedLabelText = gettext('changed')
+  const createdLabelText = gettext('created')
 
   return (
     <li className="list-group-item">
       <div className="mb-5">
         <strong>{verboseNames[element.model]}{' '}</strong>
         <code className={codeClass[element.model]}>{element.uri}</code>
-        <CreatedLabel onClick={updateShowField} show={element.created} />
 
-        <ChangedLabel onClick={updateShowField} show={(element.changed && element.updated)} />
+        <ChangedLabel text={changedLabelText} onClick={updateShowField} show={(element.changed && element.updated)} />
+
+        <CreatedLabel  text={createdLabelText} onClick={updateShowField} show={element.created} />
 
         {
           !isEmpty(element.errors) && !(element.created || element.updated) && (
@@ -28,12 +31,13 @@ const ImportSuccessElement = ({ element, importActions }) => {
         )}
         {
           !isEmpty(element.errors) && (element.created || element.updated) && (
-          <>
-            {', '}
-            <span className="text-danger">{gettext('but could not be added to parent element')}</span>
-          </>
-        )}
-        {'.'}
+            <>
+              {', '}
+              <span className="text-danger">{gettext('but could not be added to parent element')}</span>
+              {'.'}
+            </>
+          )}
+
       </div>
       <Errors elementErrors={element.errors} />
       <Warnings elementWarnings={element.warnings} elementModel={element.model} showTitle={true} shouldShowURI={false} />
diff --git a/rdmo/management/assets/js/components/import/common/ImportFilters.js b/rdmo/management/assets/js/components/import/common/ImportFilters.js
index cfb7a23e73..d368e7e44b 100644
--- a/rdmo/management/assets/js/components/import/common/ImportFilters.js
+++ b/rdmo/management/assets/js/components/import/common/ImportFilters.js
@@ -5,14 +5,19 @@ import get from 'lodash/get'
 import {getUriPrefixes} from '../../../utils/filter'
 import {Checkbox} from '../../common/Checkboxes'
 
-const ImportFilters = ({ config, elements, changedElements, filteredElements, configActions }) => {
+const ImportFilters = ({ config, elements, changedElements, filteredElements, configActions, success= false}) => {
   const updateFilterString = (value) => configActions.updateConfig('filter.import.elements.search', value)
   const getValueFilterString = () => get(config, 'filter.import.elements.search', '')
   const updateFilterUriPrefix = (value) => configActions.updateConfig('filter.import.elements.uri_prefix', value)
-  const getValueFilterUirPrefix = () => get(config, 'filter.import.elements.uri_prefix', '')
+  const getValueFilterUriPrefix = () => get(config, 'filter.import.elements.uri_prefix', '')
   const updateFilterChanged = (value) => configActions.updateConfig('filter.import.elements.changed', value)
   const getValueFilterChanged = () => get(config, 'filter.import.elements.changed', false)
 
+  const filterCheckBoxText = interpolate(
+    success ? gettext('Show only created and changed (%s)') : gettext('Show only creations and changes (%s)'),
+    [changedElements.length]
+  )
+
   return ( elements.length > 0 &&
     <div className="row">
       <div className={'col-sm-8'}>
@@ -20,7 +25,7 @@ const ImportFilters = ({ config, elements, changedElements, filteredElements, co
                       placeholder={gettext('Filter uri')}/>
       </div>
       <div className="col-sm-4">
-        <FilterUriPrefix value={getValueFilterUirPrefix()}
+        <FilterUriPrefix value={getValueFilterUriPrefix()}
                          onChange={updateFilterUriPrefix}
                          options={getUriPrefixes(elements)}/>
       </div>
@@ -28,7 +33,7 @@ const ImportFilters = ({ config, elements, changedElements, filteredElements, co
       {elements.length > 0 && (
         <div className="horizontal-container">
           <div className="checkboxes">
-            <Checkbox label={interpolate(gettext('Show only created and changed (%s)'), [changedElements.length])}
+            <Checkbox label={filterCheckBoxText}
                       value={getValueFilterChanged()} onChange={updateFilterChanged} />
           </div>
           <span className="shown-info">
@@ -45,6 +50,7 @@ ImportFilters.propTypes = {
   changedElements: PropTypes.array.isRequired,
   filteredElements: PropTypes.array.isRequired,
   configActions: PropTypes.object.isRequired,
+  success: PropTypes.bool
 }
 
 export default ImportFilters
diff --git a/rdmo/management/assets/js/components/import/common/ImportLabels.js b/rdmo/management/assets/js/components/import/common/ImportLabels.js
index 283d939d6e..b44fb7b3f1 100644
--- a/rdmo/management/assets/js/components/import/common/ImportLabels.js
+++ b/rdmo/management/assets/js/components/import/common/ImportLabels.js
@@ -2,8 +2,8 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import Label from 'rdmo/management/assets/js/components/common/Labels'
 
-const ChangedLabel = ({ onClick, show }) => {
-  return <Label text={gettext('changed')}
+const ChangedLabel = ({ text, onClick, show }) => {
+  return <Label text={text}
                 type="info"
                 onClick={onClick}
                 className={'ml-5'}
@@ -11,12 +11,13 @@ const ChangedLabel = ({ onClick, show }) => {
 }
 
 ChangedLabel.propTypes = {
+  text: PropTypes.string.isRequired,
   onClick: PropTypes.func,
   show: PropTypes.bool,
 }
 
-const CreatedLabel = ({ onClick, show }) => {
-    return <Label text={gettext('created')}
+const CreatedLabel = ({ text, onClick, show }) => {
+    return <Label text={text}
                 type="success"
                 onClick={onClick}
                 className={'ml-5'}
@@ -24,6 +25,7 @@ const CreatedLabel = ({ onClick, show }) => {
 }
 
 CreatedLabel.propTypes = {
+  text: PropTypes.string.isRequired,
   onClick: PropTypes.func,
   show: PropTypes.bool,
 }
diff --git a/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
index 4dc037c0c3..d64397ecbe 100644
--- a/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
+++ b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
@@ -4,19 +4,22 @@ import { CodeLink } from '../../common/Links'
 import { codeClass, verboseNames } from '../../../constants/elements'
 import {ChangedLabel, CreatedLabel} from './ImportLabels'
 
-const ImportSelectCheckbox = ({ element, toggleImport, updateShowField }) => (
-  <div className="checkbox">
+const ImportSelectCheckbox = ({ element, toggleImport, updateShowField }) => {
+  const changedLabelText = gettext('change')
+  const createdLabelText = gettext('create')
+  return (
+    <div className="checkbox">
     <label className="mr-5">
       <input type="checkbox" checked={element.import} onChange={toggleImport} />
       <strong>{verboseNames[element.model]}{' '}</strong>
     </label>
     <CodeLink className={codeClass[element.model]} uri={element.uri} onClick={updateShowField} />
 
-    <ChangedLabel onClick={updateShowField} show={(element.changed && element.updated)} />
+    <ChangedLabel text={changedLabelText} onClick={updateShowField} show={(element.changed && element.updated)} />
 
-    <CreatedLabel onClick={updateShowField} show={element.created} />
+    <CreatedLabel text={createdLabelText} onClick={updateShowField} show={element.created} />
   </div>
-)
+)}
 
 ImportSelectCheckbox.propTypes = {
   element: PropTypes.object.isRequired,
diff --git a/rdmo/management/assets/js/components/main/Import.js b/rdmo/management/assets/js/components/main/Import.js
index bd600cfc49..8f27c8d35d 100644
--- a/rdmo/management/assets/js/components/main/Import.js
+++ b/rdmo/management/assets/js/components/main/Import.js
@@ -43,7 +43,8 @@ const Import = ({ config, imports, configActions, importActions }) => {
           <ImportFilters config={config} elements={elements}
                          changedElements={changedElements}
                          filteredElements={filteredElements}
-                         configActions={configActions} />
+                         configActions={configActions}
+                         success={success} />
         }
         {
           <ImportAggregatedWarningsPanel config={config} elements={importWarnings}

From 550aeb51d1df0d7985f7346b9cb211717cb44878 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 3 Jul 2024 11:15:51 +0200
Subject: [PATCH 200/205] refactor(js, import): rename to buildPathForAttribute
 and remove extras

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../management/assets/js/reducers/importsReducer.js |  4 ++--
 rdmo/management/assets/js/utils/elements.js         | 13 ++++---------
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index e6a0f32b1b..775cb28d97 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -2,7 +2,7 @@ import isArray from 'lodash/isArray'
 import isNil from 'lodash/isNil'
 import isUndefined from 'lodash/isUndefined'
 
-import { buildUri, buildPath } from '../utils/elements'
+import { buildUri, buildPathForAttribute } from '../utils/elements'
 import processElementDiffs from '../utils/processElementDiffs'
 
 
@@ -49,7 +49,7 @@ export default function importsReducer(state = initialState, action) {
         const elements = [...state.elements]
         elements[index] = {...elements[index], ...action.values}
         if (elements[index].model === 'domain.attribute') {
-          elements[index].path = buildPath(elements[index].key, elements[index].parent ? elements[index].parent.uri : null, elements)
+          elements[index].path = buildPathForAttribute(elements[index].key, elements[index].parent ? elements[index].parent.uri : null)
         }
         elements[index].uri = buildUri(elements[index])
         return {...state, elements}
diff --git a/rdmo/management/assets/js/utils/elements.js b/rdmo/management/assets/js/utils/elements.js
index f56f86df04..5abdaa934a 100644
--- a/rdmo/management/assets/js/utils/elements.js
+++ b/rdmo/management/assets/js/utils/elements.js
@@ -174,16 +174,11 @@ const buildUri = (element) => {
   return uri
 }
 
-const buildPath = (key, parentUri, elements) => {
-
+const buildPathForAttribute = (key, parentUri) => {
   let path = key
   if (parentUri) {
-    const parentElement = elements.find(element => element.uri === parentUri)
-    if (parentElement) {
-      const parentPath = parentElement.path
-      path = parentPath ? `${parentPath}/${key}` : key
-    } else if (parentUri.includes('/domain/')) {
-      // If parentElement is not found, construct the path using parentUri directly
+     if (parentUri.includes('/domain/')) {
+      // construct the path using parentUri directly
       const parentPath = parentUri.split('/domain/')[1]
       path = parentPath ? `${parentPath}/${key}` : key
     }
@@ -194,4 +189,4 @@ const buildPath = (key, parentUri, elements) => {
 
 
 
-export { compareElements, updateElement, resetElement, canMoveElement, moveElement, findDescendants, buildUri, buildPath }
+export { compareElements, updateElement, resetElement, canMoveElement, moveElement, findDescendants, buildUri, buildPathForAttribute }

From fc431e57350ca26d487f2c23a2eac61e9e933048 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 3 Jul 2024 12:41:06 +0200
Subject: [PATCH 201/205] feat(js, import): change create to new

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../assets/js/components/import/common/ImportFilters.js       | 2 +-
 .../js/components/import/common/ImportSelectCheckbox.js       | 2 +-
 rdmo/management/tests/test_frontend_import_options.py         | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/ImportFilters.js b/rdmo/management/assets/js/components/import/common/ImportFilters.js
index d368e7e44b..fa556d5ddd 100644
--- a/rdmo/management/assets/js/components/import/common/ImportFilters.js
+++ b/rdmo/management/assets/js/components/import/common/ImportFilters.js
@@ -14,7 +14,7 @@ const ImportFilters = ({ config, elements, changedElements, filteredElements, co
   const getValueFilterChanged = () => get(config, 'filter.import.elements.changed', false)
 
   const filterCheckBoxText = interpolate(
-    success ? gettext('Show only created and changed (%s)') : gettext('Show only creations and changes (%s)'),
+    success ? gettext('Show only created and changed (%s)') : gettext('Show only new and changes (%s)'),
     [changedElements.length]
   )
 
diff --git a/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
index d64397ecbe..34e2cc029c 100644
--- a/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
+++ b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
@@ -6,7 +6,7 @@ import {ChangedLabel, CreatedLabel} from './ImportLabels'
 
 const ImportSelectCheckbox = ({ element, toggleImport, updateShowField }) => {
   const changedLabelText = gettext('change')
-  const createdLabelText = gettext('create')
+  const createdLabelText = gettext('new')
   return (
     <div className="checkbox">
     <label className="mr-5">
diff --git a/rdmo/management/tests/test_frontend_import_options.py b/rdmo/management/tests/test_frontend_import_options.py
index 3333635de8..14cd03d921 100644
--- a/rdmo/management/tests/test_frontend_import_options.py
+++ b/rdmo/management/tests/test_frontend_import_options.py
@@ -25,6 +25,8 @@
     "warnings": 2
 }
 OPTIONSETS_COUNTS_HEADER_INFOS = [f"{k.capitalize()}: {v}" for k,v in OPTIONSETS_COUNTS.items()]
+# Defined in filterCheckBoxText in rdmo/management/assets/js/components/import/common/ImportFilters.js
+IMPORT_FILTER_LABEL_TEXT = "Show only new and changes"
 
 @pytest.mark.parametrize("username, password", test_users)  # consumed by fixture
 def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> None:
@@ -71,7 +73,7 @@ def test_import_and_update_optionsets_in_management(logged_in_user: Page) -> Non
     # assert changed elements
     for text in OPTIONSETS_COUNTS_HEADER_INFOS:
         expect(page.locator("#main")).to_contain_text(text)
-    expect(page.get_by_text(f"Show only created and changed ({OPTIONSETS_COUNTS['changed']})")).to_be_visible()
+    expect(page.get_by_text(f"{IMPORT_FILTER_LABEL_TEXT} ({OPTIONSETS_COUNTS['changed']})")).to_be_visible()
     page.get_by_role("link", name="Show changes").click()
     expect(page.locator(".col-sm-6 > .form-group").first).to_be_visible(timeout=30_000)
     # take a screenshot of the import page

From 7649ef81f2d165fbcfa0888a1e55290c530eb2a8 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Wed, 3 Jul 2024 12:41:50 +0200
Subject: [PATCH 202/205] feat(import): add short_title to section and page

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/questions/imports.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rdmo/questions/imports.py b/rdmo/questions/imports.py
index 973a37f598..93f9eeff4b 100644
--- a/rdmo/questions/imports.py
+++ b/rdmo/questions/imports.py
@@ -36,7 +36,7 @@
 import_helper_section = ElementImportHelper(
     model = Section,
     validators=(SectionLockedValidator, SectionUniqueURIValidator),
-    lang_fields=('title',),
+    lang_fields=('title', 'short_title'),
     m2m_through_instance_fields=[
         ThroughInstanceMapper(
             field_name='pages', source_name='section',
@@ -54,7 +54,7 @@
 import_helper_page = ElementImportHelper(
     model = Page,
     validators=(PageLockedValidator, PageUniqueURIValidator),
-    lang_fields=('help', 'title', 'verbose_name'),
+    lang_fields=('help', 'title', 'verbose_name', 'short_title'),
     foreign_fields=('attribute',),
     extra_fields = (
         ExtraFieldHelper(field_name='is_collection'),

From f48791f88cfb30af79d4e1e4afdaa002e6433a57 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 8 Jul 2024 09:53:19 +0200
Subject: [PATCH 203/205] refactor(import): update created and updated, return
 False when error at save

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 rdmo/management/import_utils.py |  1 -
 rdmo/management/imports.py      | 14 +++++++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/rdmo/management/import_utils.py b/rdmo/management/import_utils.py
index bb792361af..92ec4cfa5e 100644
--- a/rdmo/management/import_utils.py
+++ b/rdmo/management/import_utils.py
@@ -37,7 +37,6 @@ def strip_uri_prefix_endswith_slash(element: dict) -> dict:
 
 def apply_field_values(instance, element, import_helper, uploaded_uris, original) -> None:
     """Applies the field values from the element to the instance."""
-    element = strip_uri_prefix_endswith_slash(element)
     # start to set values on the instance
     # set common field values from element on instance
     for field in import_helper.common_fields:
diff --git a/rdmo/management/imports.py b/rdmo/management/imports.py
index c0305cc57b..10679d100a 100644
--- a/rdmo/management/imports.py
+++ b/rdmo/management/imports.py
@@ -112,9 +112,8 @@ def import_element(
         element["errors"].append(perms_error_msg)
         return element
 
-    updated = not created
     element['created'] = created
-    element['updated'] = updated
+    element['updated'] = not created and original is not None
     # INFO: the dict element[FieldNames.diff.value] is filled by calling track_changes_on_element
 
     element = strip_uri_prefix_endswith_slash(element)
@@ -126,16 +125,21 @@ def import_element(
 
     if element.get('errors'):
         # when there is an error msg, the import can be stopped and return
+        if save:
+            element['created'] = False
+            element['updated'] = False
         return element
 
     if save:
         logger.info(msg)
         instance.save()
 
-    if save or updated:
         update_related_fields(instance, element, import_helper, original, save)
 
-    if save and settings.MULTISITE:
-        add_current_site_to_sites_and_editor(instance, current_site, import_helper)
+        if created and settings.MULTISITE:
+            add_current_site_to_sites_and_editor(instance, current_site, import_helper)
+
+    elif not created:  # when an element will be updated but not saved
+        update_related_fields(instance, element, import_helper, original, save)
 
     return element

From 7817279c7f9550242c06fab503da1a1263750e00 Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Mon, 8 Jul 2024 10:43:08 +0200
Subject: [PATCH 204/205] feat(js,import): remove onClick from label text and
 add return null for builUri

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../components/import/common/ImportSelectCheckbox.js  |  4 ++--
 rdmo/management/assets/js/reducers/importsReducer.js  | 10 ++++++++--
 rdmo/management/assets/js/utils/elements.js           | 11 +++++++++--
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
index 34e2cc029c..365ea0eedc 100644
--- a/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
+++ b/rdmo/management/assets/js/components/import/common/ImportSelectCheckbox.js
@@ -15,9 +15,9 @@ const ImportSelectCheckbox = ({ element, toggleImport, updateShowField }) => {
     </label>
     <CodeLink className={codeClass[element.model]} uri={element.uri} onClick={updateShowField} />
 
-    <ChangedLabel text={changedLabelText} onClick={updateShowField} show={(element.changed && element.updated)} />
+    <ChangedLabel text={changedLabelText} show={(element.changed && element.updated)} />
 
-    <CreatedLabel text={createdLabelText} onClick={updateShowField} show={element.created} />
+    <CreatedLabel text={createdLabelText} show={element.created} />
   </div>
 )}
 
diff --git a/rdmo/management/assets/js/reducers/importsReducer.js b/rdmo/management/assets/js/reducers/importsReducer.js
index 775cb28d97..28810fddaf 100644
--- a/rdmo/management/assets/js/reducers/importsReducer.js
+++ b/rdmo/management/assets/js/reducers/importsReducer.js
@@ -51,7 +51,10 @@ export default function importsReducer(state = initialState, action) {
         if (elements[index].model === 'domain.attribute') {
           elements[index].path = buildPathForAttribute(elements[index].key, elements[index].parent ? elements[index].parent.uri : null)
         }
-        elements[index].uri = buildUri(elements[index])
+        const newUri = buildUri(elements[index])
+        if (!isNil(newUri)) {
+          elements[index].uri = newUri
+        }
         return {...state, elements}
       } else {
         return state
@@ -87,7 +90,10 @@ export default function importsReducer(state = initialState, action) {
         element.uri_prefix = action.uriPrefix
 
         // compute a new uri and store it in the elementMap
-        element.uri = elementsMap[element.uri] = buildUri(element)
+        const newUri = buildUri(element)
+        if (!isNil(newUri)) {
+          element.uri = elementsMap[element.uri] = newUri
+        }
 
         return element
       })
diff --git a/rdmo/management/assets/js/utils/elements.js b/rdmo/management/assets/js/utils/elements.js
index 5abdaa934a..f9be3aa889 100644
--- a/rdmo/management/assets/js/utils/elements.js
+++ b/rdmo/management/assets/js/utils/elements.js
@@ -162,13 +162,20 @@ function findDescendants(element, elementType) {
 }
 
 const buildUri = (element) => {
-  let uri = element.uri_prefix + '/' + elementModules[element.model] + '/'
+  if (isUndefined(element.uri_prefix) || isUndefined(element.model)) {
+    return null
+  }
+
+  let uri = `${element.uri_prefix}/${elementModules[element.model]}/`
+
   if (!isUndefined(element.uri_path) && !isNil(element.uri_path)) {
     uri += element.uri_path
   } else if (!isUndefined(element.path) && !isNil(element.path)) {
     uri += element.path
-  } else {
+  } else if (!isUndefined(element.key) && !isNil(element.key)) {
     uri += element.key
+  } else {
+    return null
   }
 
   return uri

From 374fac8f778f9ed90bc96cafafb9173f88972fce Mon Sep 17 00:00:00 2001
From: David Wallace <david.wallace@tu-darmstadt.de>
Date: Thu, 11 Jul 2024 17:43:39 +0200
Subject: [PATCH 205/205] feat(js,import): fix end-of-message dot at import

Signed-off-by: David Wallace <david.wallace@tu-darmstadt.de>
---
 .../components/import/ImportSuccessElement.js | 21 ++++++++-----------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/rdmo/management/assets/js/components/import/ImportSuccessElement.js b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
index ed4f3c38d1..4e3e915704 100644
--- a/rdmo/management/assets/js/components/import/ImportSuccessElement.js
+++ b/rdmo/management/assets/js/components/import/ImportSuccessElement.js
@@ -25,18 +25,15 @@ const ImportSuccessElement = ({ element, importActions }) => {
 
         <CreatedLabel  text={createdLabelText} onClick={updateShowField} show={element.created} />
 
-        {
-          !isEmpty(element.errors) && !(element.created || element.updated) && (
-          <span className="text-danger">{' '}{gettext('could not be imported')}</span>
-        )}
-        {
-          !isEmpty(element.errors) && (element.created || element.updated) && (
-            <>
-              {', '}
-              <span className="text-danger">{gettext('but could not be added to parent element')}</span>
-              {'.'}
-            </>
-          )}
+       {
+        !isEmpty(element.errors) && (
+          <span className="text-danger">
+            {' '}{gettext('could not be imported')}
+            {(element.created || element.updated) && `, ${gettext('but could not be added to parent element')}`}
+            {'.'}
+          </span>
+        )
+      }
 
       </div>
       <Errors elementErrors={element.errors} />