Merge pull request #233 from qld-gov-au/QOLDEV-1221-update-to-2.11.4

[QOLDEV-1221] apply QGOV changes on top of 2.11.4 base

Author: ThrawnCA

.circleci/config.yml

@@ -24,7 +24,7 @@ defaults:
       CKAN_POSTGRES_USER: ckan_default
       CKAN_POSTGRES_PWD: pass
       PGPASSWORD: ckan
-      PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=~/junit/result/junit.xml
+      PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-ci.ini --cov=ckan --cov=ckanext --junitxml=~/junit/result/junit.xml
       # report usage of deprecated features
       SQLALCHEMY_WARN_20: 1
   pg_image: &pg_image

.github/workflows/docs.yml

@@ -31,3 +31,10 @@ jobs:
       - name: Create Docs
         run: |
            sphinx-build doc build/sphinx
+
+      #as pytest runs test_building_the_docs, we might as well have an output
+      - name: Store docs
+        uses: actions/upload-artifact@v4
+        with:
+          name: ckan-docs-${{ github.sha }}
+          path: ./build/sphinx

CHANGELOG.rst

@@ -9,6 +9,35 @@ Changelog
 
 .. towncrier release notes start
 
+v.2.11.x 2025-xx-xx
+==================
+
+Major features
+--------------
+
+- Update to Interface IUploader, on get_uploader and get_resource_uploader,
+  new to include new method signature metadata() which can be utilised by
+  archiver and other plugins instead of trying on local disk directly
+- Add get API `resource_file_metadata_show` which takes resource ID and returns
+  { 'content_type': content_type, 'size': length, 'hash': hash } if found
+- Show "Displayed name" instead of "Full name"
+- Allow sysadmins to see user emails on profile pages
+- Allow the existence of deleted datasets to be hidden from unauthorised access
+  even if private datasets are revealed
+- Delete file on disk when resource is deleted
+- Provide 'Add resource' shortcut on dataset pages alongside 'Manage'
+
+Bugfixes
+--------
+
+- Convert dict values into strings before passing to Solr
+- Add '/dataset' fallback when generating URLs
+- Avoid changing package modified timestamp when reordering resources
+- Shorten the lock timeout on dropping datastore tables to avoid deadlocks
+- Add 'en_AU' locale to DataTables
+- Instruct bots not to index search result pages
+
+
 v.2.11.4 2025-10-29
 ===================
 
@@ -1494,8 +1523,8 @@ Migration notes
   in the server, all users will be logged out of the site**.
   This can happen for instance:
 
-	* if the CKAN container is redeployed in a Docker / cloud setup and the session directory is not persisted
-	* if the sessions are periodically cleaned by an external script
+    * if the CKAN container is redeployed in a Docker / cloud setup and the session directory is not persisted
+    * if the sessions are periodically cleaned by an external script
 
   Here's a summary of the behaviour changes between CKAN versions:
 
@@ -1518,13 +1547,13 @@ Migration notes
   (note that this stores *all* session data, not just the user identifier). This will probably
   be the default behaviour in future CKAN versions::
 
-	# ckan.ini
-	beaker.session.type = cookie
-	beaker.session.validate_key = CHANGE_ME
+    # ckan.ini
+    beaker.session.type = cookie
+    beaker.session.validate_key = CHANGE_ME
 
-	beaker.session.httponly = True
-	beaker.session.secure = True
-	beaker.session.samesite = Lax # or Strict
+    beaker.session.httponly = True
+    beaker.session.secure = True
+    beaker.session.samesite = Lax # or Strict
 
   Alternatively you can configure another persistent backend for the sessions in the server,
   like an SQL Database or Redis (see the `Beaker configuration <https://beaker.readthedocs.io/en/latest/configuration.html>`_
@@ -1771,14 +1800,14 @@ Migration notes
   store the session data in the `client-side cookie <https://beaker.readthedocs.io/en/latest/sessions.html#cookie-based>`_.
   This will probably be the default behaviour in future CKAN versions::
 
-	# ckan.ini
-	beaker.session.type = cookie
+    # ckan.ini
+    beaker.session.type = cookie
     beaker.session.data_serializer = json
-	beaker.session.validate_key = CHANGE_ME
+    beaker.session.validate_key = CHANGE_ME
 
-	beaker.session.httponly = True
-	beaker.session.secure = True
-	beaker.session.samesite = Lax
+    beaker.session.httponly = True
+    beaker.session.secure = True
+    beaker.session.samesite = Lax
     # or Strict, depending on your setup
 
 v.2.9.8 2023-02-15

ckan/config/config_declaration.yaml

@@ -619,6 +619,19 @@ groups:
           If False, all unauthorised requests will receive HTTP 404 Not Found.
           Default is False.
 
+      - key: ckan.auth.reveal_deleted_datasets
+        type: bool
+        default: True
+        description: |
+          Determines whether unauthorised requests for deleted datasets should have
+          the existence of the datasets revealed (True) or hidden (False).
+          This behaves similarly to ckan.auth.reveal_private_datasets; True gives
+          either HTTP 403 Forbidden (if logged in) or a redirect to the login page,
+          while False gives HTTP 404 Not Found. However, it will only take effect
+          if ckan.auth.reveal_private_datasets is already True; otherwise,
+          return HTTP 404 on unauthorised requests regardless of this value.
+          Default is True.
+
       - key: ckan.auth.enable_cookie_auth_in_api
         type: bool
         default: True

ckan/i18n/en_AU/LC_MESSAGES/ckan.po

@@ -2,10 +2,10 @@
 # Copyright (C) 2024 ORGANIZATION
 # This file is distributed under the same license as the ckan project.
 # FIRST AUTHOR <EMAIL@ADDRESS>, 2024.
-# 
+#
 # Translators:
 # Adrià Mercader <amercadero@gmail.com>, 2024
-# 
+#
 #, fuzzy
 msgid ""
 msgstr ""
@@ -3132,7 +3132,7 @@ msgstr "Change details"
 
 #: ckan/templates/user/edit_user_form.html:17
 msgid "Full name"
-msgstr "Full name"
+msgstr "Displayed name"
 
 #: ckan/templates/user/edit_user_form.html:17
 msgid "eg. Joe Bloggs"
@@ -3304,7 +3304,7 @@ msgstr "username"
 
 #: ckan/templates/user/new_user_form.html:11
 msgid "Full Name"
-msgstr "Full Name"
+msgstr "Displayed name"
 
 #: ckan/templates/user/new_user_form.html:37
 msgid "Create Account"
@@ -3407,7 +3407,7 @@ msgstr "Open ID"
 
 #: ckan/templates/user/snippets/info.html:84
 msgid "This means only you can see this"
-msgstr "This means only you can see this"
+msgstr "This means only yourself and administrators can see this"
 
 #: ckan/templates/user/snippets/info.html:89
 msgid "Member Since"

ckan/lib/helpers.py

@@ -325,6 +325,71 @@ def _get_auto_flask_context():
 
 @core_helper
 def url_for(*args: Any, **kw: Any) -> str:
+    '''Return the URL for an endpoint given some parameters, defaulting to
+    using 'dataset' if the given custom dataset fails.
+
+    This is a wrapper for :py:func:`flask.url_for`
+    and :py:func:`routes.url_for` that adds some extra features that CKAN
+    needs.
+
+    To build a URL for a Flask view, pass the name of the blueprint and the
+    view function separated by a period ``.``, plus any URL parameters::
+        url_for('api.action', ver=3, logic_function='status_show')
+        # Returns /api/3/action/status_show
+
+    For a fully qualified URL pass the ``_external=True`` parameter. This
+    takes the ``ckan.site_url`` and ``ckan.root_path`` settings into account::
+
+        url_for('api.action', ver=3, logic_function='status_show',
+                _external=True)
+
+        # Returns http://example.com/api/3/action/status_show
+
+    URLs built by Pylons use the Routes syntax::
+
+        url_for(controller='my_ctrl', action='my_action', id='my_dataset')
+        # Returns '/dataset/my_dataset'
+
+    Or, using a named route::
+
+        url_for('dataset.read', id='changed')
+        # Returns '/dataset/changed'
+
+    Use ``qualified=True`` for a fully qualified URL when targeting a Pylons
+    endpoint.
+
+    For backwards compatibility, an effort is made to support the Pylons syntax
+    when building a Flask URL, but this support might be dropped in the future,
+    so calls should be updated.
+    '''
+    try:
+        return base_url_for(*args, **kw)
+    except FlaskRouteBuildError:
+        # If the url failed, try again but replace any custom dataset type in
+        #  use with the default 'dataset'
+        retry_with_default = False
+        # Update args if a custom dataset type was set there
+        if (len(args) and '.' in args[0]
+                and not args[0].startswith('/')
+                and not args[0].startswith('dataset.')):
+            args = args[0].split('.', 1)
+            args = ('dataset.' + args[1],)
+            retry_with_default = True
+
+        # Update kw controller if a custom dataset type was set there
+        if (kw.get('controller')
+                and kw.get('controller') != 'dataset'):
+            kw.update({'controller': 'dataset'})
+            retry_with_default = True
+
+        if retry_with_default:
+            return base_url_for(*args, **kw)
+        else:
+            raise
+
+
+@core_helper
+def base_url_for(*args: Any, **kw: Any) -> str:
     '''Return the URL for an endpoint given some parameters.
 
     This is a wrapper for :py:func:`flask.url_for`

ckan/lib/search/index.py

@@ -236,6 +236,9 @@ def index_package(self,
                 except DateParserError:
                     log.warning('%r: %r value of %r is not a valid date', pkg_dict['id'], key, value)
                     continue
+            if isinstance(value, dict):
+                # Solr 8+ chokes on passing dict values unless doing an atomic update
+                value = json.dumps(value)
             new_dict[key] = value
 
         pkg_dict = new_dict