Merge pull request #230 from qld-gov-au/QOLSVC-10243-funnelback-robots

[QOLSVC-10243] exclude dataset search results from bot indexing

Author: ThrawnCA

ckan/templates/base.html

@@ -26,6 +26,10 @@
       <meta charset="utf-8" />
       <meta name="csrf_field_name" content="{{ g.csrf_field_name }}" />
       <meta name="{{ g.csrf_field_name }}" content="{{ csrf_token() }}" />
+      {# Search result pages shouldn't be indexed by bots. #}
+      {% if has_search_args %}
+        <meta name="robots" content="noindex, follow" />
+      {% endif %}
 
       {% block meta_generator %}<meta name="generator" content="ckan {{ h.ckan_version() }}" />{% endblock %}
       {% block meta_viewport %}<meta name="viewport" content="width=device-width, initial-scale=1.0">{% endblock %}

ckan/views/dataset.py

@@ -213,6 +213,7 @@ def _get_search_details() -> dict[str, Any]:
 def search(package_type: str) -> str:
     extra_vars: dict[str, Any] = {}
 
+    extra_vars['has_search_args'] = True if request.args else False
     extra_vars['q'] = q = request.args.get('q', '')
 
     extra_vars['query_error'] = False

ckan/views/group.py

@@ -130,6 +130,7 @@ def index(group_type: str, is_organization: bool) -> str:
 
     extra_vars["q"] = q
     extra_vars["sort_by_selected"] = sort_by
+    extra_vars['has_search_args'] = True if request.args else False
 
     # pass user info to context as needed to view private datasets of
     # orgs correctly
@@ -388,7 +389,7 @@ def _get_group_dict(id: str, is_organization: bool) -> dict[str, Any]:
 def read(group_type: str,
          is_organization: bool,
          id: Optional[str] = None) -> Union[str, Response]:
-    extra_vars = {}
+    extra_vars: dict[str, Any] = {u'has_search_args': True if request.args else False}
     context: Context = {
         u'user': current_user.name,
         u'schema': _db_to_form_schema(group_type=group_type),