Retry peeruserimport task on Database or connection errors

.github/workflows/pr_build_kolibri.yml

@@ -55,10 +55,10 @@ jobs:
   apk:
     name: Build APK file
     needs: whl
-    uses: learningequality/kolibri-installer-android/.github/workflows/[email protected].6
+    uses: learningequality/kolibri-installer-android/.github/workflows/[email protected].7
     with:
       tar-file-name: ${{ needs.whl.outputs.tar-file-name }}
-      ref: v0.1.6
+      ref: v0.1.7
   zip:
     name: Build Raspberry Pi Image
     needs: deb

.github/workflows/release_kolibri.yml

@@ -119,11 +119,11 @@ jobs:
   apk:
     name: Build Android APK
     needs: whl
-    uses: learningequality/kolibri-installer-android/.github/workflows/[email protected].6
+    uses: learningequality/kolibri-installer-android/.github/workflows/[email protected].7
     with:
       tar-file-name: ${{ needs.whl.outputs.tar-file-name }}
       release: true
-      ref: v0.1.6
+      ref: v0.1.7
     secrets:
       KOLIBRI_ANDROID_APP_PRODUCTION_KEYSTORE: ${{ secrets.KOLIBRI_ANDROID_APP_PRODUCTION_KEYSTORE }}
       KOLIBRI_ANDROID_APP_PRODUCTION_KEYSTORE_PASSWORD: ${{ secrets.KOLIBRI_ANDROID_APP_PRODUCTION_KEYSTORE_PASSWORD }}
@@ -248,9 +248,9 @@ jobs:
     name: Release Android App
     if: ${{ !github.event.release.prerelease }}
     needs: [apk, block_release_step]
-    uses: learningequality/kolibri-installer-android/.github/workflows/[email protected].6
+    uses: learningequality/kolibri-installer-android/.github/workflows/[email protected].7
     with:
       version-code: ${{ needs.apk.outputs.version-code }}
-      ref: v0.1.6
+      ref: v0.1.7
     secrets:
       KOLIBRI_ANDROID_PLAY_STORE_API_SERVICE_ACCOUNT_JSON: ${{ secrets.KOLIBRI_ANDROID_PLAY_STORE_API_SERVICE_ACCOUNT_JSON }}

kolibri/conftest.py

@@ -7,19 +7,6 @@
 TEMP_KOLIBRI_HOME = "./.pytest_kolibri_home"
 
 
-@pytest.fixture(scope="session")
-def django_db_setup(
-    request,
-    django_db_setup,
-):
-    def dispose_sqlalchemy():
-        from kolibri.core.tasks.main import connection
-
-        connection.dispose()
-
-    request.addfinalizer(dispose_sqlalchemy)
-
-
 @pytest.fixture(scope="session", autouse=True)
 def global_fixture():
     if not os.path.exists(TEMP_KOLIBRI_HOME):

kolibri/core/apps.py

@@ -91,17 +91,18 @@ def activate_pragmas_on_start():
         and not on a per connection basis.
         :return:
         """
-        from django.db import connection
-
-        if connection.vendor == "sqlite":
-            cursor = connection.cursor()
-
-            # http://www.sqlite.org/wal.html
-            # WAL's main advantage allows simultaneous reads
-            # and writes (vs. the default exclusive write lock)
-            # at the cost of a slight penalty to all reads.
-            cursor.execute(START_PRAGMAS)
-            connection.close()
+        from django.db import connections
+
+        for connection in connections.all():
+            if connection.vendor == "sqlite":
+                cursor = connection.cursor()
+
+                # http://www.sqlite.org/wal.html
+                # WAL's main advantage allows simultaneous reads
+                # and writes (vs. the default exclusive write lock)
+                # at the cost of a slight penalty to all reads.
+                cursor.execute(START_PRAGMAS)
+                connection.close()
 
     @staticmethod
     def check_file_storage_settings():

kolibri/core/auth/tasks.py

@@ -6,7 +6,11 @@
 from django.core.files.storage import default_storage
 from django.core.management import call_command
 from django.core.management.base import CommandError
+from django.db.utils import OperationalError
 from django.utils import timezone
+from morango.errors import MorangoError
+from requests.exceptions import ConnectionError
+from requests.exceptions import HTTPError
 from rest_framework import serializers
 from rest_framework.exceptions import AuthenticationFailed
 from rest_framework.exceptions import ValidationError
@@ -21,13 +25,14 @@
 from kolibri.core.auth.models import FacilityUser
 from kolibri.core.auth.utils.sync import find_soud_sync_sessions
 from kolibri.core.auth.utils.sync import validate_and_create_sync_credentials
+from kolibri.core.auth.utils.users import get_remote_user_info
 from kolibri.core.auth.utils.users import get_remote_users_info
 from kolibri.core.device import soud
 from kolibri.core.device.translation import get_device_language
 from kolibri.core.device.translation import get_settings_language
 from kolibri.core.discovery.models import NetworkLocation
 from kolibri.core.discovery.utils.network.client import NetworkClient
-from kolibri.core.discovery.utils.network.errors import NetworkLocationNotFound
+from kolibri.core.discovery.utils.network.errors import NetworkClientError
 from kolibri.core.discovery.utils.network.errors import ResourceGoneError
 from kolibri.core.error_constants import DEVICE_LIMITATIONS
 from kolibri.core.serializers import HexOnlyUUIDField
@@ -43,7 +48,7 @@
 from kolibri.core.tasks.permissions import NotProvisioned
 from kolibri.core.tasks.utils import get_current_job
 from kolibri.core.tasks.validation import JobValidator
-from kolibri.utils.time_utils import naive_utc_datetime
+from kolibri.core.utils.retry import retry
 from kolibri.utils.translation import gettext as _
 
 
@@ -309,6 +314,11 @@ class PeerSyncJobValidator(SyncJobValidator):
         queryset=NetworkLocation.objects.all(), required=False
     )
 
+    @retry(NetworkClientError)
+    def _get_base_url(self, baseurl):
+        self.client = NetworkClient.build_for_address(baseurl)
+        return self.client.base_url
+
     def validate(self, data):
         job_data = super().validate(data)
         if "baseurl" not in data and "device_id" not in data:
@@ -327,8 +337,8 @@ def validate(self, data):
             except NetworkLocation.DoesNotExist:
                 pass
         try:
-            baseurl = NetworkClient.build_for_address(data["baseurl"]).base_url
-        except NetworkLocationNotFound:
+            baseurl = self._get_base_url(data["baseurl"])
+        except NetworkClientError:
             raise ResourceGoneError()
 
         if data.get("device_id", None) is not None:
@@ -469,7 +479,7 @@ def enqueue_soud_sync_processing():
 
     # Check if there is already an enqueued job
     try:
-        converted_next_run = naive_utc_datetime(timezone.now() + next_run)
+        converted_next_run = timezone.now() + next_run
         orm_job = job_storage.get_orm_job(SOUD_SYNC_PROCESSING_JOB_ID)
         if (
             orm_job.state not in (State.COMPLETED, State.FAILED, State.CANCELED)
@@ -536,33 +546,43 @@ class PeerImportSingleSyncJobValidator(PeerSyncJobValidator):
     using_admin = serializers.BooleanField(default=False, required=False)
     force_non_learner_import = serializers.BooleanField(default=False, required=False)
 
+    @retry((NetworkClientError, ResourceGoneError))
+    def _get_user_info(self, data):
+        using_admin = data.get("using_admin", False)
+        facility_id = data["facility"]
+        username = data["username"]
+        password = data["password"]
+        user_id = data.get("user_id")
+
+        if using_admin and user_id:
+            return get_remote_user_info(
+                self.client, facility_id, username, password, user_id
+            )
+
+        remote_users_info = get_remote_users_info(
+            None, facility_id, username, password, client=self.client
+        )
+        return remote_users_info["user"]
+
     def validate(self, data):
         """
         In case an admin account credentials are provided, to sync a non-admin user,
         the user_id of this non-admin user must be provided.
         """
         job_data = super().validate(data)
         user_id = data.get("user_id", None)
-        using_admin = data.get("using_admin", False)
         force_non_learner_import = data.get("force_non_learner_import", False)
         # Use pre-validated base URL
         baseurl = job_data["kwargs"]["baseurl"]
         facility_id = data["facility"]
         username = data["username"]
         password = data["password"]
         try:
-            facility_info = get_remote_users_info(
-                baseurl, facility_id, username, password
-            )
+            user_info = self._get_user_info(data)
         except AuthenticationFailed as e:
             raise ValidationError(detail=str(e.detail), code=e.detail.code)
-        user_info = facility_info["user"]
-
-        # syncing using an admin account (username & password belong to the admin):
-        if using_admin:
-            user_info = next(
-                user for user in facility_info["users"] if user["id"] == user_id
-            )
+        except (NetworkClientError, ConnectionError):
+            raise ResourceGoneError()
 
         full_name = user_info["full_name"]
         roles = user_info["roles"]
@@ -581,9 +601,13 @@ def validate(self, data):
 
         user_id = user_info["id"]
 
-        validate_and_create_sync_credentials(
-            baseurl, facility_id, username, password, user_id=user_id
-        )
+        try:
+            validate_and_create_sync_credentials(
+                baseurl, facility_id, username, password, user_id=user_id
+            )
+        except (NetworkClientError, ConnectionError):
+            raise ResourceGoneError()
+
         job_data["extra_metadata"]["user_id"] = user_id
         job_data["extra_metadata"]["username"] = user_info["username"]
         job_data["extra_metadata"]["user_full_name"] = full_name
@@ -604,12 +628,24 @@ def validate(self, data):
     cancellable=False,
     track_progress=True,
     queue=soud_sync_queue,
+    priority=Priority.HIGH,
     permission_classes=[IsSuperAdmin() | NotProvisioned()],
     status_fn=status_fn,
     long_running=True,
+    retry_on=[
+        OperationalError,
+        MorangoError,
+        HTTPError,
+        NetworkClientError,
+    ],
 )
 def peeruserimport(command, **kwargs):
-    call_command(command, **kwargs)
+    try:
+        call_command(command, **kwargs)
+    except CommandError as e:
+        if "Unable to connect" in str(e):
+            raise NetworkClientError()
+        raise
 
 
 @register_task(

kolibri/core/auth/test/test_auth_tasks.py

@@ -37,7 +37,6 @@
 from kolibri.core.tasks.exceptions import JobRunning
 from kolibri.core.tasks.job import Job
 from kolibri.core.tasks.job import State
-from kolibri.utils.time_utils import naive_utc_datetime
 
 
 DUMMY_PASSWORD = "password"
@@ -64,10 +63,11 @@ def fake_job(**kwargs):
 
 
 class dummy_orm_job_data(object):
-    scheduled_time = datetime.datetime(year=2023, month=1, day=1, tzinfo=None)
+    scheduled_time = datetime.datetime(year=2023, month=1, day=1)
     repeat = 5
     interval = 8600
     retry_interval = 5
+    max_retries = 3
 
 
 @patch("kolibri.core.tasks.api.job_storage")
@@ -701,7 +701,7 @@ def test_enqueue_soud_sync_processing__future__scheduled(
         mock_soud.get_time_to_next_attempt.return_value = datetime.timedelta(seconds=30)
         mock_job = mock_job_storage.get_orm_job.return_value
         mock_job.state = State.QUEUED
-        mock_job.scheduled_time = naive_utc_datetime(timezone.now())
+        mock_job.scheduled_time = timezone.now()
         enqueue_soud_sync_processing()
         mock_task.enqueue_in.assert_not_called()
 
@@ -714,7 +714,7 @@ def test_enqueue_soud_sync_processing__future__running(
         mock_soud.get_time_to_next_attempt.return_value = datetime.timedelta(seconds=1)
         mock_job = mock_job_storage.get_orm_job.return_value
         mock_job.state = State.RUNNING
-        mock_job.scheduled_time = naive_utc_datetime(timezone.now())
+        mock_job.scheduled_time = timezone.now()
         enqueue_soud_sync_processing()
         mock_task.enqueue_in.assert_not_called()
 
@@ -727,9 +727,7 @@ def test_enqueue_soud_sync_processing__future__reschedule(
         mock_soud.get_time_to_next_attempt.return_value = datetime.timedelta(seconds=10)
         mock_job = mock_job_storage.get_orm_job.return_value
         mock_job.state = State.QUEUED
-        mock_job.scheduled_time = naive_utc_datetime(
-            timezone.now() + datetime.timedelta(seconds=15)
-        )
+        mock_job.scheduled_time = timezone.now() + datetime.timedelta(seconds=15)
         enqueue_soud_sync_processing()
         mock_task.enqueue_in.assert_called_once_with(datetime.timedelta(seconds=10))
 
@@ -743,9 +741,7 @@ def test_enqueue_soud_sync_processing__completed__enqueue(
         mock_job = mock_job_storage.get_orm_job.return_value
         mock_job.state = State.COMPLETED
         # far in the past
-        mock_job.scheduled_time = naive_utc_datetime(
-            timezone.now() - datetime.timedelta(seconds=100)
-        )
+        mock_job.scheduled_time = timezone.now() - datetime.timedelta(seconds=100)
         enqueue_soud_sync_processing()
         mock_task.enqueue_in.assert_called_once_with(datetime.timedelta(seconds=10))
 
@@ -759,9 +755,7 @@ def test_enqueue_soud_sync_processing__race__already_running(
         mock_job = mock_job_storage.get_orm_job.return_value
         mock_job.state = State.COMPLETED
         # far in the past
-        mock_job.scheduled_time = naive_utc_datetime(
-            timezone.now() - datetime.timedelta(seconds=100)
-        )
+        mock_job.scheduled_time = timezone.now() - datetime.timedelta(seconds=100)
         mock_task.enqueue_in.side_effect = JobRunning()
         enqueue_soud_sync_processing()
         mock_task.enqueue_in.assert_called_once_with(datetime.timedelta(seconds=10))

kolibri/core/auth/utils/sync.py

@@ -19,7 +19,9 @@
 from kolibri.core.auth.constants.morango_sync import ScopeDefinitions
 from kolibri.core.auth.management.utils import get_client_and_server_certs
 from kolibri.core.auth.management.utils import get_facility_dataset_id
+from kolibri.core.discovery.utils.network.errors import NetworkClientError
 from kolibri.core.discovery.utils.network.errors import NetworkLocationResponseFailure
+from kolibri.core.utils.retry import retry
 
 
 def find_soud_sync_sessions(using=None, **filters):
@@ -69,6 +71,7 @@ def find_soud_sync_session_for_resume(user, base_url, using=None):
     return None
 
 
+@retry(NetworkClientError)
 def validate_and_create_sync_credentials(
     baseurl, facility_id, username, password, user_id=None
 ):