Add setting to pass proxies to requests

linkcheck/linkcheck_settings.py

@@ -59,3 +59,5 @@
 SITE_DOMAINS = getattr(settings, 'LINKCHECK_SITE_DOMAINS', [])
 DISABLE_LISTENERS = getattr(settings, 'LINKCHECK_DISABLE_LISTENERS', False)
 TOLERATE_BROKEN_ANCHOR = getattr(settings, 'LINKCHECK_TOLERATE_BROKEN_ANCHOR', True)
+PROXIES = getattr(settings, 'LINKCHECK_PROXIES', {})
+TRUST_PROXY_SSL = getattr(settings, 'LINKCHECK_TRUST_PROXY_SSL', False)

linkcheck/models.py

@@ -31,8 +31,10 @@
     LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT,
     MAX_URL_LENGTH,
     MEDIA_PREFIX,
+    PROXIES,
     SITE_DOMAINS,
     TOLERATE_BROKEN_ANCHOR,
+    TRUST_PROXY_SSL,
 )
 
 logger = logging.getLogger(__name__)
@@ -386,6 +388,10 @@ def check_external(self, external_recheck_interval=EXTERNAL_RECHECK_INTERVAL):
             "timeout": LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT,
             "verify": True,
         }
+        if PROXIES:
+            request_params["verify"] = not TRUST_PROXY_SSL
+            request_params["proxies"] = PROXIES
+
         try:
             try:
                 # At first try a HEAD request

linkcheck/tests/test_linkcheck.py

@@ -1,7 +1,7 @@
 import os
 from datetime import datetime, timedelta
 from io import StringIO
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import requests_mock
 import urllib3
@@ -672,6 +672,31 @@ def test_external_check_blocked_user_agent_blocked_head(self):
         self.assertEqual(uv.redirect_to, '')
         self.assertEqual(uv.type, 'external')
 
+    @patch(
+        'linkcheck.models.PROXIES',
+        {'http': 'http://proxy.example.com:8080'},
+    )
+    @patch('requests.head')
+    def test_external_proxy_request(self, mock_head):
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.reason = 'OK'
+        mock_response.history = []
+        mock_head.return_value = mock_response
+        request_url = 'http://test.com'
+        uv = Url(url=request_url)
+        uv.check_url()
+        self.assertEqual(uv.status, True)
+        self.assertEqual(uv.message, '200 OK')
+        self.assertEqual(uv.type, 'external')
+        mock_head.assert_called_once()
+        (call_url,), call_kwargs = mock_head.call_args
+        self.assertEqual(call_url, request_url)
+        self.assertEqual(
+            call_kwargs.get('proxies'),
+            {'http': 'http://proxy.example.com:8080'},
+        )
+
     def test_external_check_timedout(self):
         uv = Url(url=f"{self.live_server_url}/timeout/")
         uv.check_url()

linkcheck/utils.py

@@ -183,7 +183,7 @@ def find_all_links(linklists=None):
         linklists = linklist_cls().get_linklist()
 
         for linklist in linklists:
-            object_id = linklist["object"].id
+            object_id = linklist["object"].pk
             urls = linklist["urls"] + linklist["images"]
             if urls:
                 new = update_urls(urls, content_type, object_id)