retrieve: add error handling for if a source is unnaturally large

jarelllama · Feb 2, 2025 · c9acf76 · c9acf76
1 parent 1e8df4f
commit c9acf76
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/scripts/retrieve_domains.sh b/scripts/retrieve_domains.sh
@@ -231,6 +231,11 @@ process_source_results() {
     # Count number of unfiltered domains
     raw_count="$(wc -l < "$source_results")"
 
+    # Error in case a source wrongly retrieves too many results.
+    if (( raw_count > 10000 )) then
+        error 'Source is unusually large.'
+    fi
+
     # Remove known dead domains (dead domains file is not sorted and includes
     # subdomains)
     dead_count="$(filter \