minor adjustments v1.3.1

ezefranca · Aug 11, 2024 · 35e450a · 35e450a
1 parent a034d04
commit 35e450a
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 11 deletions.
diff --git a/Sources/GoogleScholarSwift/Constants/Constants.swift b/Sources/GoogleScholarSwift/Constants/Constants.swift
@@ -1,14 +1,30 @@
 import Foundation
 
 public struct Constants {
+    public static let userAgentList = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36",
+        "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36"
+    ]
+
+    public static func randomUserAgent() -> String {
+        return userAgentList.randomElement() ?? userAgentList[0]
+    }
 
     public static let headers: [String: String] = [
         "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
         "accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
-        "user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36"
+        "sec-ch-ua": "\"Google Chrome\";v=\"116\", \"Chromium\";v=\"116\", \"Not A(Brand\";v=\"24\"",
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "\"Windows\"",
+        "sec-fetch-dest": "document",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-site": "same-origin",
+        "sec-fetch-user": "?1",
+        "upgrade-insecure-requests": "1"
     ]
 
-    public static let cookies: [String: String] = [
+    public static var cookies: [String: String] = [
         "CONSENT": "PENDING+300"
     ]
 }
diff --git a/Sources/GoogleScholarSwift/Impl/GoogleScholarFetcher.swift b/Sources/GoogleScholarSwift/Impl/GoogleScholarFetcher.swift
@@ -65,12 +65,12 @@ public class GoogleScholarFetcher {
             }
 
             var request = URLRequest(url: url)
+            request.addValue(Constants.randomUserAgent(), forHTTPHeaderField: "User-Agent")
+            request.addValue("https://scholar.google.com/", forHTTPHeaderField: "Referer")
             for (header, value) in Constants.headers {
                 request.addValue(value, forHTTPHeaderField: header)
             }
-            for (cookie, value) in Constants.cookies {
-                request.addValue("\(cookie)=\(value)", forHTTPHeaderField: "Cookie")
-            }
+            applyCookies(to: &request)
 
             let (data, _) = try await session.data(for: request)
 
@@ -134,12 +134,12 @@ public class GoogleScholarFetcher {
         }
 
         var request = URLRequest(url: url)
+        request.addValue(Constants.randomUserAgent(), forHTTPHeaderField: "User-Agent")
+        request.addValue("https://scholar.google.com/", forHTTPHeaderField: "Referer")
         for (header, value) in Constants.headers {
             request.addValue(value, forHTTPHeaderField: header)
         }
-        for (cookie, value) in Constants.cookies {
-            request.addValue("\(cookie)=\(value)", forHTTPHeaderField: "Cookie")
-        }
+        applyCookies(to: &request)
 
         let (data, _) = try await session.data(for: request)
 
@@ -204,12 +204,12 @@ public class GoogleScholarFetcher {
         }
 
         var request = URLRequest(url: url)
+        request.addValue(Constants.randomUserAgent(), forHTTPHeaderField: "User-Agent")
+        request.addValue("https://scholar.google.com/", forHTTPHeaderField: "Referer")
         for (header, value) in Constants.headers {
             request.addValue(value, forHTTPHeaderField: header)
         }
-        for (cookie, value) in Constants.cookies {
-            request.addValue("\(cookie)=\(value)", forHTTPHeaderField: "Cookie")
-        }
+        applyCookies(to: &request)
 
         let (data, _) = try await session.data(for: request)
 
@@ -343,6 +343,23 @@ public class GoogleScholarFetcher {
 
         return Author(id: id, name: name, affiliation: affiliation, pictureURL: pictureURL)
     }
+
+    // Helper function to update cookies dynamically
+    private func updateCookies(from response: HTTPURLResponse) {
+        if let headerFields = response.allHeaderFields as? [String: String],
+           let url = response.url {
+            let cookies = HTTPCookie.cookies(withResponseHeaderFields: headerFields, for: url)
+            for cookie in cookies {
+                Constants.cookies[cookie.name] = cookie.value
+            }
+        }
+    }
+
+    // Helper function to apply cookies to the request
+    private func applyCookies(to request: inout URLRequest) {
+        let cookieString = Constants.cookies.map { "\($0.key)=\($0.value)" }.joined(separator: "; ")
+        request.addValue(cookieString, forHTTPHeaderField: "Cookie")
+    }
 }
 
 extension String {