Skip to content
This repository was archived by the owner on Jun 10, 2024. It is now read-only.

add two attribute: response.request_urls, response.redirect_urls #699

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

add two attribute: response.request_urls, response.redirect_urls #699

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions pyspider/fetcher/phantomjs_fetcher.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ if (system.args.length !== 2) {
page_loaded = false,
start_time = Date.now(),
end_time = null,
redirect_urls = new Array(),
request_urls = new Array(),
script_executed = false,
script_result = null;

Expand Down Expand Up @@ -95,7 +97,11 @@ if (system.args.length !== 2) {
end_time = Date.now() + wait_before_end;
setTimeout(make_result, wait_before_end+10, page);
};
page.onNavigationRequested = function(url) {
redirect_urls.push(url);
}
page.onResourceRequested = function(request) {
request_urls.push(request.url);
console.debug("Starting request: #"+request.id+" ["+request.method+"]"+request.url);
end_time = null;
};
Expand Down Expand Up @@ -160,6 +166,8 @@ if (system.args.length !== 2) {
content: page.content || "",
headers: {},
url: page.url || fetch.url,
redirect_urls: redirect_urls,
request_urls: request_urls,
cookies: {},
time: (Date.now() - start_time) / 1000,
js_script_result: null,
Expand Down Expand Up @@ -200,6 +208,8 @@ if (system.args.length !== 2) {
content: page.content,
headers: headers,
url: page.url,
redirect_urls: redirect_urls,
request_urls: request_urls,
cookies: cookies,
time: (Date.now() - start_time) / 1000,
js_script_result: script_result,
Expand Down
6 changes: 5 additions & 1 deletion pyspider/libs/response.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,16 @@
class Response(object):

def __init__(self, status_code=None, url=None, orig_url=None, headers=CaseInsensitiveDict(),
content='', cookies=None, error=None, traceback=None, save=None, js_script_result=None, time=0):
content='', redirect_urls=None, request_urls=None, cookies=None, error=None, traceback=None, save=None, js_script_result=None, time=0):
if cookies is None:
cookies = {}
self.status_code = status_code
self.url = url
self.orig_url = orig_url
self.headers = headers
self.content = content
self.redirect_urls = redirect_urls
self.request_urls = request_urls
self.cookies = cookies
self.error = error
self.traceback = traceback
Expand Down Expand Up @@ -197,6 +199,8 @@ def rebuild_response(r):
url=r.get('url', ''),
headers=CaseInsensitiveDict(r.get('headers', {})),
content=r.get('content', ''),
redirect_urls=r.get('redirect_urls', []),
request_urls=r.get('request_urls', []),
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

better called resources

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah

cookies=r.get('cookies', {}),
error=r.get('error'),
traceback=r.get('traceback'),
Expand Down