Merge pull request scrapy#2789 from starrify/add-response-follow-tag-link

[MRG+1] Added: Now supporting <link> tags in Response.follow

Author: dangra

scrapy/http/response/text.py

@@ -135,7 +135,7 @@ def follow(self, url, callback=None, method='GET', headers=None, body=None,
         * an attribute Selector (not SelectorList) - e.g.
           ``response.css('a::attr(href)')[0]`` or
           ``response.xpath('//img/@src')[0]``.
-        * a Selector for ``<a>`` element, e.g.
+        * a Selector for ``<a>`` or ``<link>`` element, e.g.
           ``response.css('a.my_link')[0]``.
           
         See :ref:`response-follow-example` for usage examples.
@@ -165,10 +165,11 @@ def _url_from_selector(sel):
         return strip_html5_whitespace(sel.root)
     if not hasattr(sel.root, 'tag'):
         raise ValueError("Unsupported selector: %s" % sel)
-    if sel.root.tag != 'a':
-        raise ValueError("Only <a> elements are supported; got <%s>" %
+    if sel.root.tag not in ('a', 'link'):
+        raise ValueError("Only <a> and <link> elements are supported; got <%s>" %
                          sel.root.tag)
     href = sel.root.get('href')
     if href is None:
-        raise ValueError("<a> element has no href attribute: %s" % sel)
+        raise ValueError("<%s> element has no href attribute: %s" %
+                         (sel.root.tag, sel))
     return strip_html5_whitespace(href)

tests/test_http_response.py

@@ -162,7 +162,6 @@ def test_follow_whitespace_url(self):
     def test_follow_whitespace_link(self):
         self._assert_followed_url(Link('http://example.com/foo '),
                                   'http://example.com/foo%20')
-
     def _assert_followed_url(self, follow_obj, target_url, response=None):
         if response is None:
             response = self._links_response()
@@ -402,6 +401,13 @@ def test_follow_selector(self):
             for sel, url in zip(sellist, urls):
                 self._assert_followed_url(sel, url, response=resp)
 
+        # select <link> elements
+        self._assert_followed_url(
+            Selector(text='<link href="foo"></link>').css('link')[0],
+            'http://example.com/foo',
+            response=resp
+        )
+
         # href attributes should work
         for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:
             for sel, url in zip(sellist, urls):