fix bug when aqiyi's api return same url

Signed-off-by: wwqgtxx <[email protected]>
wwqgtxx · May 13, 2016 · 1363509 · 1363509
1 parent b52c5a4
commit 1363509
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 4 deletions.
diff --git a/wwqLyParse/parsers/listparser.py b/wwqLyParse/parsers/listparser.py
@@ -59,14 +59,22 @@ def get_vinfo_list(aid):
                 vlist = []
                 # request each page
                 page_n = 0
+                urls = []
                 while True:
                     # make request url
                     page_n += 1
                     url = make_port_url(aid, page_n)
                     # get text
                     raw_text = common.getUrl(url)
+
                     # get list
                     sub_list = parse_one_page(raw_text)
+                    for sub in sub_list:
+                        url = sub['url']
+                        if url in urls:
+                            sub_list = []
+                        else:
+                            urls.append(url)
                     if len(sub_list) > 0:
                         vlist += sub_list
                     else:    # no more data

diff --git a/wwqLyParse/run.py b/wwqLyParse/run.py
@@ -25,7 +25,7 @@
 version = {
     'port_version' : "0.5.0", 
     'type' : 'parse', 
-    'version' : '0.1.9', 
+    'version' : '0.2.0', 
     'uuid' : '{C35B9DFC-559F-49E2-B80B-79B66EC77471}',
     'filter' : [],
     'name' : 'WWQ猎影解析插件', 
@@ -153,7 +153,7 @@ def main():
     #debug(Parse('http://www.iqiyi.com/a_19rrhaare5.html'))
     #debug(Parse('http://www.iqiyi.com/a_19rrhbhf6d.html#vfrm=2-3-0-1'))
     #debug(Parse('http://www.le.com'))
-    debug(Parse('http://www.letv.com/comic/10010294.html'))
+    #debug(Parse('http://www.letv.com/comic/10010294.html'))
     #debug(Parse('http://www.mgtv.com/v/1/1/'))
     #debug(Parse('http://tv.le.com/'))
     #debug(Parse('http://search.pptv.com/s_video?kw=%E5%B1%B1%E6%B5%B7%E7%BB%8F%E4%B9%8B%E8%B5%A4%E5%BD%B1%E4%BC%A0%E8%AF%B4'))
@@ -163,7 +163,8 @@ def main():
     #debug(Parse('http://v.qq.com/tv/'))
     #debug(Parse('http://www.pptv.com/'))
     #debug(Parse('http://yyfm.xyz/video/album/1300046802.html'))
-    #debug(Parse('http://list.iqiyi.com/www/2/----------------iqiyi--.html'))
+    debug(Parse('http://list.iqiyi.com/www/2/----------------iqiyi--.html'))
+    #debug(Parse('http://www.iqiyi.com/a_19rrhb8fjp.html',"list"))
     #debug(Parse('http://www.iqiyi.com/v_19rrl8pmn8.html#vfrm=2-3-0-1'))
     #debug(Parse('http://www.iqiyi.com/v_19rrl8pmn8.html',"formats"))
     #debug(ParseURL("http://www.iqiyi.com/v_19rrl8pmn8.html","4_fullhd_全高清_895.21 MB@youget"))

diff --git a/wwqLyParse/test.bat b/wwqLyParse/test.bat
@@ -1,3 +1,3 @@
 @set PYTHONDONTWRITEBYTECODE=x
-@"C:\Program Files (x86)\LieYing\PyRun.exe" run.py
+@"C:\Program Files (x86)\LieYing\PyRun.exe" --normal run.py
 @pause