scrapping-server/selenium_main.py at scraping_only · Send-wish/scrapping-server · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from selenium_price import get_price
from selenium_title import get_title
from selenium_img import get_img
from flask import jsonify
from elevenst import elevenst_get_info

DRIVER_PATH = "/app/chrome/chromedriver"
# DRIVER_PATH = "/chromedriver"
options = webdriver.ChromeOptions()
options.add_argument("--headless")
options.add_argument("--window-size=1920,1200")
options.add_argument('User-Agent=Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1')
options.add_argument('Accept-Language=ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7')
options.add_argument("lang=ko_KR")
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
# options.add_argument("disable-gpu")
# options.add_argument("disable-infobars")
# options.add_argument("--disable-extensions")
mobile_emulation = { "deviceName": "iPhone X" }
options.add_experimental_option("mobileEmulation", mobile_emulation)
browser = webdriver.Chrome(options = options, executable_path=DRIVER_PATH)

def web_scrap(url):
    try :
        if (url.find("musinsaapp") != -1): # 무신사 앱링크면
            url += "?_imcp=1"
        browser.get(url)
        if (url.find("11st.co.kr") != -1): # 11번가
            return elevenst_get_info(browser)
        else:
            title = get_title(browser, url)
            print("title", title)
            price = get_price(browser)
            print("price", price)
            img = get_img(browser, url)
            print("img", img)
        print("===Finish Scraping===")
        return jsonify({'url': url, 'title': title, 'price': price, 'img': img})

    except :
        print("===SCRAP ERROR===")
        return jsonify({'url': url, 'title': '사이트로 이동하기', 'price': 0, 'img': 'https://sendwish-img-bucket.s3.ap-northeast-2.amazonaws.com/default_image.png'})