googleimages.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 전동킥보드 사진 크롤링\n",
    "## 최소 사이즈 640\n",
    "## 라임, 킥고잉, 씽씽, 알파카=고고씽, 빔(beam)\n",
    "## 최소 1,000장"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (1) 라임 킥보드"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "검색하고 싶은 키워드 :  라임 킥보드\n",
      "원하는 이미지 수집 개수 :  1000\n"
     ]
    },
    {
     "ename": "NoSuchElementException",
     "evalue": "Message: no such element: Unable to locate element: {\"method\":\"css selector\",\"selector\":\"#Sva75c > div > div > div.pxAole > div.tvh9oe.BIB1wf > c-wiz > div > div.OUZ5W > div.zjoqD > div > div.v4dQwb > a > img\"}\n  (Session info: chrome=90.0.4430.93)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNoSuchElementException\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-0ae5e8f47a2f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     29\u001b[0m     \u001b[0msearch_path\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"Your Path\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m     \u001b[1;31m# search_maybe(search_name, search_limit, search_path)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 31\u001b[1;33m     \u001b[0msearch_selenium\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msearch_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msearch_path\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msearch_limit\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-4-0ae5e8f47a2f>\u001b[0m in \u001b[0;36msearch_selenium\u001b[1;34m(search_name, search_path, search_limit)\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[1;31m#image_count = len(browser.find_elements_by_tag_name(\"img\"))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[0mimage_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbrowser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_css_selector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"#Sva75c > div > div > div.pxAole > div.tvh9oe.BIB1wf > c-wiz > div > div.OUZ5W > div.zjoqD > div > div.v4dQwb > a > img\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\pydatavenv\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element_by_css_selector\u001b[1;34m(self, css_selector)\u001b[0m\n\u001b[0;32m    596\u001b[0m             \u001b[0melement\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_css_selector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'#foo'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    597\u001b[0m         \"\"\"\n\u001b[1;32m--> 598\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCSS_SELECTOR\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcss_selector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    599\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    600\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_elements_by_css_selector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcss_selector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\pydatavenv\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element\u001b[1;34m(self, by, value)\u001b[0m\n\u001b[0;32m    974\u001b[0m                 \u001b[0mby\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCSS_SELECTOR\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    975\u001b[0m                 \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'[name=\"%s\"]'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 976\u001b[1;33m         return self.execute(Command.FIND_ELEMENT, {\n\u001b[0m\u001b[0;32m    977\u001b[0m             \u001b[1;34m'using'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    978\u001b[0m             'value': value})['value']\n",
      "\u001b[1;32m~\\anaconda3\\envs\\pydatavenv\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    319\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 321\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    322\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    323\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32m~\\anaconda3\\envs\\pydatavenv\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNoSuchElementException\u001b[0m: Message: no such element: Unable to locate element: {\"method\":\"css selector\",\"selector\":\"#Sva75c > div > div > div.pxAole > div.tvh9oe.BIB1wf > c-wiz > div > div.OUZ5W > div.zjoqD > div > div.v4dQwb > a > img\"}\n  (Session info: chrome=90.0.4430.93)\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from bs4 import BeautifulSoup as soups\n",
    "\n",
    "\n",
    "def search_selenium(search_name, search_path, search_limit) :\n",
    "    search_url = \"https://www.google.com/search?q=\" + str(search_name) + \"&tbm=isch&tbs=isz:l&rlz=1C1JZAP_koKR757KR757&hl=ko&sa=X&ved=0CAEQpwVqFwoTCIDQ3c6ntPACFQAAAAAdAAAAABAC&biw=1903&bih=935\"\n",
    "    \n",
    "    browser = webdriver.Chrome('C:/Temp/chromedriver')\n",
    "    browser.get(search_url)\n",
    "    \n",
    "    image_count = len(browser.find_elements_by_tag_name(\"img\"))\n",
    "    \n",
    "    \n",
    "    print(\"로드된 이미지 개수 : \", image_count)\n",
    " \n",
    "    browser.implicitly_wait(2)\n",
    " \n",
    "    for i in range( search_limit ) :\n",
    "        image = browser.find_elements_by_tag_name(\"img\")[i]\n",
    "        image.screenshot(\"C:/jsy/PYDATAexam/shared_scooter/images/\" + str(i) + \".png\")\n",
    "    \n",
    "\n",
    "        \n",
    "        \n",
    "        \n",
    "\"\"\"\n",
    "def run(query, save_directory):\n",
    "    print(\"define program variables and open google images...\")\n",
    "    url = \"https://www.google.co.in/search?q=\"+query+\"&source=lnms&tbm=isch\"\n",
    "    # NEED TO DOWNLOAD CHROMEDRIVER, insert path to chromedriver inside parentheses in following line\n",
    "    browser = webdriver.Chrome('C:/Temp/chromedriver')\n",
    "    browser.get(url) # opens the page\n",
    "    header={'User-Agent':\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36\"}\n",
    "    counter = 0\n",
    "    succounter = 0\n",
    "\n",
    "    print(\"start scrolling to generate more images on the page...\")\n",
    "    # 500 time we scroll down by 10000 in order to generate more images on the website\n",
    "    for _ in range(500):\n",
    "        browser.execute_script(\"window.scrollBy(0,10000)\")\n",
    "\n",
    "    print(\"start scraping ...\")\n",
    "    for x in browser.find_elements_by_xpath('//img[contains(@class,\"rg_i Q4LuWd\")]'):\n",
    "        counter = counter + 1\n",
    "        #print(\"Total Count:\", counter)\n",
    "        #print(\"Successful Count:\", succounter)\n",
    "        #print(\"URL:\", x.get_attribute('src'))\n",
    "\n",
    "        img = x.get_attribute('src')\n",
    "        new_filename = \"/image\"+str(counter)+\".jpg\"\n",
    "        path = save_directory\n",
    "        path += new_filename\n",
    "\n",
    "        try:\n",
    "            urllib.request.urlretrieve(img, path)\n",
    "            succounter += 1\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "    print(succounter, \"pictures succesfully downloaded\")\n",
    "    browser.close()\n",
    "\"\"\"        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "    browser.close()\n",
    " \n",
    "if __name__ == \"__main__\" :\n",
    " \n",
    "    search_name = input(\"검색하고 싶은 키워드 : \")\n",
    "    search_limit = int(input(\"원하는 이미지 수집 개수 : \"))\n",
    "    search_path = \"Your Path\"\n",
    "    # search_maybe(search_name, search_limit, search_path)\n",
    "    search_selenium(search_name, search_path, search_limit)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pydatavenv",
   "language": "python",
   "name": "pydatavenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}