mission-to-Mars.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "09da2088",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- dependencies and setup ---\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from splinter import Browser\n",
    "import time\n",
    "from selenium import webdriver\n",
    "from importlib import import_module\n",
    "import types\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b386b7ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- create a browser instance using splinter (FOR MAC) ---\n",
    "executable_path = {'executable_path': 'C:\\Webdrivers\\chromedriver'}\n",
    "browser = Browser('chrome', **executable_path, headless=False)\n",
    "time.sleep(1)\n",
    "\n",
    "# --- For Windows ---\n",
    "# executable_path = {'executable_path': 'chromedriver.exe'}\n",
    "# browser = Browser('chrome', **executable_path, headless=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "01a99b27",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NASA's Perseverance Makes New Discoveries in Mars' Jezero Crater\n",
      "The rover found that Jezero Crater’s floor is made up of volcanic rocks that have interacted with water.\n"
     ]
    }
   ],
   "source": [
    "# --- visit the Mars News url ---\n",
    "MarsNews_url = 'https://mars.nasa.gov/news/'\n",
    "browser.visit(MarsNews_url)\n",
    "time.sleep(2)\n",
    "\n",
    "# --- create HTML object ---\n",
    "html = browser.html\n",
    "\n",
    "# --- parse HTML with BeautifulSoup ---\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "# --- get the first <li> item under <ul> list of headlines: this contains the latest news title and paragraph text ---\n",
    "first_li = soup.find('li', class_='slide')\n",
    "\n",
    "# --- save the news title under the <div> tag with a class of 'content_title' ---\n",
    "news_title = first_li.find('div', class_='content_title').text\n",
    "print(news_title)\n",
    "\n",
    "# --- save the paragraph text under the <div> tag with a class of 'article_teaser_body' ---\n",
    "news_para = first_li.find('div', class_='article_teaser_body').text\n",
    "print(news_para)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8a1dbc61",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- visit the JPL Featured Space Image website ---\n",
    "JPLimage_url = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'\n",
    "browser.visit(JPLimage_url)\n",
    "time.sleep(2)\n",
    "\n",
    "# --- create HTML object ---\n",
    "html = browser.html\n",
    "\n",
    "# --- parse HTML with BeautifulSoup ---\n",
    "soup = BeautifulSoup(html, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e68f5310",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# --- get the <div> with a class of 'carousel_container': this contains the current featured image details ---\n",
    "carousel = soup.find('div', class_='carousel_container')\n",
    "\n",
    "# --- get the image title found under the <a> tag ---\n",
    "featuredimage_title = carousel.find('a')['data-title']\n",
    "\n",
    "# --- use splinter to click on the 'full image' button to retrieve a full-size jpg url ---\n",
    "browser.find_by_id('full_image').click()\n",
    "time.sleep(1)\n",
    "\n",
    "# --- check if the div with the 'more info' button is visible to proceed to the download page. If false: ---\n",
    "if browser.is_element_visible_by_css('div.fancybox-title') == False:\n",
    "    \n",
    "    # --- create the base url for the image from the carousel container ---\n",
    "    base_url = 'https://www.jpl.nasa.gov/'\n",
    "    \n",
    "    # --- get the image url found under the <a> tag in the carousel ---\n",
    "    image_url = carousel.find('a')['data-fancybox-href']\n",
    "    \n",
    "    # --- complete the featured image url by adding the base url ---\n",
    "    featuredimage_url = base_url + image_url\n",
    "\n",
    "# --- if the div is visible and there is a 'more info' button to proceed --- \n",
    "else:\n",
    "    \n",
    "    # --- create the base url for the fullsize image download link ---\n",
    "    base_url = 'https:'\n",
    "    \n",
    "    # --- click the 'more info' button to go to the image detail page ---\n",
    "    browser.links.find_by_partial_text('more info').click()\n",
    "    time.sleep(1)\n",
    "    \n",
    "    # --- create a beautiful soup object with the image detail page's html ---\n",
    "    img_detail_html = browser.html\n",
    "    imagesoup = BeautifulSoup(img_detail_html, 'html.parser')\n",
    "    \n",
    "    # --- find the fullsize jpg image link and store the url ---\n",
    "    download_div = imagesoup.find_all('div', class_='download_tiff')[1]\n",
    "    fullsize_img = download_div.find('a')['href']\n",
    "\n",
    "    # --- complete the featured image url by adding the base url ---\n",
    "    featuredimage_url = base_url + fullsize_img\n",
    "\n",
    "print(featuredimage_url)\n",
    "print(featuredimage_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fa88344",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- visit the Mars Weather twitter account ---\n",
    "MarsWeather_url = 'https://twitter.com/marswxreport'\n",
    "browser.visit(MarsWeather_url)\n",
    "time.sleep(5)\n",
    "\n",
    "# --- create HTML object ---\n",
    "html = browser.html\n",
    "\n",
    "# --- parse HTML with BeautifulSoup ---\n",
    "soup = BeautifulSoup(html, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e099bc89",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# --- save the latest tweet in a variable (found in the text of the first element <span> \n",
    "# under the <div> tag with lang=\"en\" ---\n",
    "tweet = soup.find_all('div', lang='en')[0].text\n",
    "\n",
    "# --- clean up the tweet (remove newline) ---\n",
    "latest_tweet = tweet.replace('\\n', '')\n",
    "        \n",
    "latest_tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "92d5a59e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- visit the Mars Facts website ---\n",
    "MarsFacts_url = 'https://space-facts.com/mars/'\n",
    "browser.visit(MarsFacts_url)\n",
    "time.sleep(2)\n",
    "\n",
    "# --- create HTML object ---\n",
    "html = browser.html\n",
    "\n",
    "# --- use Pandas to scrape table of facts ---\n",
    "table = pd.read_html(html)\n",
    "\n",
    "# --- use indexing to slice the table to a dataframe ---\n",
    "facts_df = table[0]\n",
    "facts_df.columns =['Description', 'Value']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fd90e354",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Description</th>\n",
       "      <th>Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Equatorial Diameter:</td>\n",
       "      <td>6,792 km</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Polar Diameter:</td>\n",
       "      <td>6,752 km</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mass:</td>\n",
       "      <td>6.39 × 10^23 kg (0.11 Earths)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Moons:</td>\n",
       "      <td>2 (Phobos &amp; Deimos)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Orbit Distance:</td>\n",
       "      <td>227,943,824 km (1.38 AU)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Orbit Period:</td>\n",
       "      <td>687 days (1.9 years)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Surface Temperature:</td>\n",
       "      <td>-87 to -5 °C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>First Record:</td>\n",
       "      <td>2nd millennium BC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Recorded By:</td>\n",
       "      <td>Egyptian astronomers</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Description                          Value\n",
       "0  Equatorial Diameter:                       6,792 km\n",
       "1       Polar Diameter:                       6,752 km\n",
       "2                 Mass:  6.39 × 10^23 kg (0.11 Earths)\n",
       "3                Moons:            2 (Phobos & Deimos)\n",
       "4       Orbit Distance:       227,943,824 km (1.38 AU)\n",
       "5         Orbit Period:           687 days (1.9 years)\n",
       "6  Surface Temperature:                   -87 to -5 °C\n",
       "7         First Record:              2nd millennium BC\n",
       "8          Recorded By:           Egyptian astronomers"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --- visit the Mars Facts website ---\n",
    "MarsFacts_url = 'https://space-facts.com/mars/'\n",
    "browser.visit(MarsFacts_url)\n",
    "time.sleep(2)\n",
    "\n",
    "# --- create HTML object ---\n",
    "html = browser.html\n",
    "\n",
    "# --- use Pandas to scrape table of facts ---\n",
    "table = pd.read_html(html)\n",
    "\n",
    "# --- use indexing to slice the table to a dataframe ---\n",
    "facts_df = table[0]\n",
    "facts_df.columns =['Description', 'Value']\n",
    "facts_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54726a33",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- convert the dataframe to a HTML table and save to html file ---\n",
    "facts_df.to_html('marsfacts.html', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01640f3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- visit the Mars Hemisphere website ---\n",
    "MarsHemImage_url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'\n",
    "browser.visit(MarsHemImage_url)\n",
    "time.sleep(2)\n",
    "\n",
    "# --- create HTML object ---\n",
    "html = browser.html\n",
    "\n",
    "# --- parse HTML with BeautifulSoup ---\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "# --- retrieve all the parent div tags for each hemisphere --- \n",
    "hemisphere_divs = soup.find_all('div', class_=\"item\")\n",
    "\n",
    "# --- create an empty list to store the python dictionary ---\n",
    "hemisphere_image_data = []\n",
    "\n",
    "# --- loop through each div item to get hemisphere data ---\n",
    "for hemisphere in range(len(hemisphere_divs)):\n",
    "\n",
    "    # --- use splinter's browser to click on each hemisphere's link in order to retrieve image data ---\n",
    "    hem_link = browser.find_by_css(\"a.product-item h3\")\n",
    "    hem_link[hemisphere].click()\n",
    "    time.sleep(1)\n",
    "    \n",
    "    # --- create a beautiful soup object with the image detail page's html ---\n",
    "    img_detail_html = browser.html\n",
    "    imagesoup = BeautifulSoup(img_detail_html, 'html.parser')\n",
    "    \n",
    "    # --- create the base url for the fullsize image link ---\n",
    "    base_url = 'https://astrogeology.usgs.gov'\n",
    "    \n",
    "    # --- retrieve the full-res image url and save into a variable ---\n",
    "    hem_url = imagesoup.find('img', class_=\"wide-image\")['src']\n",
    "    \n",
    "    # --- complete the featured image url by adding the base url ---\n",
    "    img_url = base_url + hem_url\n",
    "\n",
    "    # --- retrieve the image title using the title class and save into variable ---\n",
    "    img_title = browser.find_by_css('.title').text\n",
    "    \n",
    "    # --- add the key value pairs to python dictionary and append to the list ---\n",
    "    hemisphere_image_data.append({\"title\": img_title,\n",
    "                              \"img_url\": img_url})\n",
    "    \n",
    "    # --- go back to the main page ---\n",
    "    browser.back()\n",
    "    \n",
    "# --- close the browser session ---    \n",
    "browser.quit()\n",
    "\n",
    "hemisphere_image_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3833081",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}