linux-gae-instance

#!/bin/bash
#
# Copyright (C) Extensible Service Proxy Authors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
################################################################################
#

# This script manipulates Google App Engine applications.
# It supports deploying an application, deleting it,
# and fetching backend NGINX logs from the managed virtual machines.

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
. ${ROOT}/script/all-utilities || { echo "Cannot load Bash utilities"; exit 1; }

function usage() {
  [[ -n "${1}" ]] && echo "${1}"

  cat <<EOF
usage: ${BASH_SOURCE[0]} [options ...] <command>"

  <command> is one of:
    deploy|create: Creates and deploys a managed VM application.
    delete: Deletes deployed managed VM application.
    nginx-logs: Download NGINX logs from the managed VM.

  options:
    -a ... path to the application's app.yaml file (required)
    -d ... target directory for the NGINX logs (default '/tmp/nginx-logs')
    -g ... path to gcloud tool (default 'gcloud')
    -l ... skip activate_service_account step
    -p ... project id of the target project (default 'esp-load-test')
    -s ... app engine service (module) to use for deployment (default 'default')
    -s ... path to gsutil tool (default 'gsutil')
    -v ... application version to deploy in GAE
EOF
  exit 2
}

GCLOUD='gcloud'
GSUTIL='gsutil'
APP_YAML_FILE=""
VERSION=""
SERVICE="default"
PROJECT="esp-load-test"
LOG_DIRECTORY="/tmp/nginx-logs"
SET_SERVICE_ACCOUNT=true

while getopts :g:la:v:s:p:d: arg; do
  case ${arg} in
    a) APP_YAML_FILE="${OPTARG}";;
    d) LOG_DIRECTORY="${OPTARG}";;
    g) GCLOUD="${OPTARG}";;
    l) SET_SERVICE_ACCOUNT=false;;
    p) PROJECT="${OPTARG}";;
    s) SERVICE="${OPTARG}";;
    v) VERSION="${OPTARG}";;
    *) usage "Invalid option: -${OPTARG}";;
  esac
done

shift $((OPTIND-1))
COMMAND="$1"

function deploy_application() {
  echo "Deploy GAE instace ${APP_YAML_FILE} to ${VERSION}"
  echo "gcloud version:"
  echo "$(gcloud -v)"

  APP_DIR="$(dirname ${APP_YAML_FILE})"

  local ARGS=()
  [[ -n "${VERSION}" ]] && ARGS+=(--version "${VERSION}")

  (
    # deploying an endpoints app requires the current directory
    # to be the app directory
    cd "${APP_DIR}"
    echo "gcloud deploy command:"
    echo "${GCLOUD} --project ${PROJECT} app deploy ${APP_YAML_FILE} ${ARGS[@]} --no-promote -q"
    ${GCLOUD} --project "${PROJECT}" app deploy \
      "${APP_YAML_FILE}" "${ARGS[@]}" --no-promote -q
  )
}

function delete_application() {
  echo "Stop GAE instance ${VERSION}"

  retry -n 2 ${GCLOUD} --project "${PROJECT}" app versions stop \
    ${VERSION} --service ${SERVICE} -q

  echo "Delete GAE instance ${VERSION}"

  retry -n 2 ${GCLOUD} --project "${PROJECT}" app versions delete \
    ${VERSION} --service ${SERVICE} -q
}

function set_ssh_keys() {
  local user="$(whoami)"
  local remote_ssh_keys="gs://${PROJECT}.appspot.com/ssh_keys/${user}"
  local local_ssh_keys="${HOME}/.ssh"
  local private_ssh_key="${local_ssh_keys}/google_compute_engine"
  if ! [[ -a "${private_ssh_key}" ]]; then
    ${GSUTIL} stat "${remote_ssh_keys}/google_compute_engine"
    if [[ ${?} -eq 0 ]]; then
      mkdir -p "${local_ssh_keys}" \
        || error_exit "Cannot create ${local_ssh_keys}"
      echo "Getting ssh key from ${remote_ssh_keys}"
      retry -n 3 ${GSUTIL} cp "${remote_ssh_keys}/*" "${local_ssh_keys}/" \
        || error_exit "Could not set ssh keys"
      echo "Setting right permssion on ${local_ssh_keys}"
      chmod -R 700 "${local_ssh_keys}"
    else
      echo 'Generating new ssh keys...'
      ${GCLOUD} --project ${PROJECT} compute config-ssh -q || \
        error_exit 'Unable to generate ssh keys'
      echo "Uploading new ssh keys to ${remote_ssh_keys}"
      retry -n 2 -s 10 ${GSUTIL} -h "Content-Type:text/plain" -m cp "${private_ssh_key}*" \
        "${remote_ssh_keys}/" \
        || error_exit "Failed to upload ssh keys to ${remote_ssh_keys}"
    fi
  fi
}


function download_nginx_error_logs() {
  local NGINX_ERROR_LOG_FILE="${LOG_DIRECTORY}/nginx_error.log"

  gcloud config set project "${PROJECT}"
  gcloud logging read "resource.labels.project_id=${PROJECT} AND \
    resource.labels.version_id=${VERSION} AND \
    logName=projects/${PROJECT}/logs/appengine.googleapis.com%2Fnginx.error" 2>&1 \
    | tee "${NGINX_ERROR_LOG_FILE}"
}

function download_nginx_logs() {
  local vm_id vm_zone vm_name

  [[ -n "${VERSION}" ]] \
    || error_exit "No version specified to download logs. Aborting."

  # If the output format is changed, we will fail to extract fields.
  # Here the output is always printed for easy debug.
  function instances_list () {
    ${GCLOUD} app instances list \
        --project "${PROJECT}" --service "${SERVICE}" --version "${VERSION}" \
        --format json > $ROOT/instance-list
    return ${?}
  }

  retry -n 5 -s 10 instances_list
  echo "Instance list:"
  cat $ROOT/instance-list
  read vm_id vm_zone vm_name < <(cat $ROOT/instance-list \
    | python "${ROOT}/script/find-gae-backend-vm-name.py")

  [[ -n "${vm_id}" && -n "${vm_zone}" && -n "${vm_name}" ]] \
    || error_exit "Cannot identify backend VM name and zone. Aborting."

  retry -n 5 -s 10 ${GCLOUD} app instances enable-debug ${vm_id} \
          --service "${SERVICE}" --version "${VERSION}" \
          --project ${PROJECT} -q \
    || error_exit "Cannot enable-debug on the managed VM. Aborting."

  mkdir -p "${LOG_DIRECTORY}"

  local archive_path="${LOG_DIRECTORY}/${VERSION}.tar.gz"
  local remote_archive="/tmp/logs.tar.gz"

  set_ssh_keys

  retry -n 2 -s 10 \
    ${GCLOUD} compute ssh --project=${PROJECT} --zone=${vm_zone} ${vm_name} \
        --command="sudo tar czf ${remote_archive} /var/log/; sudo chmod 777 ${remote_archive}" \
    || error_exit "Cannot tar backend VM logs."

  retry -n 2 -s 10 \
    ${GCLOUD} compute copy-files --project=${PROJECT} --zone=${vm_zone} \
        "${vm_name}:${remote_archive}" "${archive_path}" \
    || error_exit "Cannot copy logs from the backend VM."

  # It is possible that files change during taring which cause tar to return non
  # zero value.
  [[ -e "${archive_path}" ]] \
    || error_exit "Cannot copy logs from the backend VM."

}

# Not needed when run by ESP Engineers.
# This will leave the gcloud user unauhtenticated at exit.
[[ ${SET_SERVICE_ACCOUNT} == true ]] \
 && activate_service_account "${PROJECT}"

case "${COMMAND}" in
  create | deploy)  retry -n 5 -s 10 deploy_application;;
  delete         )  delete_application;;
  nginx-logs     )  download_nginx_error_logs;;
  *              )  usage "Unknown command ${COMMAND}."
esac