From df016de6e502225bf578a4c7d24baf8f009242b1 Mon Sep 17 00:00:00 2001
From: Dan Cook <danielecook@gmail.com>
Date: Thu, 1 Dec 2016 19:32:34 -0600
Subject: [PATCH] add queue and start one-time mapping pipeline.

---
 .travis.yml     |   2 +-
 app.yaml        |   3 --
 cendr/cegwas.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++--
 queue.yaml      |   5 +++
 worker.yaml     |   9 +++++
 5 files changed, 116 insertions(+), 7 deletions(-)
 create mode 100644 queue.yaml
 create mode 100644 worker.yaml

diff --git a/.travis.yml b/.travis.yml
index 026baf4a..50792a5a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,7 +24,7 @@ script:
 
 deploy:
   provider: gae
-  version: version-1-0-5
+  version: version-1-0-6
   project: andersen-lab
   keyfile: client-secret.json
   on: master
diff --git a/app.yaml b/app.yaml
index a2fb61f3..7d747499 100644
--- a/app.yaml
+++ b/app.yaml
@@ -12,9 +12,6 @@ threadsafe: yes
 
 # Handlers define how to route requests to your application.
 handlers:
-- url: /order/.*
-  secure: always
-  script: run.app  
 - url: /.*
   script: run.app  # a WSGI application in the main module's global scope
 
diff --git a/cendr/cegwas.py b/cendr/cegwas.py
index 7aff8de8..14e471fe 100644
--- a/cendr/cegwas.py
+++ b/cendr/cegwas.py
@@ -9,6 +9,107 @@
 from werkzeug.contrib.atom import AtomFeed
 from urlparse import urljoin
 
+import string
+import random
+
+def id_generator(size = 4, chars = string.ascii_lowercase):
+    return ''.join(random.choice(chars) for _ in range(size))
+
+from google.appengine.api import taskqueue
+
+def create_mapping_instance(report_slug, trait_slug):
+    from oauth2client.client import GoogleCredentials
+    credentials = GoogleCredentials.get_application_default()
+    from googleapiclient import discovery
+    compute = discovery.build('compute', 'v1', credentials=credentials)
+
+    # Get latest CeNDR mapping image
+    image_response = compute.images().getFromFamily(project='andersen-lab',
+                                                     family = 'cendr').execute()
+    source_disk_image = image_response['selfLink']
+
+    #startup_script = open(
+    #    os.path.join(
+    #        os.path.dirname(__file__), 'startup-script.sh'), 'r').read()
+    startup_script = '#! /bin/bash\n\nsleep 10s && gcloud -q compute instances delete --zone=us-central1-a `hostname`'
+
+    config = {
+        'name': 'cendr-mapping-submission-' + id_generator(4),
+        'zone': 'projects/andersen-lab/zones/us-central1-a',
+        'machineType': "zones/us-central1-a/machineTypes/n1-standard-1",
+        'tags': {
+            "items": ['cendr-mapping']
+        },
+        # Specify the boot disk and the image to use as a source.
+        'disks': [
+            {
+                'boot': True,
+                'autoDelete': True,
+                'initializeParams': {
+                    'sourceImage': source_disk_image,
+                    'diskType': "projects/andersen-lab/zones/us-central1-a/diskTypes/pd-standard",
+                    'diskSizeGb': "10"
+                }
+            }
+        ],
+
+        # Specify a network interface with NAT to access the public
+        # internet.
+        'networkInterfaces': [{
+            'network': 'projects/andersen-lab/global/networks/default',
+            'accessConfigs': [
+                {
+                  "name": "External NAT",
+                  "type": "ONE_TO_ONE_NAT"
+                }
+              ]
+        }],
+
+        # Allow the instance to access cloud storage and logging.
+        'serviceAccounts': [{
+            'email': 'andersen-lab@appspot.gserviceaccount.com',
+            'scopes': [
+                'https://www.googleapis.com/auth/devstorage.read_write',
+                'https://www.googleapis.com/auth/logging.write',
+                'https://www.googleapis.com/auth/compute'
+            ]
+        }],
+
+        # Metadata is readable from the instance and allows you to
+        # pass configuration from deployment scripts to instances.
+        'metadata': {
+            'items': [{
+                # Startup script is automatically executed by the
+                # instance upon startup.
+                'key': 'startup-script',
+                'value': startup_script
+            },  {
+                'key': 'report_slug',
+                'value': report_slug
+            },
+                {
+            'key': 'trait_slug',
+            'value': trait_slug 
+            }]
+        }
+    }
+
+    return compute.instances().insert(
+        project='andersen-lab',
+        zone='us-central1-a',
+        body=config).execute()
+
+
+@app.route("/launch_instance", methods=['POST'])
+def task_test():
+    print "Great"
+
+def create_task():
+    task = taskqueue.add(queue_name = 'map-queue', url='/launch_instance')
+    #queue = taskqueue.Queue(name='map-queue')
+    #queue.add_async(task)
+
+
 def make_external(url):
     return urljoin(request.url_root, url)
 
@@ -33,9 +134,6 @@ def sortedfiles(path):
 def main():
     page_title = "Caenorhabditis elegans Natural Diversity Resource"
     files = sortedfiles("cendr/static/content/news/")
-    print(files)
-    #files.reverse()
-    # latest mappings
     latest_mappings = list(report.filter(report.release == 0, trait.status == "complete").join(trait).order_by(
         trait.submission_complete.desc()).limit(5).select(report, trait).distinct().dicts().execute())
     return render_template('home.html', **locals())
diff --git a/queue.yaml b/queue.yaml
new file mode 100644
index 00000000..30d1178f
--- /dev/null
+++ b/queue.yaml
@@ -0,0 +1,5 @@
+queue:
+- name: map-queue
+  mode: push
+  max_concurrent_requests: 5
+  rate: 1/s
\ No newline at end of file
diff --git a/worker.yaml b/worker.yaml
new file mode 100644
index 00000000..c5b6f1d3
--- /dev/null
+++ b/worker.yaml
@@ -0,0 +1,9 @@
+runtime: python27
+api_version: 1
+threadsafe: true
+module: worker
+
+handlers:
+- url: /.*
+  script: run.app
+  login: admin
\ No newline at end of file