Url multiple diff

dpxdt/tools/url_pair_multiple_diff.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+# Copyright 2013 Brett Slatkin / 2016 Nico Haase
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utility for doing a diff between pairs of URLs.
+
+Example usage:
+
+./dpxdt/tools/url_pair_multiple_diff.py \
+    --upload_build_id=1234 \
+    --release_server_prefix=https://my-dpxdt-apiserver.example.com/api \
+    --release_client_id=<your api key> \
+    --release_client_secret=<your api secret> \
+    sourceFile
+"""
+
+import HTMLParser
+import Queue
+import datetime
+import json
+import logging
+import os
+import os.path
+import re
+import shutil
+import sys
+import tempfile
+import urlparse
+
+# Local Libraries
+import gflags
+FLAGS = gflags.FLAGS
+
+# Local modules
+from dpxdt.client import fetch_worker
+from dpxdt.client import release_worker
+from dpxdt.client import workers
+import flags
+
+
+class UrlPairMultipleDiff(workers.WorkflowItem):
+    """Workflow for diffing multiple pairs of URLs.
+
+    Args:
+        filename: Filename with a list of URL tuples to scan
+        upload_build_id: Optional. Build ID of the site being compared. When
+            supplied a new release will be cut for this build comparing it
+            to the last good release.
+        upload_release_name: Optional. Release name to use for the build. When
+            not supplied, a new release based on the current time will be
+            created.
+        heartbeat: Function to call with progress status.
+    """
+
+    def run(self,
+            filename,
+            upload_build_id,
+            upload_release_name=None,
+            heartbeat=None):
+
+        if os.path.isfile(filename) == False:
+            raise release_worker.Error('File %s is missing' % filename)
+
+        inputLines = [line.rstrip('\n') for line in open(filename)]
+
+        if(len(inputLines)) == 0:
+            raise release_worker.Error('Input file empty')
+
+        print 'Content found'
+        first = inputLines[0]
+        splitted = first.split(' ')
+        if(len(splitted)) != 2:
+            raise release_worker.Error('Wrong file format')
+
+        # TODO: Make the default release name prettier.
+        if not upload_release_name:
+            upload_release_name = str(datetime.datetime.utcnow())
+
+        yield heartbeat('Creating release %s' % upload_release_name)
+        release_number = yield release_worker.CreateReleaseWorkflow(
+            upload_build_id, upload_release_name, splitted[0])
+
+        config_dict = {
+            'viewportSize': {
+                'width': 1280,
+                'height': 1024,
+            }
+        }
+        if FLAGS.inject_css:
+            config_dict['injectCss'] = FLAGS.inject_css
+        if FLAGS.inject_js:
+            config_dict['injectJs'] = FLAGS.inject_js
+
+        if FLAGS.http_username:
+            config_dict['httpUserName'] = FLAGS.http_username
+        if FLAGS.http_password:
+            config_dict['httpPassword'] = FLAGS.http_password
+
+        config_data = json.dumps(config_dict)
+
+        for line in inputLines:
+            splitted = line.split(' ')
+            if(len(splitted)) != 2:
+                raise release_worker.Error('Wrong file format')
+
+            new_url = splitted[0]
+            baseline_url = splitted[1]
+
+            url_parts = urlparse.urlparse(new_url)
+
+            yield heartbeat('Requesting captures for comparing %s to %s' % (new_url, baseline_url))
+            yield release_worker.RequestRunWorkflow(
+                    upload_build_id,
+                    upload_release_name,
+                    release_number,
+                    url_parts.path or '/',
+                    new_url,
+                    config_data,
+                    ref_url=baseline_url,
+                    ref_config_data=config_data)
+
+        yield heartbeat('Marking runs as complete')
+        release_url = yield release_worker.RunsDoneWorkflow(
+            upload_build_id, upload_release_name, release_number)
+
+        yield heartbeat('Results viewable at: %s' % release_url)
+
+def real_main(filename=None,
+              upload_build_id=None,
+              upload_release_name=None):
+    """Runs the url_pair_multiple_diff."""
+    coordinator = workers.get_coordinator()
+    fetch_worker.register(coordinator)
+    coordinator.start()
+
+    item = UrlPairMultipleDiff(
+        filename,
+        upload_build_id,
+        upload_release_name=upload_release_name,
+        heartbeat=workers.PrintWorkflow)
+    item.root = True
+
+    coordinator.input_queue.put(item)
+    coordinator.wait_one()
+    coordinator.stop()
+    coordinator.join()
+
+
+def main(argv):
+    try:
+        argv = FLAGS(argv)
+    except gflags.FlagsError, e:
+        print '%s\nUsage: %s ARGS\n%s' % (e, sys.argv[0], FLAGS)
+        sys.exit(1)
+
+    if len(argv) != 2:
+        print 'Must supply one filename as arguments.'
+        sys.exit(1)
+
+    assert FLAGS.upload_build_id
+    assert FLAGS.release_server_prefix
+
+    if FLAGS.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    real_main(
+        filename=argv[1],
+        upload_build_id=FLAGS.upload_build_id,
+        upload_release_name=FLAGS.upload_release_name)
+
+
+if __name__ == '__main__':
+    main(sys.argv)

run_url_pair_multiple_diff.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+./dpxdt/tools/url_pair_multiple_diff.py \
+    --release_server_prefix=http://localhost:5000/api \
+    "$@"