diff --git a/dpxdt/tools/url_pair_multiple_diff.py b/dpxdt/tools/url_pair_multiple_diff.py new file mode 100644 index 0000000..8dfafcf --- /dev/null +++ b/dpxdt/tools/url_pair_multiple_diff.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python +# Copyright 2013 Brett Slatkin / 2016 Nico Haase +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility for doing a diff between pairs of URLs. + +Example usage: + +./dpxdt/tools/url_pair_multiple_diff.py \ + --upload_build_id=1234 \ + --release_server_prefix=https://my-dpxdt-apiserver.example.com/api \ + --release_client_id= \ + --release_client_secret= \ + sourceFile +""" + +import HTMLParser +import Queue +import datetime +import json +import logging +import os +import os.path +import re +import shutil +import sys +import tempfile +import urlparse + +# Local Libraries +import gflags +FLAGS = gflags.FLAGS + +# Local modules +from dpxdt.client import fetch_worker +from dpxdt.client import release_worker +from dpxdt.client import workers +import flags + + +class UrlPairMultipleDiff(workers.WorkflowItem): + """Workflow for diffing multiple pairs of URLs. + + Args: + filename: Filename with a list of URL tuples to scan + upload_build_id: Optional. Build ID of the site being compared. When + supplied a new release will be cut for this build comparing it + to the last good release. + upload_release_name: Optional. Release name to use for the build. When + not supplied, a new release based on the current time will be + created. + heartbeat: Function to call with progress status. + """ + + def run(self, + filename, + upload_build_id, + upload_release_name=None, + heartbeat=None): + + if os.path.isfile(filename) == False: + raise release_worker.Error('File %s is missing' % filename) + + inputLines = [line.rstrip('\n') for line in open(filename)] + + if(len(inputLines)) == 0: + raise release_worker.Error('Input file empty') + + print 'Content found' + first = inputLines[0] + splitted = first.split(' ') + if(len(splitted)) != 2: + raise release_worker.Error('Wrong file format') + + # TODO: Make the default release name prettier. + if not upload_release_name: + upload_release_name = str(datetime.datetime.utcnow()) + + yield heartbeat('Creating release %s' % upload_release_name) + release_number = yield release_worker.CreateReleaseWorkflow( + upload_build_id, upload_release_name, splitted[0]) + + config_dict = { + 'viewportSize': { + 'width': 1280, + 'height': 1024, + } + } + if FLAGS.inject_css: + config_dict['injectCss'] = FLAGS.inject_css + if FLAGS.inject_js: + config_dict['injectJs'] = FLAGS.inject_js + + if FLAGS.http_username: + config_dict['httpUserName'] = FLAGS.http_username + if FLAGS.http_password: + config_dict['httpPassword'] = FLAGS.http_password + + config_data = json.dumps(config_dict) + + for line in inputLines: + splitted = line.split(' ') + if(len(splitted)) != 2: + raise release_worker.Error('Wrong file format') + + new_url = splitted[0] + baseline_url = splitted[1] + + url_parts = urlparse.urlparse(new_url) + + yield heartbeat('Requesting captures for comparing %s to %s' % (new_url, baseline_url)) + yield release_worker.RequestRunWorkflow( + upload_build_id, + upload_release_name, + release_number, + url_parts.path or '/', + new_url, + config_data, + ref_url=baseline_url, + ref_config_data=config_data) + + yield heartbeat('Marking runs as complete') + release_url = yield release_worker.RunsDoneWorkflow( + upload_build_id, upload_release_name, release_number) + + yield heartbeat('Results viewable at: %s' % release_url) + +def real_main(filename=None, + upload_build_id=None, + upload_release_name=None): + """Runs the url_pair_multiple_diff.""" + coordinator = workers.get_coordinator() + fetch_worker.register(coordinator) + coordinator.start() + + item = UrlPairMultipleDiff( + filename, + upload_build_id, + upload_release_name=upload_release_name, + heartbeat=workers.PrintWorkflow) + item.root = True + + coordinator.input_queue.put(item) + coordinator.wait_one() + coordinator.stop() + coordinator.join() + + +def main(argv): + try: + argv = FLAGS(argv) + except gflags.FlagsError, e: + print '%s\nUsage: %s ARGS\n%s' % (e, sys.argv[0], FLAGS) + sys.exit(1) + + if len(argv) != 2: + print 'Must supply one filename as arguments.' + sys.exit(1) + + assert FLAGS.upload_build_id + assert FLAGS.release_server_prefix + + if FLAGS.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + real_main( + filename=argv[1], + upload_build_id=FLAGS.upload_build_id, + upload_release_name=FLAGS.upload_release_name) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/run_url_pair_multiple_diff.sh b/run_url_pair_multiple_diff.sh new file mode 100644 index 0000000..955c14a --- /dev/null +++ b/run_url_pair_multiple_diff.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +./dpxdt/tools/url_pair_multiple_diff.py \ + --release_server_prefix=http://localhost:5000/api \ + "$@"