From 1ca3a4d4b3444b5ce003879e4b80b7229c1ed60b Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Fri, 12 Apr 2024 16:33:57 -0400 Subject: [PATCH 1/5] Check in a fork of the py-tpcc code --- workloads/chbenchmark/py-tpcc/README.md | 32 + workloads/chbenchmark/py-tpcc/README.original | 52 + .../chbenchmark/py-tpcc/pytpcc/.gitignore | 3 + .../chbenchmark/py-tpcc/pytpcc/CONFIG_EXAMPLE | 17 + .../chbenchmark/py-tpcc/pytpcc/README_v1.1 | 8 + .../chbenchmark/py-tpcc/pytpcc/__init__.py | 1 + .../chbenchmark/py-tpcc/pytpcc/constants.py | 165 ++ .../chbenchmark/py-tpcc/pytpcc/coordinator.py | 218 +++ .../py-tpcc/pytpcc/drivers/__init__.py | 1 + .../py-tpcc/pytpcc/drivers/abstractdriver.py | 169 ++ .../py-tpcc/pytpcc/drivers/cassandradriver.py | 771 ++++++++ .../py-tpcc/pytpcc/drivers/couchdbdriver.py | 868 +++++++++ .../py-tpcc/pytpcc/drivers/csvdriver.py | 89 + .../py-tpcc/pytpcc/drivers/hbasedriver.py | 1408 ++++++++++++++ .../py-tpcc/pytpcc/drivers/membasedriver.py | 932 +++++++++ .../py-tpcc/pytpcc/drivers/mongodbdriver.py | 809 ++++++++ .../py-tpcc/pytpcc/drivers/redisdriver.py | 1663 +++++++++++++++++ .../py-tpcc/pytpcc/drivers/scalarisdriver.py | 1039 ++++++++++ .../py-tpcc/pytpcc/drivers/sqlitedriver.py | 470 +++++ .../pytpcc/drivers/tokyocabinetdriver.py | 1061 +++++++++++ .../chbenchmark/py-tpcc/pytpcc/message.py | 53 + .../py-tpcc/pytpcc/runtime/__init__.py | 3 + .../py-tpcc/pytpcc/runtime/executor.py | 252 +++ .../py-tpcc/pytpcc/runtime/loader.py | 376 ++++ workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py | 273 +++ workloads/chbenchmark/py-tpcc/pytpcc/tpcc.sql | 140 ++ .../py-tpcc/pytpcc/util/__init__.py | 3 + .../chbenchmark/py-tpcc/pytpcc/util/nurand.py | 61 + .../chbenchmark/py-tpcc/pytpcc/util/rand.py | 141 ++ .../py-tpcc/pytpcc/util/results.py | 130 ++ .../py-tpcc/pytpcc/util/scaleparameters.py | 82 + .../chbenchmark/py-tpcc/pytpcc/worker.py | 147 ++ workloads/chbenchmark/py-tpcc/setup.py | 26 + 33 files changed, 11463 insertions(+) create mode 100644 workloads/chbenchmark/py-tpcc/README.md create mode 100644 workloads/chbenchmark/py-tpcc/README.original create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/.gitignore create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/CONFIG_EXAMPLE create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/README_v1.1 create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/__init__.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/constants.py create mode 100755 workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/__init__.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/abstractdriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/cassandradriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/couchdbdriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/csvdriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/hbasedriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/membasedriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/mongodbdriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/redisdriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/scalarisdriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/sqlitedriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/tokyocabinetdriver.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/message.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/runtime/__init__.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/runtime/loader.py create mode 100755 workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/tpcc.sql create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/util/__init__.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/util/nurand.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/util/rand.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/util/results.py create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/util/scaleparameters.py create mode 100755 workloads/chbenchmark/py-tpcc/pytpcc/worker.py create mode 100644 workloads/chbenchmark/py-tpcc/setup.py diff --git a/workloads/chbenchmark/py-tpcc/README.md b/workloads/chbenchmark/py-tpcc/README.md new file mode 100644 index 00000000..249060a0 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/README.md @@ -0,0 +1,32 @@ +# Python TPC-C implementation + +Source: https://github.com/apavlo/py-tpcc at commit +`db36d72dfbb6bd800d257279be9bbc1a22095ff9`. + +See the individual files for the license. The license comment from `tpcc.py` is +shown below. + +> ----------------------------------------------------------------------- +> Copyright (C) 2011 +> Andy Pavlo +> http:##www.cs.brown.edu/~pavlo/ +> +> Permission is hereby granted, free of charge, to any person obtaining +> a copy of this software and associated documentation files (the +> "Software"), to deal in the Software without restriction, including +> without limitation the rights to use, copy, modify, merge, publish, +> distribute, sublicense, and/or sell copies of the Software, and to +> permit persons to whom the Software is furnished to do so, subject to +> the following conditions: +> +> The above copyright notice and this permission notice shall be +> included in all copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +> IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +> OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +> ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +> OTHER DEALINGS IN THE SOFTWARE. +> ----------------------------------------------------------------------- diff --git a/workloads/chbenchmark/py-tpcc/README.original b/workloads/chbenchmark/py-tpcc/README.original new file mode 100644 index 00000000..5c5d4b37 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/README.original @@ -0,0 +1,52 @@ ++ ----------------------------------------------- + ++ Python TPC-C + ++ ----------------------------------------------- + + + +The basic idea is that you will need to create a new driver file that +implements the functions defined in "abstractdriver.py". One function will +load in the tuples into your database for a given table. Then there are five +separate functions that execute the given transaction based on a set of input +parameters. All the work for generating the tuples and the input parameters +for the transactions has been done for you. + +Here's what you need to do to get started: + +(1) Download the source code from Github: + +https://github.com/apavlo/py-tpcc/tree/master/pytpcc + +(2) Create a new file in the 'drivers' directory for your system that follows +the proper naming convention. For example, if your system is 'MongoDB', then +your new file will be called 'mongodbdriver.py' and that file will contain a +new class called 'MongodbDriver' (note the capitalization). + +(3) Inside your class you will need to implement the required functions of +defined in AbstractDriver. There is documentation on what these need to do +also available on Github: + +https://github.com/apavlo/py-tpcc/wiki + +(3) Try running your system. I would start by defining the configuration file +that gets returned with by the 'makeDefaultConfig' function in your driver and +then implement the data loading part first, since that will guide how you +actually execute the transactions. Using 'MongoDB' as an example again, you +can print out the driver's configuration dict to a file: + +$ python ./tpcc.py --print-config mongodb > mongodb.config + +Make any changes you need to 'mongodb.config' (e.g., passwords, hostnames). +Then test the loader: + +$ python ./tpcc.py --no-execute --config=mongodb.config mongodb + +You can use the CSV driver if you want to see what the data or transaction +input parameters will look like. The following command will dump out just the +input to the driver's functions to files in /tmp/tpcc-* + +$ python ./tpcc.py csv + +You can also look at my SqliteDriver implementation to get an idea of what +your transaction implementation functions need to do: + +https://github.com/apavlo/py-tpcc/blob/master/pytpcc/drivers/sqlitedriver.py diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/.gitignore b/workloads/chbenchmark/py-tpcc/pytpcc/.gitignore new file mode 100644 index 00000000..499c008b --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/.gitignore @@ -0,0 +1,3 @@ +*.pyc +.#kate-* +*.config \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/CONFIG_EXAMPLE b/workloads/chbenchmark/py-tpcc/pytpcc/CONFIG_EXAMPLE new file mode 100644 index 00000000..c0054bc0 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/CONFIG_EXAMPLE @@ -0,0 +1,17 @@ +# HypertableDriver Configuration File +# Created 2011-05-02 01:43:37.859545 +[hypertable] + +# hostname +host = localhost + +# namespace name +namespace = tpcc + +# port +port = 38080 + +#clientnodes splited by spaces +clients =u1 u2 192.168.3.21 +#directories of the code on the client node +path =./code/tpcc/py-tpcc/mtpcc diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/README_v1.1 b/workloads/chbenchmark/py-tpcc/pytpcc/README_v1.1 new file mode 100644 index 00000000..c41e2c1f --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/README_v1.1 @@ -0,0 +1,8 @@ +1. 3 newly added files message.py worker.py coordinator.py. Copy them into the old pytpcc directory. +2. Coordinator is the main part. Use it like the old tpcc.py. e.g., python coordinator.py --config hypertable.config --clientprocs 5 hypertable +3. Old argument --clients is replaced with --clientprocs, which specifies how many worker processes you want to run on each client node. +4. All clientnodes(name or ip) must be specified in the configure file. +5. The directory of the code on the client side should be specified in the configure file,too. The default address is user's home address, which is default using ssh. + It should remain the same for each client node, which is not a problem for now. +6. Execnet python module should be installed on each client. Here is how to install it. http://codespeak.net/execnet/install.html + diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/__init__.py b/workloads/chbenchmark/py-tpcc/pytpcc/__init__.py new file mode 100644 index 00000000..792d6005 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/__init__.py @@ -0,0 +1 @@ +# diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/constants.py b/workloads/chbenchmark/py-tpcc/pytpcc/constants.py new file mode 100644 index 00000000..b5a9bbef --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/constants.py @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Original Java Version: +# Copyright (C) 2008 +# Evan Jones +# Massachusetts Institute of Technology +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +MONEY_DECIMALS = 2 + +# Item constants +NUM_ITEMS = 100000 +MIN_IM = 1 +MAX_IM = 10000 +MIN_PRICE = 1.00 +MAX_PRICE = 100.00 +MIN_I_NAME = 14 +MAX_I_NAME = 24 +MIN_I_DATA = 26 +MAX_I_DATA = 50 + +# Warehouse constants +MIN_TAX = 0 +MAX_TAX = 0.2000 +TAX_DECIMALS = 4 +INITIAL_W_YTD = 300000.00 +MIN_NAME = 6 +MAX_NAME = 10 +MIN_STREET = 10 +MAX_STREET = 20 +MIN_CITY = 10 +MAX_CITY = 20 +STATE = 2 +ZIP_LENGTH = 9 +ZIP_SUFFIX = "11111" + +# Stock constants +MIN_QUANTITY = 10 +MAX_QUANTITY = 100 +DIST = 24 +STOCK_PER_WAREHOUSE = 100000 + +# District constants +DISTRICTS_PER_WAREHOUSE = 10 +INITIAL_D_YTD = 30000.00 # different from Warehouse +INITIAL_NEXT_O_ID = 3001 + +# Customer constants +CUSTOMERS_PER_DISTRICT = 3000 +INITIAL_CREDIT_LIM = 50000.00 +MIN_DISCOUNT = 0.0000 +MAX_DISCOUNT = 0.5000 +DISCOUNT_DECIMALS = 4 +INITIAL_BALANCE = -10.00 +INITIAL_YTD_PAYMENT = 10.00 +INITIAL_PAYMENT_CNT = 1 +INITIAL_DELIVERY_CNT = 0 +MIN_FIRST = 6 +MAX_FIRST = 10 +MIDDLE = "OE" +PHONE = 16 +MIN_C_DATA = 300 +MAX_C_DATA = 500 +GOOD_CREDIT = "GC" +BAD_CREDIT = "BC" + +# Order constants +MIN_CARRIER_ID = 1 +MAX_CARRIER_ID = 10 +# HACK: This is not strictly correct, but it works +NULL_CARRIER_ID = 0L +# o_id < than this value, carrier != null, >= -> carrier == null +NULL_CARRIER_LOWER_BOUND = 2101 +MIN_OL_CNT = 5 +MAX_OL_CNT = 15 +INITIAL_ALL_LOCAL = 1 +INITIAL_ORDERS_PER_DISTRICT = 3000 + +# Used to generate new order transactions +MAX_OL_QUANTITY = 10 + +# Order line constants +INITIAL_QUANTITY = 5 +MIN_AMOUNT = 0.01 + +# History constants +MIN_DATA = 12 +MAX_DATA = 24 +INITIAL_AMOUNT = 10.00 + +# New order constants +INITIAL_NEW_ORDERS_PER_DISTRICT = 900 + +# TPC-C 2.4.3.4 (page 31) says this must be displayed when new order rolls back. +INVALID_ITEM_MESSAGE = "Item number is not valid" + +# Used to generate stock level transactions +MIN_STOCK_LEVEL_THRESHOLD = 10 +MAX_STOCK_LEVEL_THRESHOLD = 20 + +# Used to generate payment transactions +MIN_PAYMENT = 1.0 +MAX_PAYMENT = 5000.0 + +# Indicates "brand" items and stock in i_data and s_data. +ORIGINAL_STRING = "ORIGINAL" + +# Table Names +TABLENAME_ITEM = "ITEM" +TABLENAME_WAREHOUSE = "WAREHOUSE" +TABLENAME_DISTRICT = "DISTRICT" +TABLENAME_CUSTOMER = "CUSTOMER" +TABLENAME_STOCK = "STOCK" +TABLENAME_ORDERS = "ORDERS" +TABLENAME_NEW_ORDER = "NEW_ORDER" +TABLENAME_ORDER_LINE = "ORDER_LINE" +TABLENAME_HISTORY = "HISTORY" + +ALL_TABLES = [ + TABLENAME_ITEM, + TABLENAME_WAREHOUSE, + TABLENAME_DISTRICT, + TABLENAME_CUSTOMER, + TABLENAME_STOCK, + TABLENAME_ORDERS, + TABLENAME_NEW_ORDER, + TABLENAME_ORDER_LINE, + TABLENAME_HISTORY, +] + +# Transaction Types +def enum(*sequential, **named): + enums = dict(map(lambda x: (x, x), sequential)) + # dict(zip(sequential, range(len(sequential))), **named) + return type('Enum', (), enums) +TransactionTypes = enum( + "DELIVERY", + "NEW_ORDER", + "ORDER_STATUS", + "PAYMENT", + "STOCK_LEVEL", +) diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py b/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py new file mode 100755 index 00000000..cece61f3 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo & Yang Lu +# http:##www.cs.brown.edu/~pavlo/ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import sys +import os +import string +import datetime +import logging +import re +import argparse +import glob +import time +import pickle +import execnet +import worker +import message +from ConfigParser import SafeConfigParser +from pprint import pprint,pformat + +from util import * +from runtime import * +import drivers + +logging.basicConfig(level = logging.INFO, + format="%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s", + datefmt="%m-%d-%Y %H:%M:%S", + stream = sys.stdout) + +## ============================================== +## createDriverClass +## ============================================== +def createDriverClass(name): + full_name = "%sDriver" % name.title() + mod = __import__('drivers.%s' % full_name.lower(), globals(), locals(), [full_name]) + klass = getattr(mod, full_name) + return klass +## DEF + +## ============================================== +## getDrivers +## ============================================== +def getDrivers(): + drivers = [ ] + for f in map(lambda x: os.path.basename(x).replace("driver.py", ""), glob.glob("./drivers/*driver.py")): + if f != "abstract": drivers.append(f) + return (drivers) +## DEF + +## ============================================== +## startLoading +## ============================================== +def startLoading(scalParameters,args,config,channels): + #Split the warehouses into chunks + procs = len(channels) + w_ids = map(lambda x:[], range(procs)) + for w_id in range(scaleParameters.starting_warehouse, scaleParameters.ending_warehouse+1): + idx = w_id % procs + w_ids[idx].append(w_id) + print w_ids + + load_start=time.time() + for i in range(len(channels)): + m=message.Message(header=message.CMD_LOAD,data=[scalParameters,args,config,w_ids[i]]) + channels[i].send(pickle.dumps(m,-1)) + for ch in channels: + ch.receive() + pass + return time.time()-load_start + + +## ============================================== +## startExecution +## ============================================== +def startExecution(scaleParameters, args, config,channels): + procs = len(channels) + total_results = results.Results() + + for ch in channels: + m=message.Message(header=message.CMD_EXECUTE,data=[scaleParameters,args,config]) + ch.send(pickle.dumps(m,-1)) + for ch in channels: + r=pickle.loads(ch.receive()).data + total_results.append(r) + return (total_results) +## DEF + + +## ============================================== +## main +## ============================================== +if __name__ == '__main__': + aparser = argparse.ArgumentParser(description='Python implementation of the TPC-C Benchmark') + aparser.add_argument('system', choices=getDrivers(), + help='Target system driver') + aparser.add_argument('--config', type=file, + help='Path to driver configuration file') + aparser.add_argument('--reset', action='store_true', + help='Instruct the driver to reset the contents of the database') + aparser.add_argument('--scalefactor', default=1, type=float, metavar='SF', + help='Benchmark scale factor') + aparser.add_argument('--warehouses', default=4, type=int, metavar='W', + help='Number of Warehouses') + aparser.add_argument('--duration', default=60, type=int, metavar='D', + help='How long to run the benchmark in seconds') + aparser.add_argument('--ddl', default=os.path.realpath(os.path.join(os.path.dirname(__file__), "tpcc.sql")), + help='Path to the TPC-C DDL SQL file') + ## number of processes per node + aparser.add_argument('--clientprocs', default=1, type=int, metavar='N', + help='Number of processes on each client node.') + + aparser.add_argument('--stop-on-error', action='store_true', + help='Stop the transaction execution when the driver throws an exception.') + aparser.add_argument('--no-load', action='store_true', + help='Disable loading the data') + aparser.add_argument('--no-execute', action='store_true', + help='Disable executing the workload') + aparser.add_argument('--print-config', action='store_true', + help='Print out the default configuration file for the system and exit') + aparser.add_argument('--debug', action='store_true', + help='Enable debug log messages') + args = vars(aparser.parse_args()) + + if args['debug']: logging.getLogger().setLevel(logging.DEBUG) + + ## Arguments validation + assert args['reset'] == False or args['no_load'] == False, \ + "'--reset' and '--no-load' are incompatible with each other" + + ## Create a handle to the target client driver + driverClass = createDriverClass(args['system']) + assert driverClass != None, "Failed to find '%s' class" % args['system'] + driver = driverClass(args['ddl']) + assert driver != None, "Failed to create '%s' driver" % args['system'] + if args['print_config']: + config = driver.makeDefaultConfig() + print driver.formatConfig(config) + print + sys.exit(0) + + ## Load Configuration file + if args['config']: + logging.debug("Loading configuration file '%s'" % args['config']) + cparser = SafeConfigParser() + cparser.read(os.path.realpath(args['config'].name)) + config = dict(cparser.items(args['system'])) + else: + logging.debug("Using default configuration for %s" % args['system']) + defaultConfig = driver.makeDefaultConfig() + config = dict(map(lambda x: (x, defaultConfig[x][1]), defaultConfig.keys())) + config['reset'] = args['reset'] + config['load'] = False + config['execute'] = False + if config['reset']: logging.info("Reseting database") + driver.loadConfig(config) + logging.info("Initializing TPC-C benchmark using %s" % driver) + + + ##Get a list of clientnodes from configuration file. + clients=[] + channels=[] + assert config['clients']!='' + clients=re.split(r"\s+",str(config['clients'])) + #print clients, len(clients),args['clientprocs'] + ##Create ssh channels to client nodes + for node in clients: + cmd = 'ssh='+ node + cmd += r"//chdir=" + cmd += config['path'] + #print cmd + for i in range(args['clientprocs']): + gw=execnet.makegateway(cmd) + ch=gw.remote_exec(worker) + channels.append(ch) + + ## Create ScaleParameters + scaleParameters = scaleparameters.makeWithScaleFactor(args['warehouses'], args['scalefactor']) + nurand = rand.setNURand(nurand.makeForLoad()) + if args['debug']: logging.debug("Scale Parameters:\n%s" % scaleParameters) + + ## DATA LOADER!!! + load_time = None + if not args['no_load']: + load_time = startLoading(scaleParameters, args, config,channels) + #print load_time + ## IF + + ## WORKLOAD DRIVER!!! + if not args['no_execute']: + results = startExecution(scaleParameters, args, config,channels) + assert results + print results.show(load_time) + ## IF + +## MAIN \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/__init__.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/__init__.py new file mode 100644 index 00000000..792d6005 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/__init__.py @@ -0,0 +1 @@ +# diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/abstractdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/abstractdriver.py new file mode 100644 index 00000000..d4bc137f --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/abstractdriver.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +from datetime import datetime + +import constants + +## ============================================== +## AbstractDriver +## ============================================== +class AbstractDriver(object): + def __init__(self, name, ddl): + self.name = name + self.driver_name = "%sDriver" % self.name.title() + self.ddl = ddl + + def __str__(self): + return self.driver_name + + def makeDefaultConfig(self): + """This function needs to be implemented by all sub-classes. + It should return the items that need to be in your implementation's configuration file. + Each item in the list is a triplet containing: ( , , ) + """ + raise NotImplementedError("%s does not implement makeDefaultConfig" % (self.driver_name)) + + def loadConfig(self, config): + """Initialize the driver using the given configuration dict""" + raise NotImplementedError("%s does not implement loadConfig" % (self.driver_name)) + + def formatConfig(self, config): + """Return a formatted version of the config dict that can be used with the --config command line argument""" + ret = "# %s Configuration File\n" % (self.driver_name) + ret += "# Created %s\n" % (datetime.now()) + ret += "[%s]" % self.name + + for name in config.keys(): + desc, default = config[name] + if default == None: default = "" + ret += "\n\n# %s\n%-20s = %s" % (desc, name, default) + return (ret) + + def loadStart(self): + """Optional callback to indicate to the driver that the data loading phase is about to begin.""" + return None + + def loadFinish(self): + """Optional callback to indicate to the driver that the data loading phase is finished.""" + return None + + def loadFinishItem(self): + """Optional callback to indicate to the driver that the ITEM data has been passed to the driver.""" + return None + + def loadFinishWarehouse(self, w_id): + """Optional callback to indicate to the driver that the data for the given warehouse is finished.""" + return None + + def loadFinishDistrict(self, w_id, d_id): + """Optional callback to indicate to the driver that the data for the given district is finished.""" + return None + + def loadTuples(self, tableName, tuples): + """Load a list of tuples into the target table""" + raise NotImplementedError("%s does not implement loadTuples" % (self.driver_name)) + + def executeStart(self): + """Optional callback before the execution phase starts""" + return None + + def executeFinish(self): + """Callback after the execution phase finishes""" + return None + + def executeTransaction(self, txn, params): + """Execute a transaction based on the given name""" + + if constants.TransactionTypes.DELIVERY == txn: + result = self.doDelivery(params) + elif constants.TransactionTypes.NEW_ORDER == txn: + result = self.doNewOrder(params) + elif constants.TransactionTypes.ORDER_STATUS == txn: + result = self.doOrderStatus(params) + elif constants.TransactionTypes.PAYMENT == txn: + result = self.doPayment(params) + elif constants.TransactionTypes.STOCK_LEVEL == txn: + result = self.doStockLevel(params) + else: + assert False, "Unexpected TransactionType: " + txn + return result + + def doDelivery(self, params): + """Execute DELIVERY Transaction + Parameters Dict: + w_id + o_carrier_id + ol_delivery_d + """ + raise NotImplementedError("%s does not implement doDelivery" % (self.driver_name)) + + def doNewOrder(self, params): + """Execute NEW_ORDER Transaction + Parameters Dict: + w_id + d_id + c_id + o_entry_d + i_ids + i_w_ids + i_qtys + """ + raise NotImplementedError("%s does not implement doNewOrder" % (self.driver_name)) + + def doOrderStatus(self, params): + """Execute ORDER_STATUS Transaction + Parameters Dict: + w_id + d_id + c_id + c_last + """ + raise NotImplementedError("%s does not implement doOrderStatus" % (self.driver_name)) + + def doPayment(self, params): + """Execute PAYMENT Transaction + Parameters Dict: + w_id + d_id + h_amount + c_w_id + c_d_id + c_id + c_last + h_date + """ + raise NotImplementedError("%s does not implement doPayment" % (self.driver_name)) + + def doStockLevel(self, params): + """Execute STOCK_LEVEL Transaction + Parameters Dict: + w_id + d_id + threshold + """ + raise NotImplementedError("%s does not implement doStockLevel" % (self.driver_name)) +## CLASS \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/cassandradriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/cassandradriver.py new file mode 100644 index 00000000..3f597e17 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/cassandradriver.py @@ -0,0 +1,771 @@ +# -*- coding: utf-8 -*- +# copyright (C) 2011 +# Jingxin Feng, Xiaowei Wang +# jxfeng@cs.brown.edu +# xiaowei@cs.brown.edu +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + + +import pycassa +from pycassa.index import * +from pycassa.system_manager import * + +import os +import logging +import commands +import uuid +from pprint import pprint,pformat +import constants + +from abstractdriver import * +## ============================================== +## AbstractDriver +## ============================================== +class CassandraDriver(AbstractDriver): + + DEFAULT_CONFIG = { + "hostname": ("The host address to the Cassandra database","localhost"), + "port": ("Port number",9160), + "name": ("Name","tpcc"), + "keyspace":("Keyspace", "Keyspace1"), + "replicationfactor": ("ReplicationFactor", 1) + } + + + + def __init__(self, ddl): + super(CassandraDriver,self).__init__("cassandra",ddl) + self.conn = None + self.name = "cassandra" + self.database = None + + self.new_ordercf= None + self.orderscf= None + self.order_linecf= None + self.customercf = None + self.warehousecf = None + self.districtcf = None + self.historycf = None + self.stockcf = None + self.itemcf = None + def makeDefaultConfig(self): + return CassandraDriver.DEFAULT_CONFIG + + def loadConfig(self,config): + for key in CassandraDriver.DEFAULT_CONFIG.keys(): + assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) + + + connection = str(config["hostname"]+':'+str(config["port"])) + + + keyspace = str(config["keyspace"]) + self.sys = SystemManager(connection) + keyspaces = self.sys.list_keyspaces() + fl = 0 + for i in range(len(keyspaces)): + if str(keyspaces[i]) == keyspace: + fl = 1 + break + if fl == 0: + self.sys.create_keyspace(keyspace, SIMPLE_STRATEGY,{'replication_factor' : str(config["replicationfactor"])}) + self.sys.create_column_family(keyspace, 'NEW_ORDER', comparator_type = UTF8_TYPE) + self.sys.create_column_family(keyspace, 'ORDERS', comparator_type = UTF8_TYPE) + self.sys.create_column_family(keyspace, 'ORDER_LINE', comparator_type = UTF8_TYPE) + self.sys.create_column_family(keyspace, 'CUSTOMER', comparator_type = UTF8_TYPE) + self.sys.create_column_family(keyspace, 'WAREHOUSE', comparator_type = UTF8_TYPE) + self.sys.create_column_family(keyspace, 'DISTRICT', comparator_type = UTF8_TYPE) + self.sys.create_column_family(keyspace, 'HISTORY', comparator_type = UTF8_TYPE) + self.sys.create_column_family(keyspace, 'STOCK', comparator_type = UTF8_TYPE) + self.sys.create_column_family(keyspace, 'ITEM', comparator_type = UTF8_TYPE) + + + self.sys.alter_column(keyspace,'WAREHOUSE','W_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'WAREHOUSE','W_NAME',UTF8_TYPE) + self.sys.alter_column(keyspace,'WAREHOUSE','W_STREET_1',UTF8_TYPE) + self.sys.alter_column(keyspace,'WAREHOUSE','W_STREET_2',UTF8_TYPE) + self.sys.alter_column(keyspace,'WAREHOUSE','W_CITY',UTF8_TYPE) + self.sys.alter_column(keyspace,'WAREHOUSE','W_STATE',UTF8_TYPE) + self.sys.alter_column(keyspace,'WAREHOUSE','W_ZIP',UTF8_TYPE) + self.sys.alter_column(keyspace,'WAREHOUSE','W_TAX',UTF8_TYPE) + self.sys.alter_column(keyspace,'WAREHOUSE','W_YTD',UTF8_TYPE) + + + self.sys.alter_column(keyspace,'DISTRICT','D_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'DISTRICT','D_W_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'DISTRICT','D_NAME',UTF8_TYPE) + self.sys.alter_column(keyspace,'DISTRICT','D_STREET_1',UTF8_TYPE) + self.sys.alter_column(keyspace,'DISTRICT','D_STREET_2',UTF8_TYPE) + self.sys.alter_column(keyspace,'DISTRICT','D_CITY',UTF8_TYPE) + self.sys.alter_column(keyspace,'DISTRICT','D_STATE',UTF8_TYPE) + self.sys.alter_column(keyspace,'DISTRICT','D_ZIP',UTF8_TYPE) + self.sys.alter_column(keyspace,'DISTRICT','D_TAX',UTF8_TYPE) + self.sys.alter_column(keyspace,'DISTRICT','D_YTD',UTF8_TYPE) + self.sys.alter_column(keyspace,'DISTRICT','D_NEXT_O_ID',UTF8_TYPE) + + + self.sys.alter_column(keyspace,'CUSTOMER','C_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_D_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_W_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_FIRST',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_MIDDLE',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_LAST',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_STREET_1',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_STREET_2',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_CITY',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_STATE',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_ZIP',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_PHONE',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_SINCE',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_CREDIT',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_CREDIT_LIM',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_DISCOUNT',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_BALANCE',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_YTD_PAYMENT',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_PAYMENT_CNT',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_DELIVERY_CNT',UTF8_TYPE) + self.sys.alter_column(keyspace,'CUSTOMER','C_DATA',UTF8_TYPE) + + + + + + + + + self.sys.alter_column(keyspace,'HISTORY','H_C_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'HISTORY','H_C_D_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'HISTORY','H_C_W_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'HISTORY','H_D_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'HISTORY','H_W_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'HISTORY','H_DATE',UTF8_TYPE) + self.sys.alter_column(keyspace,'HISTORY','H_AMOUNT',UTF8_TYPE) + self.sys.alter_column(keyspace,'HISTORY','H_DATA',UTF8_TYPE) + + + + self.sys.alter_column(keyspace,'NEW_ORDER','NO_O_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'NEW_ORDER','NO_D_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'NEW_ORDER','NO_W_ID',UTF8_TYPE) + + self.sys.alter_column(keyspace,'ORDERS','O_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDERS','O_D_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDERS','O_W_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDERS','O_C_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDERS','O_ENTRY_D',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDERS','O_CARRIER_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDERS','O_OL_CNT',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDERS','O_ALL_LOCAL',UTF8_TYPE) + + + + self.sys.alter_column(keyspace,'ORDER_LINE','OL_O_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDER_LINE','OL_D_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDER_LINE','OL_W_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDER_LINE','OL_NUMBER',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDER_LINE','OL_I_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDER_LINE','OL_SUPPLY_W_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDER_LINE','OL_DELIVERY_D',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDER_LINE','OL_QUANTITY',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDER_LINE','OL_AMOUNT',UTF8_TYPE) + self.sys.alter_column(keyspace,'ORDER_LINE','OL_DIST_INFO',UTF8_TYPE) + + + self.sys.alter_column(keyspace,'ITEM','I_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ITEM','I_IM_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'ITEM','I_NAME',UTF8_TYPE) + self.sys.alter_column(keyspace,'ITEM','I_PRICE',UTF8_TYPE) + self.sys.alter_column(keyspace,'ITEM','I_DATA',UTF8_TYPE) + + + self.sys.alter_column(keyspace,'STOCK','S_I_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_W_ID',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_QUANTITY',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DIST_01',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DIST_02',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DIST_03',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DIST_04',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DIST_05',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DIST_06',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DIST_07',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DIST_08',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DIST_09',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DIST_10',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_YTD',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_ORDER_CNT',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_REMOTE_CNT',UTF8_TYPE) + self.sys.alter_column(keyspace,'STOCK','S_DATA',UTF8_TYPE) + + + self.sys.create_index(keyspace,'CUSTOMER','C_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'CUSTOMER','C_D_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'CUSTOMER','C_W_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'CUSTOMER','C_LAST', UTF8_TYPE) + self.sys.create_index(keyspace,'NEW_ORDER','NO_O_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'NEW_ORDER','NO_D_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'NEW_ORDER','NO_W_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'ORDERS','O_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'ORDERS','O_D_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'ORDERS','O_W_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'ORDERS','O_C_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'ORDER_LINE','OL_O_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'ORDER_LINE','OL_D_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'ORDER_LINE','OL_W_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'STOCK','S_W_ID', UTF8_TYPE) + self.sys.create_index(keyspace,'STOCK','S_QUANTITY', UTF8_TYPE) + + + self.conn = pycassa.connect(str(config["keyspace"]),[connection]) + self.new_ordercf=pycassa.ColumnFamily(self.conn,'NEW_ORDER') + self.orderscf=pycassa.ColumnFamily(self.conn, 'ORDERS') + self.order_linecf=pycassa.ColumnFamily(self.conn, 'ORDER_LINE') + self.customercf=pycassa.ColumnFamily(self.conn, 'CUSTOMER') + self.warehousecf = pycassa.ColumnFamily(self.conn,'WAREHOUSE') + self.districtcf = pycassa.ColumnFamily(self.conn, 'DISTRICT') + self.historycf = pycassa.ColumnFamily(self.conn,'HISTORY') + self.stockcf = pycassa.ColumnFamily(self.conn,'STOCK') + self.itemcf = pycassa.ColumnFamily(self.conn, 'ITEM') + + def loadTuples(self, tableName, tuples): + if len(tuples) == 0: + return + logging.debug("loading") + col_fam = pycassa.ColumnFamily(self.conn, tableName) + if tableName == 'ITEM': + for row in tuples: + row_key = str(row[0]).zfill(5) + i_id = str(row[0]) + i_im_id = str(row[1]) + i_name = str(row[2]) + i_price = str(row[3]) + i_data = str(row[4]) + col_fam.insert(row_key, {'I_ID':i_id, 'I_IM_ID':i_im_id, 'I_NAME':i_name, 'I_PRICE':i_price, 'I_DATA':i_data}) + if tableName == 'WAREHOUSE': + if len(tuples[0])!=9: return + for row in tuples: + row_key = str(row[0]).zfill(5) #w_ID + w_id = str(row[0]) + w_name =str(row[1]) + w_street_1 = str(row[2]) + w_street_2 = str(row[3]) + w_city = str(row[4]) + w_state = str(row[5]) + w_zip = str(row[6]) + w_tax = str(row[7]) + w_ytd = str(row[8]) + col_fam.insert(row_key,{'W_ID':w_id, 'W_NAME':w_name, 'W_STREET_1': w_street_1, 'W_STREET_2': w_street_2, 'W_CITY':w_city, 'W_STATE':w_state, 'W_ZIP':w_zip, 'W_TAX':w_tax, 'W_YTD':w_ytd}) + if tableName == 'CUSTOMER': + for row in tuples: + row_key = str(row[0]).zfill(5)+ str(row[1]).zfill(5)+ str(row[2]).zfill(5) + c_id = str(row[0]) + c_d_id =str(row[1]) + c_w_id =str(row[2]) + c_first =str(row[3]) + c_middle = str(row[4]) + c_last = str(row[5]) + c_street_1 = str(row[6]) + c_street_2 = str(row[7]) + c_city = str(row[8]) + c_state = str(row[9]) + c_zip = str(row[10]) + c_phone = str(row[11]) + c_since = str(row[12]) + c_credit = str(row[13]) + c_credit_lim = str(row[14]) + c_discount = str(row[15]) + c_balance = str(row[16]) + c_ytd_payment = str(row[17]) + c_payment_cnt = str(row[18]) + c_delivery_cnt = str(row[19]) + c_data = str(row[20]) + col_fam.insert(row_key, {'C_ID':c_id, 'C_D_ID':c_d_id, 'C_W_ID':c_w_id, 'C_FIRST':c_first, 'C_MIDDLE':c_middle, 'C_LAST':c_last, 'C_STREET_1':c_street_1,'C_STREET_2':c_street_2, 'C_CITY':c_city, 'C_STATE':c_state, 'C_ZIP':c_zip, 'C_PHONE':c_phone, 'C_SINCE':c_since, 'C_CREDIT':c_credit, 'C_CREDIT_LIM':c_credit_lim, 'C_DISCOUNT':c_discount, 'C_BALANCE':c_balance, 'C_YTD_PAYMENT':c_ytd_payment, 'C_PAYMENT_CNT':c_payment_cnt, 'C_DELIVERY_CNT':c_delivery_cnt, 'C_DATA':c_data}) + + if tableName == 'ORDERS': + for row in tuples: + row_key = str(row[0]).zfill(5)+str(row[1]).zfill(5)+ str(row[2]).zfill(5) + o_id = str(row[0]) + o_d_id = str(row[1]) + o_w_id = str(row[2]) + o_c_id = str(row[3]) + o_entry_d = str(row[4]) + o_carrier_id = str(row[5]) + o_ol_cnt = str(row[6]) + o_all_local = str(row[7]) + col_fam.insert(row_key,{'O_ID':o_id, 'O_D_ID':o_d_id, 'O_W_ID':o_w_id, 'O_C_ID':o_c_id, 'O_ENTRY_D':o_entry_d, 'O_CARRIER_ID':o_carrier_id, 'O_OL_CNT':o_ol_cnt, 'O_ALL_LOCAL':o_all_local}) + + + if tableName == 'STOCK': + for row in tuples: + row_key = str(row[0]).zfill(5)+str(row[1]).zfill(5) + s_i_id = str(row[0]) + s_w_id = str(row[1]) + s_quantity = str(row[2]) + s_dist_01 = str(row[3]) + s_dist_02 = str(row[4]) + s_dist_03 = str(row[5]) + s_dist_04 = str(row[6]) + s_dist_05 = str(row[7]) + s_dist_06 = str(row[8]) + s_dist_07 = str(row[9]) + s_dist_08 = str(row[10]) + s_dist_09 = str(row[11]) + s_dist_10 = str(row[12]) + s_ytd = str(row[13]) + s_order_cnt = str(row[14]) + s_remote_cnt = str(row[15]) + s_data = str(row[16]) + col_fam.insert(row_key,{'S_I_ID':s_i_id, 'S_W_ID':s_w_id, 'S_QUANTITY':s_quantity, 'S_DIST_01':s_dist_01,'S_DIST_02':s_dist_02,'S_DIST_03':s_dist_03,'S_DIST_04':s_dist_04,'S_DIST_05':s_dist_05,'S_DIST_06':s_dist_06,'S_DIST_07':s_dist_07,'S_DIST_08':s_dist_08,'S_DIST_09':s_dist_09,'S_DIST_10':s_dist_10, 'S_YTD': s_ytd, 'S_ORDER_CNT':s_order_cnt, 'S_REMOTE_CNT':s_remote_cnt, 'S_DATA':s_data}) + + if tableName == 'DISTRICT': + for row in tuples: + row_key = str(row[0]).zfill(5)+str(row[1]).zfill(5) + d_id = str(row[0]) + d_w_id = str(row[1]) + d_name = str(row[2]) + d_street_1 = str(row[3]) + d_street_2 = str(row[4]) + d_city = str(row[5]) + d_state = str(row[6]) + d_zip = str(row[7]) + d_tax =str(row[8]) + d_ytd = str(row[9]) + d_next_o_id = str(row[10]) + col_fam.insert(row_key,{'D_ID':d_id, 'D_W_ID':d_w_id, 'D_NAME':d_name, 'D_STREET_1':d_street_1, 'D_STREET_2':d_street_2,'D_CITY':d_city, 'D_STATE':d_state, 'D_ZIP':d_zip, 'D_TAX':d_tax, 'D_YTD':d_ytd, 'D_NEXT_O_ID':d_next_o_id}) + + if tableName == 'NEW_ORDER': + for row in tuples: + row_key = str(row[0]).zfill(5)+str(row[1]).zfill(5)+str(row[2]).zfill(5) + no_o_id = str(row[0]) + no_d_id = str(row[1]) + no_w_id = str(row[2]) + col_fam.insert(row_key,{'NO_O_ID':no_o_id, 'NO_D_ID':no_d_id, 'NO_W_ID':no_w_id}) + if tableName == 'ORDER_LINE': + for row in tuples: + row_key = str(row[0]).zfill(5)+str(row[1]).zfill(5)+str(row[2]).zfill(5)+str(row[3]).zfill(5) + ol_o_id = str(row[0]) + ol_d_id = str(row[1]) + ol_w_id = str(row[2]) + ol_number = str(row[3]) + ol_i_id = str(row[4]) + ol_supply_w_id = str(row[5]) + ol_delivery_d = str(row[6]) + ol_quantity = str(row[7]) + ol_amount = str(row[8]) + ol_dist_info = str(row[9]) + col_fam.insert(row_key,{'OL_O_ID':ol_o_id, 'OL_D_ID':ol_d_id, 'OL_W_ID':ol_w_id, 'OL_NUMBER':ol_number, 'OL_I_ID':ol_i_id, 'OL_SUPPLY_W_ID':ol_supply_w_id, 'OL_DELIVERY_D': ol_delivery_d, 'OL_QUANTITY':ol_quantity,'OL_AMOUNT':ol_amount, 'OL_DIST_INFO':ol_dist_info}) + + if tableName == 'HISTORY': + for i in range(len(tuples)): + #row_key = str(i) + row_key = str(uuid.uuid1()) + h_c_id = str(tuples[i][0]) + h_c_d_id = str(tuples[i][1]) + h_c_w_id = str(tuples[i][2]) + h_d_id = str(tuples[i][3]) + h_w_id = str(tuples[i][4]) + h_date = str(tuples[i][5]) + h_amount = str(tuples[i][6]) + h_data = str(tuples[i][7]) + col_fam.insert(row_key, {'H_C_ID':h_c_id, 'H_C_D_ID':h_c_d_id, 'H_C_W_ID':h_c_w_id, 'H_D_ID':h_d_id, 'H_W_ID':h_w_id, 'H_DATE':h_date,'H_AMOUNT':h_amount, 'H_DATA':h_data}) +# print tableName+'--' + str(len(tuples)) + + def loadFinish(self): + logging.info("Commiting changes to database") + + + + + ##----------------------------------- + ## doDelivery + ##---------------------------------- + def doDelivery(self, params): + logging.debug("do delivery") + + w_id = params["w_id"] + o_carrier_id = params["o_carrier_id"] + ol_delivery_d = params["ol_delivery_d"] + + + + result = [ ] + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): + did_expr = create_index_expression('NO_D_ID',str(d_id)) + wid_expr = create_index_expression('NO_W_ID',str(w_id)) + clause = create_index_clause([did_expr,wid_expr],count=1) + newOrder=self.new_ordercf.get_indexed_slices(clause) + flag=0 + for key, column in newOrder: + #print column + no_o_id=column['NO_O_ID'] + flag=1 + if flag==0: + continue + if int(no_o_id)<=-1: + continue + if no_o_id==None: + continue + # print no_o_id + # print d_id + # print w_id + orders_rowkey=no_o_id.zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) + #print orders_rowkey + o=self.orderscf.get(orders_rowkey) + + + c_id=str(o['O_C_ID']) + + oid_expr = create_index_expression('OL_O_ID',str(no_o_id)) + did_expr = create_index_expression('OL_D_ID',str(d_id)) + wid_expr = create_index_expression('OL_W_ID',str(w_id)) + + clause = create_index_clause([oid_expr,did_expr,wid_expr],count=100000) + orderLine=self.order_linecf.get_indexed_slices(clause) + + ol_total=0 + for key, column in orderLine: + ol_total+=float(column['OL_AMOUNT']) + + deleteKey=no_o_id.zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) + self.new_ordercf.remove(deleteKey) + self.orderscf.insert(deleteKey, {'O_CARRIER_ID': str(o_carrier_id)}) + self.order_linecf.insert(deleteKey,{'OL_DELIVERY_D':str(ol_delivery_d)}) + + c=self.customercf.get(str(c_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5)) + old_balance=float(c['C_BALANCE']) + new_balance=str(old_balance+ol_total) + self.customercf.insert(str(c_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5),{'C_BALANCE': str(new_balance)}) + + result.append((str(d_id),str(no_o_id))) + ##for + + return result + ##----------------------------------- + ## doNewOrder + ##----------------------------------- + + def doNewOrder(self, params): + logging.debug("do new order") + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + o_entry_d = params["o_entry_d"] + i_ids = params["i_ids"] + i_w_ids = params["i_w_ids"] + i_qtys = params["i_qtys"] + + + assert len(i_ids) > 0 + assert len(i_ids) ==len(i_w_ids) + assert len(i_ids) ==len(i_qtys) + + all_local = True + items = [ ] + for i in range(len(i_ids)): + all_local = all_local and i_w_ids[i] == w_id + ol_i_id = i_ids[i] + itm=self.itemcf.get(str(ol_i_id).zfill(5), columns=['I_PRICE','I_NAME','I_DATA']) + items.append(itm) + assert len(items)==len(i_ids) + + + for itm in items: + if len(itm)==0: + return + + + + + + #getWarehouseTaxRate + w_tax_c = self.warehousecf.get(str(w_id).zfill(5),columns=['W_TAX']) + w_tax =float(w_tax_c['W_TAX']) + #getDistrict + row_key = str(d_id).zfill(5) +str(w_id).zfill(5) + o=self.districtcf.get(row_key, columns=['D_TAX','D_NEXT_O_ID']) + d_tax = float(o['D_TAX']) + #incrementNextOrderId + d_next_o_id = int(o['D_NEXT_O_ID']) + + #getCustomer + row_key = str(c_id).zfill(5) +str(d_id).zfill(5)+str(w_id).zfill(5) + customer_info = self.customercf.get(row_key,columns=['C_DISCOUNT','C_LAST','C_CREDIT']) + c_discount=float(customer_info['C_DISCOUNT']) + + o_carrier_id = constants.NULL_CARRIER_ID + ol_cnt = len(i_ids) + + #incrementNextOrderId + row_key = str(d_id).zfill(5)+str(w_id).zfill(5) + self.districtcf.insert(row_key,{'D_NEXT_O_ID':str(d_next_o_id+1)}) + + #createOrder + + order_rowkey=str(d_next_o_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) + # print "d_next_o_id " +str(d_next_o_id) + # print "d_id "+str(d_id) + # print "order_rowkey " + order_rowkey + self.orderscf.insert(order_rowkey,{'O_ID':str(d_next_o_id), 'O_D_ID':str(d_id), 'O_W_ID':str(w_id), 'O_C_ID':str(c_id), 'O_ENTRY_D':str(o_entry_d), 'O_CARRIER_ID':str(o_carrier_id), 'O_OL_CNT':str(ol_cnt), 'O_ALL_LOCAL':str(all_local)}) + + #createNewOrder + neworder_rowkey=str(d_next_o_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) + # print 'neworder_rowkey ' + neworder_rowkey + self.new_ordercf.insert(neworder_rowkey, {'NO_O_ID':str(d_next_o_id), 'NO_D_ID':str(d_id), 'NO_W_ID':str(w_id)}) + #getItemInfo + total = 0 + item_data = [ ] + for i in range(len(i_ids)): + itemInfo = items[i] + i_name = itemInfo['I_NAME'] + i_data = itemInfo['I_DATA'] + i_price =float(itemInfo['I_PRICE']) + + #"getStockInfo": "SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DIST_%02d FROM STOCK WHERE S_I_ID = ? AND S_W_ID = ?", # d_id, ol_i_id, ol_supply_w_id + ol_i_id = i_ids[i] + ol_number = i+1 + ol_supply_w_id = i_w_ids[i] + ol_quantity = i_qtys[i] + + stockInfo = self.stockcf.get(str(ol_i_id).zfill(5)+str(ol_supply_w_id).zfill(5)) + #"updateStock": "UPDATE STOCK SET S_QUANTITY = ?, S_YTD = ?, S_ORDER_CNT = ?, S_REMOTE_CNT = ? WHERE S_I_ID = ? AND S_W_ID = ?", # s_quantity, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id + if len(stockInfo)==0: + logging.warn("No STOCK record for (ol_i_id=%d, ol_supply_w_id=%d)" % (ol_i_id, ol_supply_w_id)) + continue + s_quantity = int(stockInfo['S_QUANTITY']) + s_ytd = int(stockInfo['S_YTD']) + s_order_cnt = int(stockInfo['S_ORDER_CNT']) + s_remote_cnt = int(stockInfo['S_REMOTE_CNT']) + s_data = stockInfo['S_DATA'] + if d_id < 10: + s_dist_col='S_DIST_'+'0'+str(d_id) + else: + s_dist_col='S_DIST_'+str(d_id) + s_dist_xx = stockInfo[s_dist_col] + + + ## Update stock + s_ytd += ol_quantity + if s_quantity >= ol_quantity + 10: + s_quantity = s_quantity - ol_quantity + else: + s_quantity = s_quantity + 91 - ol_quantity + s_order_cnt += 1 + if ol_supply_w_id != w_id: s_remote_cnt += 1 + self.stockcf.insert(str(ol_i_id).zfill(5)+str(ol_supply_w_id).zfill(5),{'S_QUANTITY':str(s_quantity), 'S_YTD':str(s_ytd), 'S_ORDER_CNT':str(s_order_cnt) , 'S_REMOTE_CNT':str(s_remote_cnt)}) + + ##"createOrderLine": "INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", # o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info + if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING)!= -1: + brand_generic = 'B' + else: + brand_generic = 'G' + ol_amount = ol_quantity * i_price + total += ol_amount + + orderline_rowkey=str(d_next_o_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) + self.order_linecf.insert(orderline_rowkey,{'OL_O_ID': str(d_next_o_id), 'OL_D_ID':str(d_id), 'OL_W_ID':str(w_id), 'OL_NUMBER':str(ol_number), 'OL_I_ID':str(ol_i_id), 'OL_SUPPLY_W_ID':str(ol_supply_w_id), 'OL_DELIVERY_D': str(o_entry_d), 'OL_QUANTITY':str(ol_quantity),'OL_AMOUNT':str(ol_amount), 'OL_DIST_INFO':str(s_dist_xx)}) + item_data.append( (i_name, s_quantity, brand_generic,i_price, ol_amount) ) + total *= (1 - c_discount) * (1 + w_tax + d_tax) + misc = [ (w_tax, d_tax, d_next_o_id, total) ] + return [ customer_info, misc, item_data ] + ##---------------------------- + ## doPayment + ##---------------------------- + + def doPayment(self, params): + logging.debug("do payment") + w_id = params["w_id"] + d_id = params["d_id"] + h_amount = params["h_amount"] + c_w_id = params["c_w_id"] + c_d_id = params["c_d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + h_date = params["h_date"] + + + if c_id != None: + #getCustomerByCustomerId + row_key = str(c_id).zfill(5) +str(d_id).zfill(5)+str(w_id).zfill(5) + customer = self.customercf.get(row_key) + assert len(customer)>0 + c_balance = float(str(customer['C_BALANCE']))- h_amount + c_ytd_payment = float(str(customer['C_YTD_PAYMENT'])) + h_amount + c_payment_cnt = int(str(customer['C_PAYMENT_CNT']))+1 + c_data = str(customer['C_DATA']) + c_credit = str(customer['C_CREDIT']) + else: + #getCustomerByLastName + c_1_expr = create_index_expression('C_W_ID',str(w_id)) + c_2_expr = create_index_expression('C_D_ID',str(d_id)) + c_3_expr = create_index_expression('C_LAST',str(c_last)) + clause = create_index_clause([c_1_expr,c_2_expr,c_3_expr],count=1000) + + newcustomer = self.customercf.get_indexed_slices(clause) + firstnames=[] + + namecnt=0 + for key, column in newcustomer: + firstnames.append(column['C_FIRST']) + namecnt+=1 + # print namecnt + index = (namecnt-1)/2 + firstname=firstnames[index] + c_4_expr = create_index_expression('C_LAST',str(c_last)) + clause = create_index_clause([c_1_expr,c_2_expr,c_3_expr,c_4_expr],count=1) + newcustomer = self.customercf.get_indexed_slices(clause) + for key, column in newcustomer: + c_id = column['C_ID'] + c_balance = float(column['C_BALANCE'])- h_amount + c_ytd_payment = float(column['C_YTD_PAYMENT']) + h_amount + c_payment_cnt = int(column['C_PAYMENT_CNT'])+1 + c_data = column['C_DATA'] + c_credit =column['C_CREDIT'] + row_key = str(c_id).zfill(5) +str(d_id).zfill(5)+str(w_id).zfill(5) + customer = self.customercf.get(row_key) + warehouse = self.warehousecf.get(str(w_id).zfill(5)) + district = self.districtcf.get(str(d_id).zfill(5)+str(w_id).zfill(5)) + + self.warehousecf.insert(str(w_id).zfill(5),{'W_YTD':str(float(warehouse['W_YTD'])+h_amount)}) + + self.districtcf.insert(str(d_id).zfill(5)+str(w_id).zfill(5),{'D_YTD': str(float(district['D_YTD'])+h_amount)}) + + if c_credit == constants.BAD_CREDIT: + newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) + c_data = (newData + "|" + c_data) + if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] + self.customercf.insert(str(c_id).zfill(5)+str(c_d_id).zfill(5)+str(c_w_id).zfill(5),{ 'C_BALANCE' : str(c_balance), 'C_YTD_PAYMENT':str(c_ytd_payment) , 'C_PAYMENT_CNT':str(c_payment_cnt), 'C_DATA' : str(c_data)}) + else: + c_data = "" + self.customercf.insert(str(c_id).zfill(5)+str(c_d_id).zfill(5)+str(c_w_id).zfill(5),{ 'C_BALANCE' : str(c_balance), 'C_YTD_PAYMENT':str(c_ytd_payment) , 'C_PAYMENT_CNT':str(c_payment_cnt)}) + h_data= "%s %s" % (warehouse['W_NAME'], district['D_NAME']) + self.historycf.insert(str(uuid.uuid1()), {'H_C_ID':str(c_id), 'H_C_D_ID':str(c_d_id), 'H_C_W_ID':str(c_w_id), 'H_D_ID':str(d_id), 'H_W_ID':str(w_id), 'H_DATE':str(h_date),'H_AMOUNT':str(h_amount), 'H_DATA':str(h_data)}) + return [warehouse, district, customer] + ##----------------------------------- + ## doOrderStatus + ##----------------------------------- + def doOrderStatus(self, params): + logging.info("do orderStatus") + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + + assert w_id, pformat(params) + assert d_id, pformat(params) + + + if c_id == None: + last_expr = create_index_expression('C_LAST',str(c_last)) + did_expr = create_index_expression('C_D_ID',str(d_id)) + wid_expr = create_index_expression('C_W_ID',str(w_id)) + clause = create_index_clause([last_expr,did_expr,wid_expr],count=10000) + all_customers=self.customercf.get_indexed_slices(clause) + first_names=[ ] + c_ids=[] + namecnt=0 + for key, column in all_customers: + first_names.append(column['C_FIRST']) + c_ids.append(column['C_ID']) + namecnt = namecnt+1 + namecnt=len(first_names) + assert namecnt>0 + index=(namecnt-1)/2 + first_name=first_names[index] + assert first_name!=None + c_id=c_ids[index] + assert c_id!=None + + key1=str(c_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) + res1=self.customercf.get(key1) + customer=[res1['C_ID'],res1['C_FIRST'],res1['C_MIDDLE'],res1['C_LAST'],res1['C_BALANCE']] + + cid_expr = create_index_expression('O_C_ID',str(c_id)) + did_expr = create_index_expression('O_D_ID',str(d_id)) + wid_expr = create_index_expression('O_W_ID',str(w_id)) + clause = create_index_clause([cid_expr,did_expr,wid_expr],count=100000) + all_orders=self.orderscf.get_indexed_slices(clause) + + last_order_oid=0 + order=[] + for key, column in all_orders: + if int(column['O_ID'])>last_order_oid: + last_order_oid=int(column['O_ID']) + if last_order_oid>0: + o=self.orderscf.get(str(last_order_oid).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5)) + order=[o['O_ID'],o['O_CARRIER_ID'],o['O_ENTRY_D']] + + + orderLines = [] + if last_order_oid>0: + oid_expr = create_index_expression('OL_O_ID',str(last_order_oid)) + did_expr = create_index_expression('OL_D_ID',str(d_id)) + wid_expr = create_index_expression('OL_W_ID',str(w_id)) + clause = create_index_clause([oid_expr,did_expr,wid_expr]) + orderLine=self.order_linecf.get_indexed_slices(clause) + for key, column in orderLine: + orderLines.append([column['OL_SUPPLY_W_ID'],column['OL_I_ID'],column['OL_QUANTITY'],column['OL_AMOUNT'],column['OL_DELIVERY_D']]) + + return [ customer, order, orderLines ] + + ##---------------------------- + ## doStockLevel + ##---------------------------- + + + def doStockLevel(self, params): + logging.info("do stocklevel") + w_id = params["w_id"] + d_id = params["d_id"] + threshold = params["threshold"] + + + #"getOId": "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", + d = self.districtcf.get(str(d_id).zfill(5)+str(w_id).zfill(5),columns=['D_NEXT_O_ID']) + assert d + #getStockCount + o_id = d['D_NEXT_O_ID'] + + + s_q_expr = create_index_expression('S_QUANTITY',str(threshold), LT) + s_q_expr2 = create_index_expression('S_W_ID',str(w_id)) + clause = create_index_clause([s_q_expr,s_q_expr2]) + newstock = self.stockcf.get_indexed_slices(clause) + + + ol_expr = create_index_expression('OL_W_ID',str(w_id)) + ol_expr2 = create_index_expression('OL_D_ID',str(d_id)) + ol_expr3 = create_index_expression('OL_O_ID',str(o_id),LT) + ol_expr4 = create_index_expression('OL_O_ID', str(int(o_id)-20),GTE) + clause2 = create_index_clause([ol_expr,ol_expr2]) + neworderline = self.order_linecf.get_indexed_slices(clause2) + + count = 0 + for key, column in newstock: + for key2, column2 in neworderline: + tmp1 = column['S_I_ID'] + s_i_id = int(tmp1) + tmp2 = column2['OL_I_ID'] + ol_i_id = int(tmp2) + if s_i_id == ol_i_id: + count= count+1 + + return count + diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/couchdbdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/couchdbdriver.py new file mode 100644 index 00000000..0f1aab05 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/couchdbdriver.py @@ -0,0 +1,868 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Alex Kalinin +# http://www.cs.brown.edu/~akalinin/ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import logging +from pprint import pformat + +import constants +from abstractdriver import * + +import couchdb +from uuid import uuid4 + +# for parallel view fetching +import threading + +# This describes our scheme: +# db -- the name of the corresponding db in CouchDB (we're using table per database approach) +# attrs -- attributes from the table (will become keys in JSON documents, one document per row) +# prim_key -- primary key from the original table (after being concatenated will become _id in the JSON document) +# distr_key -- defines sharding key (sharding is done in a round-robin manner) +# indexes -- will become CouchDB views (should be seen as CREATE INDEX in SQL; we maintain secondary indexes this way) +# +# To sum up: +# -- We use one CouchDB database per table, one JSON document per row, one key/value per column/value. +# -- Secondary indexes are emulated through CouchDB views. +# +TPCC_SCM = { + "WAREHOUSE": { + "db" : "warehouse", + "attrs" : ["W_ID", "W_NAME", "W_STREET_1", "W_STREET_2", "W_CITY", "W_STATE", "W_ZIP", "W_TAX", "W_YTD"], + "prim_key" : ["W_ID"], + "distr_key" : "W_ID", + }, + "DISTRICT": { + "db" : "district", + "attrs" : ["D_ID", "D_W_ID", "D_NAME", "D_STREET_1", "D_STREET_2", "D_CITY", "D_STATE", "D_ZIP", "D_TAX", + "D_YTD", "D_NEXT_O_ID"], + "prim_key" : ["D_W_ID", "D_ID"], + "distr_key" : "D_W_ID", + }, + "ITEM" : { + "db" : "item", + "attrs" : ["I_ID", "I_IM_ID", "I_NAME", "I_PRICE", "I_DATA"], + "prim_key" : ["I_ID"] + }, + "CUSTOMER": { + "db" : "customer", + "attrs" : ["C_ID", "C_D_ID", "C_W_ID", "C_FIRST", "C_MIDDLE", "C_LAST", "C_STREET_1", "C_STREET_2", + "C_CITY", "C_STATE", "C_ZIP", "C_PHONE", "C_SINCE", "C_CREDIT", "C_CREDIT_LIM", + "C_DISCOUNT", "C_BALANCE", "C_YTD_PAYMENT", "C_PAYMENT_CNT", "C_DELIVERY_CNT", "C_DATA"], + "prim_key" : ["C_W_ID" ,"C_D_ID", "C_ID"], + "distr_key" : "C_W_ID", + "indexes" : + { + "w_d_last" : { + "map" : """ + function(doc) { + emit([doc.C_W_ID, doc.C_D_ID, doc.C_LAST], doc.C_FIRST); + } + """, + }, + } + }, + "HISTORY" : { + "db" : "history", + "attrs" : ["H_C_ID", "H_C_D_ID", "H_C_W_ID", "H_D_ID", "H_W_ID", "H_DATE", "H_AMOUNT", "H_DATA"], + "prim_key" : [], + "distr_key" : "H_C_W_ID", + }, + "STOCK" : { + "db" : "stock", + "attrs" : ["S_I_ID", "S_W_ID", "S_QUANTITY", "S_DIST_01", "S_DIST_02", "S_DIST_03", "S_DIST_04", "S_DIST_05", + "S_DIST_06", "S_DIST_07", "S_DIST_08", "S_DIST_09", "S_DIST_10", "S_YTD", "S_ORDER_CNT", + "S_REMOTE_CNT", "S_DATA"], + "prim_key" : ["S_W_ID", "S_I_ID"], + "distr_key" : "S_W_ID", + "indexes" : + { + "w_i" : { + "map" : """ + function(doc) { + emit([doc.S_W_ID, doc.S_I_ID], doc.S_QUANTITY); + } + """, + }, + } + }, + "ORDERS" : { + "db" : "orders", + "attrs" : ["O_ID", "O_C_ID", "O_D_ID", "O_W_ID", "O_ENTRY_D", "O_CARRIER_ID", "O_OL_CNT", "O_ALL_LOCAL"], + "prim_key" : ["O_W_ID", "O_D_ID", "O_ID"], + "distr_key" : "O_W_ID", + "indexes" : + { + "w_d_c_o" : { + "map" : """ + function(doc) { + emit([doc.O_W_ID, doc.O_D_ID, doc.O_C_ID, doc.O_ID], null); + } + """, + }, + } + }, + "NEW_ORDER": { + "db" : "new_order", + "attrs" : ["NO_O_ID", "NO_D_ID", "NO_W_ID"], + "prim_key" : ["NO_D_ID", "NO_W_ID", "NO_O_ID"], + "distr_key" : "NO_W_ID", + }, + "ORDER_LINE":{ + "db" : "order_line", + "attrs" : ["OL_O_ID", "OL_D_ID", "OL_W_ID", "OL_NUMBER", "OL_I_ID", "OL_SUPPLY_W_ID", "OL_DELIVERY_D", + "OL_QUANTITY", "OL_AMOUNT", "OL_DIST_INFO"], + "prim_key": ["OL_W_ID", "OL_D_ID", "OL_O_ID", "OL_NUMBER"], + "distr_key" : "OL_W_ID", + "indexes" : + { + "o_d_w" : { + "map" : """ + function(doc) { + emit([doc.OL_O_ID, doc.OL_D_ID, doc.OL_W_ID], doc.OL_AMOUNT); + } + """, + "reduce": """ + function(keys, values, rereduce) { + return sum(values); + } + """, + }, + "o_d_w_i" : { + "map" : """ + function(doc) { + emit([doc.OL_O_ID, doc.OL_D_ID, doc.OL_W_ID], doc.OL_I_ID); + } + """, + }, + } + }, +} + +def db_from_table(table_name): + """ + Converts the name of the table to the corresponding CouchDB database name. + Note, that CouchDB doesn't like CAPITAL database names. + """ + return TPCC_SCM[table_name]['db'] + +def gen_pk_doc(table_name, doc): + """ + Generate primary key for the row-doc from the table_name. + It is done by just concatenating all 'prim_key' attributes of the table + + If we don't have a key in the primary table, then we just generate it via uuid4. + It is usually recommended to generate an id on the client side. + """ + table_schema = TPCC_SCM[table_name] + if len(table_schema['prim_key']): + pk = '_'.join([str(doc[attr]) for attr in table_schema['prim_key']]) + else: + pk = uuid4().hex + + return pk + +def touch_view(db, view_name): + """ + Touches the 'view_name' view from the given db object. + + The main point here is to make CouchDB actually create the view. Otherwise it would only + create it on the first query. We don't want that, since that would make things very slow during + the actual transaction processing! + """ + logging.debug("HACK: Fetching view '%s' from '%s' with 'limit = 1'" % (view_name, str(db))) + # the result is unimportant here, just use limit=1 + db.view('tpcc/%s' % view_name, limit = 1).rows + logging.debug("HACK: Fetched view '%s' from '%s' with 'limit = 1'" % (view_name, str(db))) + +class TouchThread(threading.Thread): + """ + This is a class to handle "touch-view" threads, which + are used to initialize views in the loadFinish function + + The main scheme here is that in case of several shards, we want to fetch the view from all + the shards simultaneously. 'n' shards equals 'n' threads. + + So, the thread just executes 'touch_view' function and then quits. + """ + def __init__(self, *args): + self._target = touch_view + self._args = args + threading.Thread.__init__(self) + + def run(self): + self._target(*self._args) + +## ============================================== +## CouchdbDriver +## ============================================== +class CouchdbDriver(AbstractDriver): + DEFAULT_CONFIG = { + "node_urls": ("CouchDB URL:", '["http://localhost:5984"]'), # usual "out-of-the-box" value + } + + def __init__(self, ddl): + super(CouchdbDriver, self).__init__("couchdb", ddl) + self.servers = [] # list of shards (couchdb server objects) + self.dbs = None # dict: 'db_name' -> (list of db_obj (shards)) + + ## ---------------------------------------------- + ## makeDefaultConfig + ## ---------------------------------------------- + def makeDefaultConfig(self): + return CouchdbDriver.DEFAULT_CONFIG + + ## ---------------------------------------------- + ## loadConfig + ## ---------------------------------------------- + def loadConfig(self, config): + for key in CouchdbDriver.DEFAULT_CONFIG.keys(): + assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) + + # open servers + for srv_name in eval(config["node_urls"]): + logging.debug("Got a CouchDB node from config: '%s'" % srv_name) + # we use delayed commits here since we don't care much about durability + # note, that couchdb would commit the data once per several seconds anyway + self.servers.append(couchdb.Server(url = srv_name, full_commit = False)) + + db_names = [db_from_table(table) for table in TPCC_SCM.keys()] + + # delete the dbs if we're resetting + if config["reset"]: + for db in db_names: + for srv in self.servers: + if db in srv: + logging.debug("Deleting database '%s' on server '%s'" % (db, str(srv))) + srv.delete(db) + + # creating databases + self.dbs = dict() + for db in db_names: + sdb = [] # list of shards for the db + for srv in self.servers: + if not db in srv: + logging.debug("Creating database '%s' on server '%s'" % (db, str(srv))) + sdb.append(srv.create(db)) + else: + logging.debug("Database exists: '%s', server: '%s'" % (db, str(srv))) + sdb.append(srv[db]) + + self.dbs[db] = sdb + + ## ---------------------------------------------- + ## tuples_to_docs + ## ---------------------------------------------- + def shard_from_id(self, key): + """ + Get the shard number from the key. Key is assumed to be integer. + + Just a dumb round-robin. + """ + return key % len(self.servers) + + ## ---------------------------------------------- + ## tuples_to_docs + ## ---------------------------------------------- + def tuples_to_docs(self, table_name, tuples): + """ + This function converts tuples belonging to the table_name to a list + of documents suitable for loading into CouchDB database with the name table_name + + This is actually not very well written and takes the most CPU time from the loader. + However, do we actually care? It's just loading. Fetching the views will probably kill us anyway... + """ + table_schema = TPCC_SCM[table_name] + + # create list of lists for documents (one list of docs per shard) + docs = [list() for s in self.servers] + tuple_len = len(tuples[0]) + + assert(tuple_len == len(table_schema['attrs'])), "Number of attributes and the tuple length differ: %s" % table_name + + for tup in tuples: + doc = dict() + + # generate the doc as a simple dict + for i, attr in enumerate(table_schema['attrs']): + doc[attr] = tup[i] + + # determine the shard number we want to put the doc into. + # + # we use distr_key for that. + # + # if the table doesn't have a distr key, we assume it's + # replicated over all shard nodes + # + # it is assumed that the 'distr_key' is integer + if TPCC_SCM[table_name].has_key("distr_key"): + distr_key = int(doc[TPCC_SCM[table_name]["distr_key"]]) + shard = self.shard_from_id(distr_key) + else: + shard = -1 + + # emulate primary key with "id" or generate a random one + doc['_id'] = gen_pk_doc(table_name, doc) + + # put the doc to the proper list. + # '-1' means 'replicate to all' + if shard != -1: + docs[shard].append(doc) + else: + for l in docs: + l.append(doc) + + return docs + + ## ---------------------------------------------- + ## loadTuples + ## ---------------------------------------------- + def loadTuples(self, tableName, tuples): + if len(tuples) == 0: return + + # create docs for tuples + docs = self.tuples_to_docs(tableName, tuples) + db_name = db_from_table(tableName) + + # load all documents in bulk on every node + for srv_num, srv in enumerate(self.servers): + if len(docs[srv_num]): + logging.debug("Loading tuples from the table '%s' into database '%s' on server '%s'" % (tableName, db_name, str(srv))) + # should we check the result here? we're assuming a fresh load. + self.dbs[db_name][srv_num].update(docs[srv_num]) + + ## ---------------------------------------------- + ## loadFinish + ## ---------------------------------------------- + def loadFinish(self): + """ + Creates some additional views to speed-up the execution and commits + + This is the tricky part. We want not only to create indexes (views), but also fetch them. Otherwise, + CouchDB would do it in a lazy way, during a first query. We don't want that at all! + """ + view_touch_jobs = [] + for table in TPCC_SCM.keys(): + if 'indexes' in TPCC_SCM[table]: + for srv_num, srv in enumerate(self.servers): + # load the design doc: _design/tpcc + try: + logging.debug("Creating indexes for '%s' on server '%s'" % (table, str(srv))) + cdb = self.dbs[db_from_table(table)][srv_num] + design_doc = {'views' : TPCC_SCM[table]['indexes']} + cdb['_design/tpcc'] = design_doc + except couchdb.http.ResourceConflict: + # happens if we have multiple loaders. This is okay. The design doc is still the same. + pass + finally: + for view_name in TPCC_SCM[table]['indexes'].keys(): + view_touch_jobs.append((cdb, view_name)) + + # we want actually to initialize views in parallel on all shard nodes + # to speed-up loading times + touch_thread_pool = [] + logging.debug("We have %d views to touch" % len(view_touch_jobs)) + for job in view_touch_jobs: + t = TouchThread(job[0], job[1]) + t.start() + touch_thread_pool.append(t) + + logging.debug("Waiting for %d view touchers to finish" % len(touch_thread_pool)) + for t in touch_thread_pool: + t.join() + + ## ---------------------------------------------- + ## doDelivery + ## ---------------------------------------------- + def doDelivery(self, params): + w_id = params["w_id"] + o_carrier_id = params["o_carrier_id"] + ol_delivery_d = str(params["ol_delivery_d"]) + + # Note, we want to do this cycle ASAP, since we're deleting the 'NEW_ORDER' docs and + # are very vulnerable to conflicts + no_o_ids = [] + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1): + while True: + # fetch any 'NEW_ORDER' doc ('0' as the 'NO_O_ID') + newOrder = self.dbs[db_from_table('NEW_ORDER')][self.shard_from_id(w_id)].view('_all_docs', limit = 1, + include_docs = 'true', startkey = gen_pk_doc('NEW_ORDER', {'NO_D_ID': d_id, 'NO_W_ID': w_id, 'NO_O_ID' : 0})).rows + + # it seems that we might fetch a deleted doc in case there are no more. Nice... + if newOrder[0]['value'].has_key('deleted') and newOrder[0]['value']['deleted'] == True: + logging.debug("No documents: _all_docs returned a deleted one. Skipping...") + newOrder = [] + + if len(newOrder) == 0: + ## No orders for this district: skip it. Note: This must be reported if > 1% + break + + newOrder = newOrder[0].doc + + try: + self.dbs[db_from_table('NEW_ORDER')][self.shard_from_id(w_id)].delete(newOrder) + no_o_ids.append((d_id, newOrder['NO_O_ID'])) + break + except couchdb.http.ResourceNotFound: + # in case somebody got this order first, try to fetch another one + logging.debug('Pessimistic concurrency control: Delete failed: Restarting...') + pass + except couchdb.http.ResourceConflict: + # in case somebody got this order first, try to fetch another one + logging.debug('Pessimistic concurrency control: Delete failed: Restarting...') + pass + + if len(newOrder) == 0: + ## No orders for this district: skip it. Note: This must be reported if > 1% + continue + ## FOR + + # Now we're "isolated" from concurrent transactions... + # We're trying to fetch all info using as least requests as possible + order_keys = [gen_pk_doc('ORDERS', {'O_ID': no_o_id, 'O_W_ID': w_id, 'O_D_ID': d_id}) for d_id, no_o_id in no_o_ids] + order_docs = self.dbs[db_from_table('ORDERS')][self.shard_from_id(w_id)].view('_all_docs', + include_docs = 'true', + keys = order_keys).rows + order_docs = [od.doc for od in order_docs] + + # use the view for the sum aggregate + ol_totals = self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].view('tpcc/o_d_w', group = 'true', + keys = [[no_o_id, d_id, w_id] for d_id, no_o_id in no_o_ids]).rows + + # put the fetched information together for every client + c_ids = [] + for i in range(len(no_o_ids)): + # find the total for the current (order, district, warehouse) + # is there some way to find stuff in a list fast? + ol_total = filter(lambda x: x.key == [no_o_ids[i][1], no_o_ids[i][0], w_id], ol_totals)[0].value + # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) + # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure + # them out + # If there are no order lines, SUM returns null. There should always be order lines. + assert ol_total != None, "ol_total is NULL: there are no order lines. This should not happen" + assert ol_total > 0.0 + c_ids.append((order_docs[i]['O_C_ID'], no_o_ids[i][0], ol_total)) + + # this should be safe. no conflicts... + for order_doc in order_docs: + order_doc['O_CARRIER_ID'] = o_carrier_id + self.dbs[db_from_table('ORDERS')][self.shard_from_id(w_id)].update(order_docs) + + # ditto... + # we must do the second retrieval from ORDER_LINES, since now we need docs, not aggregates + order_lines = self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].view('tpcc/o_d_w', + keys = [[no_o_id, d_id, w_id] for d_id, no_o_id in no_o_ids], + reduce = 'false', + include_docs = 'true').rows + order_lines = [r.doc for r in order_lines] + + for ol in order_lines: + ol['OL_DELIVERY_D'] = ol_delivery_d + + self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].update(order_lines) + + # again, updating clients may introduce conflicts. another bottleneck.... + for c_id, d_id, ol_total in c_ids: + while True: + customer_info = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].get(gen_pk_doc('CUSTOMER', + {'C_W_ID': w_id, 'C_D_ID': d_id, 'C_ID': c_id})) + customer_info['C_BALANCE'] += ol_total + + try: + self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].save(customer_info) + break + except couchdb.http.ResourceConflict: + # in case somebody updated the customer first, try again with the new revision + logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + pass + + result = no_o_ids + + return result + + ## ---------------------------------------------- + ## doNewOrder + ## ---------------------------------------------- + def doNewOrder(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + o_entry_d = str(params["o_entry_d"]) + i_ids = params["i_ids"] + i_w_ids = params["i_w_ids"] + i_qtys = params["i_qtys"] + + assert len(i_ids) > 0 + assert len(i_ids) == len(i_w_ids) + assert len(i_ids) == len(i_qtys) + + all_local = True + items = [] + + # retrieve and store info about all the items + item_data = self.dbs[db_from_table('ITEM')][self.shard_from_id(w_id)].view('_all_docs', + include_docs = 'true', + keys = [str(i) for i in i_ids]).rows + + for i in range(len(i_ids)): + ## Determine if this is an all local order or not + all_local = all_local and i_w_ids[i] == w_id + + # get info about the item from the just retrieved bundle + # filter is just for finding an item in a list + doc = filter(lambda it: it.id == str(i_ids[i]), item_data)[0].doc + + ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. + ## Note that this will happen with 1% of transactions on purpose. + if doc is None: + ## TODO Abort here! + return + + items.append((doc['I_PRICE'], doc['I_NAME'], doc['I_DATA'])) + assert len(items) == len(i_ids) + + ## ---------------- + ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER + ## ---------------- + doc = self.dbs[db_from_table('WAREHOUSE')][self.shard_from_id(w_id)].get(str(w_id)) + w_tax = doc['W_TAX'] + + # conflict is possible. this is a bottleneck... + while True: + district_info = self.dbs[db_from_table('DISTRICT')][self.shard_from_id(w_id)].get(gen_pk_doc('DISTRICT', + {'D_ID': d_id, 'D_W_ID': w_id})) + d_tax = district_info['D_TAX'] + d_next_o_id = district_info['D_NEXT_O_ID'] + + district_info['D_NEXT_O_ID'] += 1 + try: + self.dbs[db_from_table('DISTRICT')][self.shard_from_id(w_id)].save(district_info) + break + except couchdb.http.ResourceConflict: + # want to get a unique order id! + logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + pass + + customer_info = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].get(gen_pk_doc('CUSTOMER', + {'C_W_ID': w_id, 'C_D_ID': d_id, 'C_ID': c_id})) + c_discount = customer_info['C_DISCOUNT'] + + ol_cnt = len(i_ids) + o_carrier_id = constants.NULL_CARRIER_ID + order_line_docs = [] + + ## ---------------- + ## Insert Order Item Information + ## ---------------- + item_data = [] + total = 0 + for i in range(len(i_ids)): + ol_number = i + 1 + ol_supply_w_id = i_w_ids[i] + ol_i_id = i_ids[i] + ol_quantity = i_qtys[i] + + itemInfo = items[i] + i_name = itemInfo[1] + i_data = itemInfo[2] + i_price = itemInfo[0] + + # we have potential conflict for every stock + while True: + stockInfo = self.dbs[db_from_table('STOCK')][self.shard_from_id(ol_supply_w_id)].get(gen_pk_doc('STOCK', + {'S_I_ID': ol_i_id, 'S_W_ID': ol_supply_w_id})) + s_quantity = stockInfo['S_QUANTITY'] + s_ytd = stockInfo['S_YTD'] + s_order_cnt = stockInfo['S_ORDER_CNT'] + s_remote_cnt = stockInfo['S_REMOTE_CNT'] + s_data = stockInfo['S_DATA'] + s_dist_xx = stockInfo['S_DIST_%02d' % d_id] # Fetches data from the s_dist_[d_id] column + + ## Update stock + s_ytd += ol_quantity + if s_quantity >= ol_quantity + 10: + s_quantity = s_quantity - ol_quantity + else: + s_quantity = s_quantity + 91 - ol_quantity + s_order_cnt += 1 + + if ol_supply_w_id != w_id: s_remote_cnt += 1 + + # update stock + stockInfo['S_QUANTITY'] = s_quantity + stockInfo['S_YTD'] = s_ytd + stockInfo['S_ORDER_CNT'] = s_order_cnt + stockInfo['S_REMOTE_CNT'] = s_remote_cnt + + try: + self.dbs[db_from_table('STOCK')][self.shard_from_id(ol_supply_w_id)].save(stockInfo) + break + except couchdb.http.ResourceConflict: + # if somebody had reserved the stock before us, repeat. + logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + pass + + + if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: + brand_generic = 'B' + else: + brand_generic = 'G' + + ## Transaction profile states to use "ol_quantity * i_price" + ol_amount = ol_quantity * i_price + total += ol_amount + + # don't insert the order line right now + # we'll do it in bulk later + order_line_row = dict(zip(TPCC_SCM['ORDER_LINE']['attrs'], [d_next_o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, + o_entry_d, ol_quantity, ol_amount, s_dist_xx])) + order_line_row['_id'] = gen_pk_doc('ORDER_LINE', order_line_row) + order_line_docs.append(order_line_row) + + ## Add the info to be returned + item_data.append((i_name, s_quantity, brand_generic, i_price, ol_amount)) + ## FOR + + ## ---------------- + ## Insert Order Information + ## ---------------- + self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].update(order_line_docs) + + orders_row = dict(zip(TPCC_SCM['ORDERS']['attrs'], [d_next_o_id, c_id, d_id, w_id, o_entry_d, o_carrier_id, ol_cnt, all_local])) + orders_row['_id'] = gen_pk_doc('ORDERS', orders_row) + self.dbs[db_from_table('ORDERS')][self.shard_from_id(w_id)].save(orders_row) + + new_order_row = dict(zip(TPCC_SCM['NEW_ORDER']['attrs'], [d_next_o_id, d_id, w_id])) + new_order_row['_id'] = gen_pk_doc('NEW_ORDER', new_order_row) + self.dbs[db_from_table('NEW_ORDER')][self.shard_from_id(w_id)].save(new_order_row) + + ## Adjust the total for the discount + total *= (1 - c_discount) * (1 + w_tax + d_tax) + + ## Pack up values the client is missing (see TPC-C 2.4.3.5) + misc = [ (w_tax, d_tax, d_next_o_id, total) ] + customer_info = [ (customer_info['C_DISCOUNT'], customer_info['C_LAST'], customer_info['C_CREDIT']) ] + return [ customer_info, misc, item_data ] + + ## ---------------------------------------------- + ## doOrderStatus + ## ---------------------------------------------- + def doOrderStatus(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + + assert w_id, pformat(params) + assert d_id, pformat(params) + + if c_id != None: + customer = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].get(gen_pk_doc('CUSTOMER', + {'C_W_ID': w_id, 'C_D_ID': d_id, 'C_ID': c_id})) + else: + # Get the midpoint customer's id + all_customers = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].view('tpcc/w_d_last', + key = [w_id, d_id, c_last], + reduce = 'false').rows + all_customers.sort(lambda x, y: cmp(x['value'], y['value'])) + + assert len(all_customers) > 0 + namecnt = len(all_customers) + index = (namecnt - 1) / 2 + customer = all_customers[index] + customer = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].get(customer['id']) + c_id = customer['C_ID'] + assert len(customer) > 0 + assert c_id != None + + # get the last order from the customer + order = self.dbs[db_from_table('ORDERS')][self.shard_from_id(w_id)].view('tpcc/w_d_c_o', + limit = 1, + include_docs = 'true', + startkey = [w_id, d_id, c_id, 'a'], # 'a' is just to give all numbers + endkey = [w_id, d_id, c_id, -1], + descending = 'true', + reduce = 'false').rows + + if len(order) > 0: + order = order[0].doc + orderLines = self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].view('tpcc/o_d_w', + key = [order['O_ID'], d_id, w_id], + reduce = 'false', + include_docs = 'true').rows + + orderLines = [(o.doc['OL_SUPPLY_W_ID'], o.doc['OL_I_ID'], + o.doc['OL_QUANTITY'], o.doc['OL_AMOUNT'], + o.doc['OL_DELIVERY_D']) for o in orderLines] + else: + orderLines = [] + + customer = (customer['C_ID'], customer['C_FIRST'], customer['C_MIDDLE'], customer['C_LAST'], customer['C_BALANCE']) + order = (order['O_ID'], order['O_CARRIER_ID'], order['O_ENTRY_D']) + return [ customer, order, orderLines ] + + ## ---------------------------------------------- + ## doPayment + ## ---------------------------------------------- + def doPayment(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + h_amount = params["h_amount"] + c_w_id = params["c_w_id"] + c_d_id = params["c_d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + h_date = str(params["h_date"]) + + if c_id != None: + cus_doc_id = gen_pk_doc('CUSTOMER', {'C_W_ID': w_id, 'C_D_ID': d_id, 'C_ID': c_id}) + else: + # Get the midpoint customer's id + all_customers = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].view('tpcc/w_d_last', + key = [w_id, d_id, c_last], + reduce = 'false').rows + all_customers.sort(lambda x, y: cmp(x['value'], y['value'])) + + assert len(all_customers) > 0 + namecnt = len(all_customers) + index = (namecnt - 1) / 2 + customer = all_customers[index] + cus_doc_id = customer['id'] + + # try to update the customer record. conflicts expected. + while True: + customer = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].get(cus_doc_id) + assert len(customer) > 0 + c_id = customer['C_ID'] + + c_balance = customer['C_BALANCE'] - h_amount + c_ytd_payment = customer['C_YTD_PAYMENT'] + h_amount + c_payment_cnt = customer['C_PAYMENT_CNT'] + 1 + + # Customer Credit Information + try: + if customer['C_CREDIT'] == constants.BAD_CREDIT: + c_data = customer['C_DATA'] + newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) + c_data = (newData + "|" + c_data) + if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] + customer['C_DATA'] = c_data + + customer['C_BALANCE'] = c_balance + customer['C_YTD_PAYMENT'] = c_ytd_payment + customer['C_PAYMENT_CNT'] = c_payment_cnt + self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].save(customer) + break + except couchdb.http.ResourceConflict: + logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + pass + + # conflicts when updating warehouse record and... + while True: + warehouse = self.dbs[db_from_table('WAREHOUSE')][self.shard_from_id(w_id)].get(str(w_id)) + warehouse['W_YTD'] += h_amount + + try: + self.dbs[db_from_table('WAREHOUSE')][self.shard_from_id(w_id)].save(warehouse) + break + except couchdb.http.ResourceConflict: + # pessimistic concurrency control... + logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + pass + + # the district record + while True: + district = self.dbs[db_from_table('DISTRICT')][self.shard_from_id(w_id)].get(gen_pk_doc('DISTRICT', + {'D_ID': d_id, 'D_W_ID': w_id})) + district['D_YTD'] += h_amount + + try: + self.dbs[db_from_table('DISTRICT')][self.shard_from_id(w_id)].save(district) + break + except couchdb.http.ResourceConflict: + # pessimistic concurrency control... + logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + pass + + # Concatenate w_name, four spaces, d_name + h_data = "%s %s" % (warehouse['W_NAME'], district['D_NAME']) + # Create the history record + hist = dict(zip(TPCC_SCM['HISTORY']['attrs'],[c_id, c_d_id, c_w_id, d_id, w_id, h_date, h_amount, h_data])) + self.dbs[db_from_table('HISTORY')][self.shard_from_id(c_w_id)].save(hist) + + # TPC-C 2.5.3.3: Must display the following fields: + # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, + # D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, + # C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, + # C_DISCOUNT, C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = "BC"), + # H_AMOUNT, and H_DATE. + + # Hand back all the warehouse, district, and customer data + warehouse = (warehouse['W_NAME'], warehouse['W_STREET_1'], warehouse['W_STREET_2'], + warehouse['W_CITY'], warehouse['W_STATE'], warehouse['W_ZIP']) + district = (district['D_NAME'], district['D_STREET_1'], district['D_STREET_2'], + district['D_CITY'], district['D_STATE'], district['D_ZIP']) + customer = (customer['C_ID'], customer['C_FIRST'], customer['C_MIDDLE'], customer['C_LAST'], customer['C_STREET_1'], + customer['C_STREET_2'], customer['C_CITY'], customer['C_STATE'], customer['C_ZIP'], customer['C_PHONE'], + customer['C_SINCE'], customer['C_CREDIT'], customer['C_CREDIT_LIM'], customer['C_DISCOUNT'], + customer['C_BALANCE'], customer['C_YTD_PAYMENT'], customer['C_PAYMENT_CNT'], customer['C_DATA']) + + # Hand back all the warehouse, district, and customer data + return [ warehouse, district, customer ] + + ## ---------------------------------------------- + ## doStockLevel + ## ---------------------------------------------- + def doStockLevel(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + threshold = params["threshold"] + + result = self.dbs[db_from_table('DISTRICT')][self.shard_from_id(w_id)].get(gen_pk_doc('DISTRICT', + {'D_ID': d_id, 'D_W_ID': w_id})) + assert result + o_id = result['D_NEXT_O_ID'] + + # note, that we might get only parts of some orders because of isolation issues with NewOrder on 'D_NEXT_O_ID' + # I really doubt anything can be done about it + orderLines = self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].view('tpcc/o_d_w_i', + startkey = [o_id - 20, d_id, w_id], + endkey = [o_id - 1, d_id, w_id], + reduce = 'false').rows + + # 'set' operation in the next line just filters out duplicates + stock_keys = [[w_id, i_id] for i_id in set([r['value'] for r in orderLines])] + # do an index scan join! + stock_items = self.dbs[db_from_table('STOCK')][self.shard_from_id(w_id)].view('tpcc/w_i', + keys = stock_keys).rows + + count = 0 + for item in stock_items: + if item.value < threshold: + count += 1 + + return count + +## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/csvdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/csvdriver.py new file mode 100644 index 00000000..e5253f56 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/csvdriver.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import os +import csv +from datetime import datetime +from pprint import pprint,pformat + +from abstractdriver import * + +## ============================================== +## CSVDriver +## ============================================== +class CsvDriver(AbstractDriver): + DEFAULT_CONFIG = { + "table_directory": ("The path to the directory to store the table CSV files", "/tmp/tpcc-tables" ), + "txn_directory": ("The path to the directory to store the txn CSV files", "/tmp/tpcc-txns" ), + } + + def __init__(self, ddl): + super(CsvDriver, self).__init__("csv", ddl) + self.table_directory = None + self.table_outputs = { } + self.txn_directory = None + self.txn_outputs = { } + self.txn_params = { } + ## DEF + + def makeDefaultConfig(self): + return CsvDriver.DEFAULT_CONFIG + ## DEF + + def loadConfig(self, config): + for key in CsvDriver.DEFAULT_CONFIG.keys(): + assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) + + self.table_directory = config["table_directory"] + assert self.table_directory + if not os.path.exists(self.table_directory): os.makedirs(self.table_directory) + + self.txn_directory = config["txn_directory"] + assert self.txn_directory + if not os.path.exists(self.txn_directory): os.makedirs(self.txn_directory) + ## DEF + + def loadTuples(self, tableName, tuples): + if not tableName in self.table_outputs: + path = os.path.join(self.table_directory, "%s.csv" % tableName) + self.table_outputs[tableName] = csv.writer(open(path, 'wb'), quoting=csv.QUOTE_ALL) + ## IF + self.table_outputs[tableName].writerows(tuples) + ## DEF + + def executeTransaction(self, txn, params): + if not txn in self.txn_outputs: + path = os.path.join(self.txn_directory, "%s.csv" % txn) + self.txn_outputs[txn] = csv.writer(open(path, 'wb'), quoting=csv.QUOTE_ALL) + self.txn_params[txn] = params.keys()[:] + self.txn_outputs[txn].writerow(["Timestamp"] + self.txn_params[txn]) + ## IF + row = [datetime.now()] + [params[k] for k in self.txn_params[txn]] + self.txn_outputs[txn].writerow(row) + ## DEF +## CLASS + + diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/hbasedriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/hbasedriver.py new file mode 100644 index 00000000..3eb5bf9e --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/hbasedriver.py @@ -0,0 +1,1408 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# HBase version: +# Copyright (C) 2011 +# Zikai Wang +# galaxyngc1309@gmail.com +# +# Original SQLite version: +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Original Java Version: +# Copyright (C) 2008 +# Evan Jones +# Massachusetts Institute of Technology +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- +import uuid +import constants +from java.lang import Integer, Float, String +from org.apache.hadoop.hbase import HBaseConfiguration, HTableDescriptor, HColumnDescriptor +from org.apache.hadoop.hbase.client import HBaseAdmin, HTable, Put, Get, Scan, Delete, Result, ResultScanner +from org.apache.hadoop.hbase.util import Bytes +from org.apache.hadoop.hbase.filter import PrefixFilter +from abstractdriver import AbstractDriver +from pprint import pprint,pformat + + +## ============================================== +## HBase Tables Layout +## ============================================== +## TABLE WAREHOUSE +## W_ID W_NAME W_STREET_1 W_STREET_2 W_CITY W_STATE W_ZIP W_TAX W_YTD +## Row key: W_ID +## Column Family 1: W_NAME W_STREET_1 W_STREET_2 W_CITY W_STATE W_ZIP +## Column Family 2: W_TAX +## Column Family 3: W_YTD + +## TABLE DISTRICT +## D_ID D_W_ID D_NAME D_STREET_1 D_STREET_2 D_CITY D_STATE_ D_ZIP D_TAX D_YTD D_NEXT_O_ID +## Row key: D_W_ID D_ID +## Column Family 1: D_TAX D_NEXT_O_ID +## Column Family 2: D_NAME D_STREET_1 D_STREET_2 D_CITY D_STATE D_ZIP +## Column Family 3: D_YTD + +## TABLE CUSTOMER +## C_ID C_D_ID C_W_ID C_FIRST C_MIDDLE C_LAST C_STREET_1 C_STREET_2 C_CITY #C_STATE C_ZIP C_PHONE C_SINCE C_CREDIT C_CREDIT_LIM C_DISCOUNT C_BALANCE C_YTD_PAYMENT C_PAYMENT_CNT C_DELIVERY_CNT C_DATA +## Row key: C_W_ID C_D_ID C_ID +## Column Family 1: C_FIRST C_MIDDLE C_LAST C_STREET_1 C_STREET_2 C_CITY #C_STATE C_ZIP C_PHONE C_SINCE C_CREDIT C_CREDIT_LIM C_DISCOUNT C_BALANCE C_YTD_PAYMENT C_PAYMENT_CNT C_DATA +## Colmn Family 2: C_DELIVERY_CNT + +## TABLE HISTORY +## H_C_ID H_C_D_ID H_C_W_ID H_D_ID H_W_ID H_DATE H_AMOUNT H_DATA +## Row key: UUID +## Column Family 1: H_C_ID H_C_D_ID H_C_W_ID H_D_ID H_W_ID H_DATE H_AMOUNT H_DATA + +## TABLE NEW_ORDER +## NO_O_ID NO_D_ID NO_W_ID +## Row key: (NO_W_ID NO_D_ID NO_O_ID) +## Column Family 1: (NO_O_ID NO_D_ID NO_W_ID) + +## TABLE ORDERS +## O_ID O_D_ID O_W_ID O_C_ID O_ENTRY_D O_CARRIER_ID O_OL_CNT O_ALL_LOCAL +## Row key: O_W_ID O_D_ID O_ID +## Column Family 1: (O_C_ID O_ENTRY_D O_CARRIER_ID) +## Column Family 2: (O_OL_CNT O_ALL_LOCAL) + +## TABLE ORDER_LINE +## OL_O_ID OL_D_ID OL_W_ID OL_NUMBER OL_I_ID OL_SUPPLY_W_ID OL_DELIVERY_D OL_QUANTITY OL_AMOUNT OL_DIST_INFO +## Row key: OL_W_ID OL_D_ID OL_O_ID OL_NUMBER +## Column Family 1: OL_I_ID OL_SUPPLY_W_ID OL_DELIVERY_D OL_QUANTITY OL_AMOUNT +## Column Family 2: OL_DIST_INFO + +## TABLE ITEM +## I_ID I_IM_ID I_NAME I_PRICE I_DATA +## Row key: I_ID +## Column Family1: I_NAME I_PRICE I_DATA +## Column Family2: I_IM_ID + +## TABLE STOCK +## S_I_ID S_W_ID S_QUANTITY S_DIST_01 S_DIST_02 S_DIST_03 S_DIST_04 S_DIST_05 S_DIST_06 S_DIST_07 S_DIST_08 S_DIST_09 S_DIST_10 S_YTD S_ORDER_CNT S_REMOTE_CNT S_DATA +## Row key: S_W_ID S_I_ID +## Column Family1: S_QUANTITY S_DIST_01 S_DIST_02 S_DIST_03 S_DIST_04 S_DIST_05 S_DIST_06 S_DIST_07 S_DIST_08 S_DIST_09 S_DIST_10 S_YTD S_ORDER_CNT S_REMOTE_CNT S_DATA + +## Note that HBase cell values are uninterpreted bytes. Therefore, TPC-C values are serialized from their original types with org.apache.hadoop.hbase.util.Bytes package + +## TPC-C tables +TABLES = ["WAREHOUSE", "DISTRICT", "CUSTOMER", "HISTORY", "NEW_ORDER", "ORDERS", "ORDER_LINE", "ITEM", "STOCK"] + +## column qualifiers for all columns in TPC-C tables +COLUMN_NAME = { + "W_ID" : Bytes.toBytes("a"), + "W_NAME" : Bytes.toBytes("b"), + "W_STREET_1" : Bytes.toBytes("c"), + "W_STREET_2" : Bytes.toBytes("d"), + "W_CITY" : Bytes.toBytes("e"), + "W_STATE" : Bytes.toBytes("f"), + "W_ZIP" : Bytes.toBytes("g"), + "W_TAX" : Bytes.toBytes("h"), + "W_YTD" : Bytes.toBytes("i"), + "D_ID" : Bytes.toBytes("a"), + "D_W_ID" : Bytes.toBytes("b"), + "D_NAME" : Bytes.toBytes("c"), + "D_STREET_1" : Bytes.toBytes("d"), + "D_STREET_2" : Bytes.toBytes("e"), + "D_CITY" : Bytes.toBytes("f"), + "D_STATE" : Bytes.toBytes("g"), + "D_ZIP" : Bytes.toBytes("h"), + "D_TAX" : Bytes.toBytes("i"), + "D_YTD" : Bytes.toBytes("j"), + "D_NEXT_O_ID" : Bytes.toBytes("k"), + "C_ID" : Bytes.toBytes("a"), + "C_D_ID" : Bytes.toBytes("b"), + "C_W_ID" : Bytes.toBytes("c"), + "C_FIRST" : Bytes.toBytes("d"), + "C_MIDDLE" : Bytes.toBytes("e"), + "C_LAST" : Bytes.toBytes("f"), + "C_STREET_1" : Bytes.toBytes("g"), + "C_STREET_2" : Bytes.toBytes("h"), + "C_CITY" : Bytes.toBytes("i"), + "C_STATE" : Bytes.toBytes("j"), + "C_ZIP" : Bytes.toBytes("k"), + "C_PHONE" : Bytes.toBytes("l"), + "C_SINCE" : Bytes.toBytes("m"), + "C_CREDIT" : Bytes.toBytes("n"), + "C_CREDIT_LIM" : Bytes.toBytes("o"), + "C_DISCOUNT" : Bytes.toBytes("p"), + "C_BALANCE" : Bytes.toBytes("q"), + "C_YTD_PAYMENT" : Bytes.toBytes("r"), + "C_PAYMENT_CNT" : Bytes.toBytes("s"), + "C_DELIVERY_CNT" : Bytes.toBytes("t"), + "C_DATA" : Bytes.toBytes("u"), + "H_C_ID" : Bytes.toBytes("a"), + "H_C_D_ID" : Bytes.toBytes("b"), + "H_C_W_ID" : Bytes.toBytes("c"), + "H_D_ID" : Bytes.toBytes("d"), + "H_W_ID" : Bytes.toBytes("e"), + "H_DATE" : Bytes.toBytes("f"), + "H_AMOUNT" : Bytes.toBytes("g"), + "H_DATA" : Bytes.toBytes("h"), + "NO_O_ID" : Bytes.toBytes("a"), + "NO_D_ID" : Bytes.toBytes("b"), + "NO_W_ID" : Bytes.toBytes("c"), + "O_ID" : Bytes.toBytes("a"), + "O_D_ID" : Bytes.toBytes("b"), + "O_W_ID" : Bytes.toBytes("c"), + "O_C_ID" : Bytes.toBytes("d"), + "O_ENTRY_D" : Bytes.toBytes("e"), + "O_CARRIER_ID" : Bytes.toBytes("f"), + "O_OL_CNT" : Bytes.toBytes("g"), + "O_ALL_LOCAL" : Bytes.toBytes("h"), + "OL_O_ID" : Bytes.toBytes("a"), + "OL_D_ID" : Bytes.toBytes("b"), + "OL_W_ID" : Bytes.toBytes("c"), + "OL_NUMBER" : Bytes.toBytes("d"), + "OL_I_ID" : Bytes.toBytes("e"), + "OL_SUPPLY_W_ID" : Bytes.toBytes("f"), + "OL_DELIVERY_D" : Bytes.toBytes("g"), + "OL_QUANTITY" : Bytes.toBytes("h"), + "OL_AMOUNT" : Bytes.toBytes("i"), + "OL_DIST_INFO" : Bytes.toBytes("j"), + "I_ID" : Bytes.toBytes("a"), + "I_IM_ID" : Bytes.toBytes("b"), + "I_NAME" : Bytes.toBytes("c"), + "I_PRICE" : Bytes.toBytes("d"), + "I_DATA" : Bytes.toBytes("e"), + "S_I_ID" : Bytes.toBytes("a"), + "S_W_ID" : Bytes.toBytes("b"), + "S_QUANTITY" : Bytes.toBytes("c"), + "S_DIST_01" : Bytes.toBytes("d"), + "S_DIST_02" : Bytes.toBytes("e"), + "S_DIST_03" : Bytes.toBytes("f"), + "S_DIST_04" : Bytes.toBytes("g"), + "S_DIST_05" : Bytes.toBytes("h"), + "S_DIST_06" : Bytes.toBytes("i"), + "S_DIST_07" : Bytes.toBytes("j"), + "S_DIST_08" : Bytes.toBytes("k"), + "S_DIST_09" : Bytes.toBytes("l"), + "S_DIST_10" : Bytes.toBytes("m"), + "S_YTD" : Bytes.toBytes("n"), + "S_ORDER_CNT" : Bytes.toBytes("o"), + "S_REMOTE_CNT" : Bytes.toBytes("p"), + "S_DATA" : Bytes.toBytes("q"), +} + + +## ============================================== +## Optimization Options +## ============================================== + +## do not write to WAL for PUT +WRITE_TO_WAL = False +## a scanner will pre-fetch 100 tuples at once +SCANNER_CACHING = 100 +## turn off auto flushing for PUT +AUTOFLUSH = False +## in-mem column family +DATA_IN_MEM = True + +## ============================================== +## HBaseDriver +## ============================================== +class HbaseDriver(AbstractDriver): + DEFAULT_CONFIG = {} + + def __init__(self, ddl): + super(HbaseDriver, self).__init__("hbase", ddl) + self.config = None + self.admin = None + + self.warehouse_tbl = None + self.district_tbl = None + self.customer_tbl = None + self.history_tbl = None + self.new_order_tbl = None + self.order_tbl = None + self.order_line_tbl = None + self.item_tbl = None + self.stock_tbl = None + + ## column families for all columns in TPC-C tables + self.col_fam1 = Bytes.toBytes("1") + self.col_fam2 = Bytes.toBytes("2") + self.col_fam3 = Bytes.toBytes("3") + + def makeDefaultConfig(self): + return HbaseDriver.DEFAULT_CONFIG + + def loadConfig(self, config): + pass + + ## ============================================== + ## loadStart + ## ============================================== + def loadStart(self): + ## get HBase client + self.config = HBaseConfiguration.create() + self.admin = HBaseAdmin(self.config) + + ## Drop tables if they already exist + for table in TABLES: + if self.admin.tableExists(table) and self.admin.isTableAvailable(table): + if self.admin.isTableEnabled(table): + self.admin.disableTable(table) + self.admin.deleteTable(table) + ## FOR + + ## CREATE TABLE WAREHOUSE + htd = HTableDescriptor("WAREHOUSE") + hcd = HColumnDescriptor("1") + hcd.setInMemory(DATA_IN_MEM) + htd.addFamily(hcd) + hcd = HColumnDescriptor("2") + hcd.setInMemory(DATA_IN_MEM) + htd.addFamily(hcd) + hcd = HColumnDescriptor("3") + hcd.setInMemory(DATA_IN_MEM) + htd.addFamily(hcd) + self.admin.createTable(htd) + + ## CREATE TABLE DISTRICT + htd = HTableDescriptor("DISTRICT") + hcd = HColumnDescriptor("1") + hcd.setInMemory(DATA_IN_MEM) + htd.addFamily(hcd) + hcd = HColumnDescriptor("2") + hcd.setInMemory(DATA_IN_MEM) + htd.addFamily(hcd) + hcd = HColumnDescriptor("3") + hcd.setInMemory(DATA_IN_MEM) + htd.addFamily(hcd) + self.admin.createTable(htd) + + ## CREATE TABLE CUSTOMER + htd = HTableDescriptor("CUSTOMER") + hcd = HColumnDescriptor("1") + htd.addFamily(hcd) + htd.addFamily(HColumnDescriptor("2")) + self.admin.createTable(htd) + + ## CREATE TABLE HISTORY + htd = HTableDescriptor("HISTORY") + htd.addFamily(HColumnDescriptor("1")) + self.admin.createTable(htd) + + ## CREATE TABLE NEW_ORDER + htd = HTableDescriptor("NEW_ORDER") + hcd = HColumnDescriptor("1") + htd.addFamily(hcd) + self.admin.createTable(htd) + + ## CREATE TABLE ORDERS + htd = HTableDescriptor("ORDERS") + hcd = HColumnDescriptor("1") + htd.addFamily(hcd) + htd.addFamily(HColumnDescriptor("2")) + self.admin.createTable(htd) + + ## CREATE TABLE ORDER_LINE + htd = HTableDescriptor("ORDER_LINE") + hcd = HColumnDescriptor("1") + htd.addFamily(hcd) + htd.addFamily(HColumnDescriptor("2")) + self.admin.createTable(htd) + + ## CREATE TABLE ITEM + htd = HTableDescriptor("ITEM") + htd.addFamily(HColumnDescriptor("1")) + htd.addFamily(HColumnDescriptor("2")) + self.admin.createTable(htd) + + ## CREATE TABLE STOCK + htd = HTableDescriptor("STOCK") + htd.addFamily(HColumnDescriptor("1")) + self.admin.createTable(htd) + + ## get handlers to all tables + self.warehouse_tbl = HTable(self.config, "WAREHOUSE") + self.warehouse_tbl.setAutoFlush(AUTOFLUSH) + self.district_tbl = HTable(self.config, "DISTRICT") + self.district_tbl.setAutoFlush(AUTOFLUSH) + self.customer_tbl = HTable(self.config, "CUSTOMER") + self.customer_tbl.setAutoFlush(AUTOFLUSH) + self.history_tbl = HTable(self.config, "HISTORY") + self.history_tbl.setAutoFlush(AUTOFLUSH) + self.new_order_tbl = HTable(self.config, "NEW_ORDER") + self.new_order_tbl.setAutoFlush(AUTOFLUSH) + self.order_tbl = HTable(self.config, "ORDERS") + self.order_tbl.setAutoFlush(AUTOFLUSH) + self.order_line_tbl = HTable(self.config, "ORDER_LINE") + self.order_line_tbl.setAutoFlush(AUTOFLUSH) + self.item_tbl = HTable(self.config, "ITEM") + self.item_tbl.setAutoFlush(AUTOFLUSH) + self.stock_tbl = HTable(self.config, "STOCK") + self.stock_tbl.setAutoFlush(AUTOFLUSH) + + ## ============================================== + ## loadFinish + ## ============================================== + def loadFinish(self): + ## close handlers to all tables + self.warehouse_tbl.close() + self.district_tbl.close() + self.customer_tbl.close() + self.history_tbl.close() + self.new_order_tbl.close() + self.order_tbl.close() + self.order_line_tbl.close() + self.item_tbl.close() + self.stock_tbl.close() + + ## ============================================== + ## loadTuples + ## ============================================== + def loadTuples(self, tableName, tuples): + ## load TPC-C data + if tableName == "WAREHOUSE": + for tuplei in tuples: + w_id = Bytes.toBytes(Integer(tuplei[0])) + w_name = Bytes.toBytes(String(tuplei[1])) + w_street_1 = Bytes.toBytes(String(tuplei[2])) + w_street_2 = Bytes.toBytes(String(tuplei[3])) + w_city = Bytes.toBytes(String(tuplei[4])) + w_state = Bytes.toBytes(String(tuplei[5])) + w_zip = Bytes.toBytes(String(tuplei[6])) + w_tax = Bytes.toBytes(Float(tuplei[7])) + w_ytd = Bytes.toBytes(Float(tuplei[8])) + + row_key = w_id + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["W_NAME"], w_name) + put.add(self.col_fam1, COLUMN_NAME["W_STREET_1"], w_street_1) + put.add(self.col_fam1, COLUMN_NAME["W_STREET_2"], w_street_2) + put.add(self.col_fam1, COLUMN_NAME["W_CITY"], w_city) + put.add(self.col_fam1, COLUMN_NAME["W_STATE"], w_state) + put.add(self.col_fam1, COLUMN_NAME["W_ZIP"], w_zip) + put.add(self.col_fam2, COLUMN_NAME["W_TAX"], w_tax) + put.add(self.col_fam3, COLUMN_NAME["W_YTD"], w_ytd) + self.warehouse_tbl.put(put) + ## FOR + + elif tableName == "DISTRICT": + for tuplei in tuples: + d_id = Bytes.toBytes(Integer(tuplei[0])) + d_w_id = Bytes.toBytes(Integer(tuplei[1])) + d_name = Bytes.toBytes(String(tuplei[2])) + d_street_1 = Bytes.toBytes(String(tuplei[3])) + d_street_2 = Bytes.toBytes(String(tuplei[4])) + d_city = Bytes.toBytes(String(tuplei[5])) + d_state = Bytes.toBytes(String(tuplei[6])) + d_zip = Bytes.toBytes(String(tuplei[7])) + d_tax = Bytes.toBytes(Float(tuplei[8])) + d_ytd = Bytes.toBytes(Float(tuplei[9])) + d_next_o_id = Bytes.toBytes(Integer(tuplei[10])) + + row_key = d_w_id + row_key.append(0) + row_key.extend(d_id) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["D_TAX"], d_tax) + put.add(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"], d_next_o_id) + put.add(self.col_fam2, COLUMN_NAME["D_NAME"], d_name) + put.add(self.col_fam2, COLUMN_NAME["D_STREET_1"], d_street_1) + put.add(self.col_fam2, COLUMN_NAME["D_STREET_2"], d_street_2) + put.add(self.col_fam2, COLUMN_NAME["D_CITY"], d_city) + put.add(self.col_fam2, COLUMN_NAME["D_STATE"], d_state) + put.add(self.col_fam2, COLUMN_NAME["D_ZIP"], d_zip) + put.add(self.col_fam3, COLUMN_NAME["D_YTD"], d_ytd) + self.district_tbl.put(put) + ## FOR + + + elif tableName == "CUSTOMER": + for tuplei in tuples: + c_id = Bytes.toBytes(Integer(tuplei[0])) + c_d_id = Bytes.toBytes(Integer(tuplei[1])) + c_w_id = Bytes.toBytes(Integer(tuplei[2])) + c_first = Bytes.toBytes(String(tuplei[3])) + c_middle = Bytes.toBytes(String(tuplei[4])) + c_last = Bytes.toBytes(String(tuplei[5])) + c_street_1 = Bytes.toBytes(String(tuplei[6])) + c_street_2 = Bytes.toBytes(String(tuplei[7])) + c_city = Bytes.toBytes(String(tuplei[8])) + c_state = Bytes.toBytes(String(tuplei[9])) + c_zip = Bytes.toBytes(String(tuplei[10])) + c_phone = Bytes.toBytes(String(tuplei[11])) + c_since = Bytes.toBytes(String(str(tuplei[12]))) + c_credit = Bytes.toBytes(String(tuplei[13])) + c_credit_lim = Bytes.toBytes(Float(tuplei[14])) + c_discount = Bytes.toBytes(Float(tuplei[15])) + c_balance = Bytes.toBytes(Float(tuplei[16])) + c_ytd_payment = Bytes.toBytes(Float(tuplei[17])) + c_payment_cnt = Bytes.toBytes(Integer(tuplei[18])) + c_delivery_cnt = Bytes.toBytes(Integer(tuplei[19])) + c_data = Bytes.toBytes(String(tuplei[20])) + + row_key = c_w_id + row_key.append(0) + row_key.extend(c_d_id) + row_key.append(0) + row_key.extend(c_id) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["C_FIRST"], c_first) + put.add(self.col_fam1, COLUMN_NAME["C_MIDDLE"], c_middle) + put.add(self.col_fam1, COLUMN_NAME["C_LAST"], c_last) + put.add(self.col_fam1, COLUMN_NAME["C_STREET_1"], c_street_1) + put.add(self.col_fam1, COLUMN_NAME["C_STREET_2"], c_street_2) + put.add(self.col_fam1, COLUMN_NAME["C_CITY"], c_city) + put.add(self.col_fam1, COLUMN_NAME["C_STATE"], c_state) + put.add(self.col_fam1, COLUMN_NAME["C_ZIP"], c_zip) + put.add(self.col_fam1, COLUMN_NAME["C_PHONE"], c_phone) + put.add(self.col_fam1, COLUMN_NAME["C_SINCE"], c_since) + put.add(self.col_fam1, COLUMN_NAME["C_CREDIT"], c_credit) + put.add(self.col_fam1, COLUMN_NAME["C_CREDIT_LIM"], c_credit_lim) + put.add(self.col_fam1, COLUMN_NAME["C_DISCOUNT"], c_discount) + put.add(self.col_fam1, COLUMN_NAME["C_BALANCE"], c_balance) + put.add(self.col_fam1, COLUMN_NAME["C_YTD_PAYMENT"], c_ytd_payment) + put.add(self.col_fam1, COLUMN_NAME["C_PAYMENT_CNT"], c_payment_cnt) + put.add(self.col_fam1, COLUMN_NAME["C_DATA"], c_data) + put.add(self.col_fam2, COLUMN_NAME["C_DELIVERY_CNT"], c_delivery_cnt) + self.customer_tbl.put(put) + ## FOR + + elif tableName == "HISTORY": + for tuplei in tuples: + h_c_id = Bytes.toBytes(Integer(tuplei[0])) + h_c_d_id = Bytes.toBytes(Integer(tuplei[1])) + h_c_w_id = Bytes.toBytes(Integer(tuplei[2])) + h_d_id = Bytes.toBytes(Integer(tuplei[3])) + h_w_id = Bytes.toBytes(Integer(tuplei[4])) + h_date = Bytes.toBytes(String(str(tuplei[5]))) + h_amount = Bytes.toBytes(Float(tuplei[6])) + h_data = Bytes.toBytes(String(tuplei[7])) + + row_key = str(uuid.uuid1()) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["H_C_ID"], h_c_id) + put.add(self.col_fam1, COLUMN_NAME["H_C_D_ID"], h_c_d_id) + put.add(self.col_fam1, COLUMN_NAME["H_C_W_ID"], h_c_w_id) + put.add(self.col_fam1, COLUMN_NAME["H_D_ID"], h_d_id) + put.add(self.col_fam1, COLUMN_NAME["H_W_ID"], h_w_id) + put.add(self.col_fam1, COLUMN_NAME["H_DATE"], h_date) + put.add(self.col_fam1, COLUMN_NAME["H_AMOUNT"], h_amount) + put.add(self.col_fam1, COLUMN_NAME["H_DATA"], h_data) + self.history_tbl.put(put) + ## FOR + + elif tableName == "NEW_ORDER": + for tuplei in tuples: + no_o_id = Bytes.toBytes(Integer(tuplei[0])) + no_d_id = Bytes.toBytes(Integer(tuplei[1])) + no_w_id = Bytes.toBytes(Integer(tuplei[2])) + + row_key = no_w_id + row_key.append(0) + row_key.extend(no_d_id) + row_key.append(0) + row_key.extend(no_o_id) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["NO_O_ID"], no_o_id) + put.add(self.col_fam1, COLUMN_NAME["NO_D_ID"], no_d_id) + put.add(self.col_fam1, COLUMN_NAME["NO_W_ID"], no_w_id) + self.new_order_tbl.put(put) + ## FOR + + elif tableName == "ORDERS": + for tuplei in tuples: + o_id = Bytes.toBytes(Integer(tuplei[0])) + o_d_id = Bytes.toBytes(Integer(tuplei[2])) + o_w_id = Bytes.toBytes(Integer(tuplei[3])) + o_c_id = Bytes.toBytes(Integer(tuplei[1])) + o_entry_d = Bytes.toBytes(String(str(tuplei[4]))) + o_carrier_id = Bytes.toBytes(String(str(tuplei[5]))) + o_ol_cnt = Bytes.toBytes(Integer(tuplei[6])) + o_all_local = Bytes.toBytes(Integer(tuplei[7])) + + row_key = o_w_id + row_key.append(0) + row_key.extend(o_d_id) + row_key.append(0) + row_key.extend(o_id) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["O_C_ID"], o_c_id) + put.add(self.col_fam1, COLUMN_NAME["O_ENTRY_D"], o_entry_d) + put.add(self.col_fam1, COLUMN_NAME["O_CARRIER_ID"], o_carrier_id) + put.add(self.col_fam2, COLUMN_NAME["O_OL_CNT"], o_ol_cnt) + put.add(self.col_fam2, COLUMN_NAME["O_ALL_LOCAL"], o_all_local) + self.order_tbl.put(put) + ## FOR + + elif tableName == "ORDER_LINE": + for tuplei in tuples: + ol_o_id= Bytes.toBytes(Integer(tuplei[0])) + ol_d_id = Bytes.toBytes(Integer(tuplei[1])) + ol_w_id = Bytes.toBytes(Integer(tuplei[2])) + ol_number = Bytes.toBytes(Integer(tuplei[3])) + ol_i_id = Bytes.toBytes(Integer(tuplei[4])) + ol_supply_w_id = Bytes.toBytes(Integer(tuplei[5])) + ol_delivery_d = Bytes.toBytes(String(str(tuplei[6]))) + ol_quantity = Bytes.toBytes(Integer(tuplei[7])) + ol_amount = Bytes.toBytes(Float(tuplei[8])) + ol_dist_info = Bytes.toBytes(String(tuplei[9])) + + row_key = ol_w_id + row_key.append(0) + row_key.extend(ol_d_id) + row_key.append(0) + row_key.extend(ol_o_id) + row_key.append(0) + row_key.extend(ol_number) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["OL_I_ID"], ol_i_id) + put.add(self.col_fam1, COLUMN_NAME["OL_SUPPLY_W_ID"], ol_supply_w_id) + put.add(self.col_fam1, COLUMN_NAME["OL_DELIVERY_D"], ol_delivery_d) + put.add(self.col_fam1, COLUMN_NAME["OL_QUANTITY"], ol_quantity) + put.add(self.col_fam1, COLUMN_NAME["OL_AMOUNT"], ol_amount) + put.add(self.col_fam2, COLUMN_NAME["OL_DIST_INFO"], ol_dist_info) + self.order_line_tbl.put(put) + ## FOR + + elif tableName == "ITEM": + for tuplei in tuples: + i_id = Bytes.toBytes(Integer(tuplei[0])) + i_im_id = Bytes.toBytes(Integer(tuplei[1])) + i_name = Bytes.toBytes(String(tuplei[2])) + i_price = Bytes.toBytes(Float(tuplei[3])) + i_data = Bytes.toBytes(String(tuplei[4])) + + row_key = i_id + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["I_NAME"], i_name) + put.add(self.col_fam1, COLUMN_NAME["I_PRICE"], i_price) + put.add(self.col_fam1, COLUMN_NAME["I_DATA"], i_data) + put.add(self.col_fam2, COLUMN_NAME["I_IM_ID"], i_im_id) + self.item_tbl.put(put) + ## FOR + + elif tableName == "STOCK": + for tuplei in tuples: + s_i_id = Bytes.toBytes(Integer(tuplei[0])) + s_w_id = Bytes.toBytes(Integer(tuplei[1])) + s_quantity = Bytes.toBytes(Integer(tuplei[2])) + s_dist_01 = Bytes.toBytes(String(tuplei[3])) + s_dist_02 = Bytes.toBytes(String(tuplei[4])) + s_dist_03 = Bytes.toBytes(String(tuplei[5])) + s_dist_04 = Bytes.toBytes(String(tuplei[6])) + s_dist_05 = Bytes.toBytes(String(tuplei[7])) + s_dist_06 = Bytes.toBytes(String(tuplei[8])) + s_dist_07 = Bytes.toBytes(String(tuplei[9])) + s_dist_08 = Bytes.toBytes(String(tuplei[10])) + s_dist_09 = Bytes.toBytes(String(tuplei[11])) + s_dist_10 = Bytes.toBytes(String(tuplei[12])) + s_ytd = Bytes.toBytes(Integer(tuplei[13])) + s_order_cnt = Bytes.toBytes(Integer(tuplei[14])) + s_remote_cnt = Bytes.toBytes(Integer(tuplei[15])) + s_data = Bytes.toBytes(String(tuplei[16])) + + row_key = s_w_id + row_key.append(0) + row_key.extend(s_i_id) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["S_QUANTITY"], s_quantity) + put.add(self.col_fam1, COLUMN_NAME["S_DIST_01"], s_dist_01) + put.add(self.col_fam1, COLUMN_NAME["S_DIST_02"], s_dist_02) + put.add(self.col_fam1, COLUMN_NAME["S_DIST_03"], s_dist_03) + put.add(self.col_fam1, COLUMN_NAME["S_DIST_04"], s_dist_04) + put.add(self.col_fam1, COLUMN_NAME["S_DIST_05"], s_dist_05) + put.add(self.col_fam1, COLUMN_NAME["S_DIST_06"], s_dist_06) + put.add(self.col_fam1, COLUMN_NAME["S_DIST_07"], s_dist_07) + put.add(self.col_fam1, COLUMN_NAME["S_DIST_08"], s_dist_08) + put.add(self.col_fam1, COLUMN_NAME["S_DIST_09"], s_dist_09) + put.add(self.col_fam1, COLUMN_NAME["S_DIST_10"], s_dist_10) + put.add(self.col_fam1, COLUMN_NAME["S_YTD"], s_ytd) + put.add(self.col_fam1, COLUMN_NAME["S_ORDER_CNT"], s_order_cnt) + put.add(self.col_fam1, COLUMN_NAME["S_REMOTE_CNT"], s_remote_cnt) + put.add(self.col_fam1, COLUMN_NAME["S_DATA"], s_data) + self.stock_tbl.put(put) + ## FOR + + ## ============================================== + ## executeStart + ## ============================================== + def executeStart(self): + ## get HBase client + self.config = HBaseConfiguration.create() + self.admin = HBaseAdmin(self.config) + + ## get handlers to all tables + self.warehouse_tbl = HTable(self.config, "WAREHOUSE") + self.warehouse_tbl.setScannerCaching(SCANNER_CACHING) + self.district_tbl = HTable(self.config, "DISTRICT") + self.district_tbl.setScannerCaching(SCANNER_CACHING) + self.customer_tbl = HTable(self.config, "CUSTOMER") + self.customer_tbl.setScannerCaching(SCANNER_CACHING) + self.history_tbl = HTable(self.config, "HISTORY") + self.history_tbl.setScannerCaching(SCANNER_CACHING) + self.new_order_tbl = HTable(self.config, "NEW_ORDER") + self.new_order_tbl.setScannerCaching(SCANNER_CACHING) + self.order_tbl = HTable(self.config, "ORDERS") + self.order_tbl.setScannerCaching(SCANNER_CACHING) + self.order_line_tbl = HTable(self.config, "ORDER_LINE") + self.order_line_tbl.setScannerCaching(SCANNER_CACHING) + self.item_tbl = HTable(self.config, "ITEM") + self.item_tbl.setScannerCaching(SCANNER_CACHING) + self.stock_tbl = HTable(self.config, "STOCK") + self.stock_tbl.setScannerCaching(SCANNER_CACHING) + + ## ============================================== + ## executeFinish + ## ============================================== + def executeFinish(self): + ## close handlers to all tables + self.warehouse_tbl.close() + self.district_tbl.close() + self.customer_tbl.close() + self.history_tbl.close() + self.new_order_tbl.close() + self.order_tbl.close() + self.order_line_tbl.close() + self.item_tbl.close() + self.stock_tbl.close() + + ## ============================================== + ## doDelivery + ## ============================================== + def doDelivery(self, params): + w_id = params["w_id"] + o_carrier_id = params["o_carrier_id"] + ol_delivery_d = params["ol_delivery_d"] + + result = [ ] + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): + ## getNewOrder + start_row = self.getRowKey([w_id, d_id]) + s = Scan(start_row, self.getLexNextRowKey(start_row)) + s.addColumn(self.col_fam1, COLUMN_NAME["NO_W_ID"]) + scanner = self.new_order_tbl.getScanner(s) + newOrderRes = [ ] + for res in scanner: + current_row = res.getRow() + no_o_id = Bytes.toInt(current_row[10:14]) + if no_o_id > -1: + newOrderRes.append(res) + ## FOR + scanner.close() + + if len(newOrderRes) == 0: + ## No orders for this district: skip it. Note: This must be reported if > 1% + continue + assert len(newOrderRes) > 0 + newOrder = newOrderRes[0] + no_o_id = Bytes.toInt(newOrder.getRow()[10:14]) + + ## getCId + row_key = self.getRowKey([w_id, d_id, no_o_id]) + get = Get(row_key) + get.addColumn(self.col_fam1, COLUMN_NAME["O_C_ID"]) + res = self.order_tbl.get(get) + c_id = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["O_C_ID"])) + + ## sumOLAmount + start_row = self.getRowKey([w_id, d_id, no_o_id]) + s = Scan(start_row, self.getLexNextRowKey(start_row)) + s.addColumn(self.col_fam1, COLUMN_NAME["OL_AMOUNT"]) + scanner = self.order_line_tbl.getScanner(s) + ol_total = 0.0 + scanner_count = 0 + rows = [ ] + for res in scanner: + rows.append(res.getRow()) + scanner_count = scanner_count + 1 + ol_total = ol_total + Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["OL_AMOUNT"])) + ## FOR + scanner.close() + + ## deleteNewOrder + row_key = self.getRowKey([w_id, d_id, no_o_id]) + delete = Delete(row_key) + self.new_order_tbl.delete(delete) + + ## updateOrders + row_key = self.getRowKey([w_id, d_id, no_o_id]) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["O_CARRIER_ID"], Bytes.toBytes(String(str(o_carrier_id)))) + self.order_tbl.put(put) + + ## updateOrderLine + ## get the rows to update from results of sumOLAmount + for current_row in rows: + put = Put(current_row) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["OL_DELIVERY_D"], Bytes.toBytes(String(str(ol_delivery_d)))) + self.order_line_tbl.put(put) + ## FOR + + # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) + # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure + # them out + assert scanner_count>0 + assert ol_total > 0.0 + + ## "updateCustomer" + row_key = self.getRowKey([w_id, d_id, c_id]) + get = Get(row_key) + get.addColumn(self.col_fam1, COLUMN_NAME["C_BALANCE"]) + res = self.customer_tbl.get(get) + c_balance = Bytes.toFloat(res.getValue(self.col_fam1, COLUMN_NAME["C_BALANCE"])) + ol_total + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["C_BALANCE"], Bytes.toBytes(Float(c_balance))) + self.customer_tbl.put(put) + + result.append((d_id, no_o_id)) + ## FOR + return result + + ## ============================================== + ## doNewOrder + ## ============================================== + def doNewOrder(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + o_entry_d = params["o_entry_d"] + i_ids = params["i_ids"] + i_w_ids = params["i_w_ids"] + i_qtys = params["i_qtys"] + + assert len(i_ids) > 0 + assert len(i_ids) == len(i_w_ids) + assert len(i_ids) == len(i_qtys) + + all_local = True + items = [ ] + for i in range(len(i_ids)): + ## Determine if this is an all local order or not + all_local = all_local and i_w_ids[i] == w_id + ## getItemInfo + row_key = self.getRowKey([i_ids[i]]) + get = Get(row_key) + get.addColumn(self.col_fam1, COLUMN_NAME["I_PRICE"]) + get.addColumn(self.col_fam1, COLUMN_NAME["I_NAME"]) + get.addColumn(self.col_fam1, COLUMN_NAME["I_DATA"]) + res = self.item_tbl.get(get) + current_item = [ ] + if res.getRow() is not None: + current_price = Bytes.toFloat(res.getValue(self.col_fam1, COLUMN_NAME["I_PRICE"])) + byte_arr = res.getValue(self.col_fam1, COLUMN_NAME["I_NAME"]) + current_name = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1, COLUMN_NAME["I_DATA"]) + current_data = Bytes.toString(byte_arr, 0, len(byte_arr)) + current_item = [current_price, current_name, current_data] + items.append(current_item) + ## FOR + + assert len(items) == len(i_ids) + + ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. + ## Note that this will happen with 1% of transactions on purpose. + for item in items: + if len(item) == 0: + return + ## FOR + + ## ---------------- + ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER + ## ---------------- + ## getWarehouseTaxRate + row_key = self.getRowKey([w_id]) + get = Get(row_key) + get.addColumn(self.col_fam2, COLUMN_NAME["W_TAX"]) + res = self.warehouse_tbl.get(get) + w_tax = Bytes.toFloat(res.getValue(self.col_fam2, COLUMN_NAME["W_TAX"])) + + ## getDistrict + row_key = self.getRowKey([w_id, d_id]) + get = Get(row_key) + get.addColumn(self.col_fam1, COLUMN_NAME["D_TAX"]) + get.addColumn(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"]) + res = self.district_tbl.get(get) + d_tax = Bytes.toFloat(res.getValue(self.col_fam1, COLUMN_NAME["D_TAX"])) + d_next_o_id = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"])) + + ## getCustomer + row_key = self.getRowKey([w_id, d_id, c_id]) + get = Get(row_key) + get.addColumn(self.col_fam1, COLUMN_NAME["C_DISCOUNT"]) + get.addColumn(self.col_fam1, COLUMN_NAME["C_LAST"]) + get.addColumn(self.col_fam1, COLUMN_NAME["C_CREDIT"]) + res = self.customer_tbl.get(get) + c_discount = Bytes.toFloat(res.getValue(self.col_fam1, COLUMN_NAME["C_DISCOUNT"])) + byte_arr_1 = res.getValue(self.col_fam1, COLUMN_NAME["C_LAST"]) + byte_arr_2 = res.getValue(self.col_fam1, COLUMN_NAME["C_CREDIT"]) + customer_info = [c_discount, Bytes.toString(byte_arr_1, 0, len(byte_arr_1)), Bytes.toString(byte_arr_2, 0 ,len(byte_arr_2))] + + ## ---------------- + ## Insert Order Information + ## ---------------- + ol_cnt = len(i_ids) + o_carrier_id = constants.NULL_CARRIER_ID + + ## incrementNextOrderId + row_key = self.getRowKey([w_id, d_id]) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"], Bytes.toBytes(Integer(d_next_o_id+1))) + self.district_tbl.put(put) + + ## createOrder + row_key = self.getRowKey([w_id, d_id, d_next_o_id]) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["O_C_ID"], Bytes.toBytes(Integer(c_id))) + put.add(self.col_fam1, COLUMN_NAME["O_ENTRY_D"], Bytes.toBytes(String(str(o_entry_d)))) + put.add(self.col_fam1, COLUMN_NAME["O_CARRIER_ID"], Bytes.toBytes(String(str(o_carrier_id)))) + put.add(self.col_fam2, COLUMN_NAME["O_OL_CNT"], Bytes.toBytes(Integer(ol_cnt))) + put.add(self.col_fam2, COLUMN_NAME["O_ALL_LOCAL"], Bytes.toBytes(Integer(all_local))) + self.order_tbl.put(put) + + ## createNewOrder + row_key = self.getRowKey([w_id, d_id, d_next_o_id]) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["NO_O_ID"], Bytes.toBytes(Integer(d_next_o_id))) + put.add(self.col_fam1, COLUMN_NAME["NO_D_ID"], Bytes.toBytes(Integer(d_id))) + put.add(self.col_fam1, COLUMN_NAME["NO_W_ID"], Bytes.toBytes(Integer(w_id))) + self.new_order_tbl.put(put) + + ## ---------------- + ## Insert Order Item Information + ## ---------------- + item_data = [ ] + total = 0 + for i in range(len(i_ids)): + ol_number = i + 1 + ol_supply_w_id = i_w_ids[i] + ol_i_id = i_ids[i] + ol_quantity = i_qtys[i] + + itemInfo = items[i] + i_name = itemInfo[1] + i_data = itemInfo[2] + i_price = itemInfo[0] + + ## getStockInfo + row_key = self.getRowKey([ol_supply_w_id, ol_i_id]) + get = Get(row_key) + get.addColumn(self.col_fam1, COLUMN_NAME["S_QUANTITY"]) + get.addColumn(self.col_fam1, COLUMN_NAME["S_DATA"]) + get.addColumn(self.col_fam1, COLUMN_NAME["S_YTD"]) + get.addColumn(self.col_fam1, COLUMN_NAME["S_ORDER_CNT"]) + get.addColumn(self.col_fam1, COLUMN_NAME["S_REMOTE_CNT"]) + get.addColumn(self.col_fam1, COLUMN_NAME["S_DIST_"+("%02d" % d_id)]) + res = self.stock_tbl.get(get) + s_quantity = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["S_QUANTITY"])) + s_ytd = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["S_YTD"])) + s_order_cnt = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["S_ORDER_CNT"])) + s_remote_cnt = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["S_REMOTE_CNT"])) + byte_arr = res.getValue(self.col_fam1, COLUMN_NAME["S_DATA"]) + s_data = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1, COLUMN_NAME["S_DIST_"+("%02d" % d_id)]) + s_dist_xx = Bytes.toString(byte_arr, 0, len(byte_arr)) # Fetches data from the s_dist_[d_id] column + + ## Update stock + s_ytd += ol_quantity + if s_quantity >= ol_quantity + 10: + s_quantity = s_quantity - ol_quantity + else: + s_quantity = s_quantity + 91 - ol_quantity + s_order_cnt += 1 + + if ol_supply_w_id != w_id: s_remote_cnt += 1 + + ## updateStock + row_key = self.getRowKey([ol_supply_w_id, ol_i_id]) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["S_QUANTITY"], Bytes.toBytes(Integer(s_quantity))) + put.add(self.col_fam1, COLUMN_NAME["S_YTD"], Bytes.toBytes(Integer(s_ytd))) + put.add(self.col_fam1, COLUMN_NAME["S_ORDER_CNT"], Bytes.toBytes(Integer(s_order_cnt))) + put.add(self.col_fam1, COLUMN_NAME["S_REMOTE_CNT"], Bytes.toBytes(Integer(s_remote_cnt))) + self.stock_tbl.put(put) + + if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: + brand_generic = 'B' + else: + brand_generic = 'G' + + ## Transaction profile states to use "ol_quantity * i_price" + ol_amount = ol_quantity * i_price + total += ol_amount + + ## createOrderLine + row_key = self.getRowKey([w_id, d_id, d_next_o_id, ol_number]) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["OL_I_ID"], Bytes.toBytes(Integer(ol_i_id))) + put.add(self.col_fam1, COLUMN_NAME["OL_SUPPLY_W_ID"], Bytes.toBytes(Integer(ol_supply_w_id))) + put.add(self.col_fam1, COLUMN_NAME["OL_DELIVERY_D"], Bytes.toBytes(String(str(o_entry_d)))) + put.add(self.col_fam1, COLUMN_NAME["OL_QUANTITY"], Bytes.toBytes(Integer(ol_quantity))) + put.add(self.col_fam1, COLUMN_NAME["OL_AMOUNT"], Bytes.toBytes(Float(ol_amount))) + put.add(self.col_fam1, COLUMN_NAME["OL_DIST_INFO"], Bytes.toBytes(String(s_dist_xx))) + self.order_line_tbl.put(put) + + ## Add the info to be returned + item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) + ## FOR + + ## Adjust the total for the discount + total *= (1 - c_discount) * (1 + w_tax + d_tax) + + ## Pack up values the client is missing (see TPC-C 2.4.3.5) + misc = [ (w_tax, d_tax, d_next_o_id, total) ] + + return [ customer_info, misc, item_data ] + + ## ============================================== + ## doOrderStatus + ## ============================================== + def doOrderStatus(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + + assert w_id, pformat(params) + assert d_id, pformat(params) + + customer = None + if c_id != None: + ## getCustomerByCustomerId + row_key = self.getRowKey([w_id, d_id, c_id]) + get = Get(row_key) + get.addColumn(self.col_fam1, COLUMN_NAME["C_FIRST"]) + get.addColumn(self.col_fam1, COLUMN_NAME["C_MIDDLE"]) + get.addColumn(self.col_fam1, COLUMN_NAME["C_LAST"]) + get.addColumn(self.col_fam1, COLUMN_NAME["C_BALANCE"]) + customer = self.customer_tbl.get(get) + + else: + ## getCustomersByLastName + getCustomerRes = [ ] + start_row = self.getRowKey([w_id, d_id]) + s = Scan(start_row, self.getLexNextRowKey(start_row)) + s.addColumn(self.col_fam1, COLUMN_NAME["C_FIRST"]) + s.addColumn(self.col_fam1, COLUMN_NAME["C_MIDDLE"]) + s.addColumn(self.col_fam1, COLUMN_NAME["C_LAST"]) + s.addColumn(self.col_fam1, COLUMN_NAME["C_BALANCE"]) + scanner = self.customer_tbl.getScanner(s) + for res in scanner: + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_LAST"]) + current_c_last = Bytes.toString(byte_arr, 0, len(byte_arr)) + if current_c_last == c_last: + getCustomerRes.append(res) + ## FOR + scanner.close() + + assert len(getCustomerRes) > 0 + namecnt = len(getCustomerRes) + + getCustomerRes.sort(self.compareCustomerByFirst) + + index = (namecnt-1)/2 + customer = getCustomerRes[index] + c_id = Bytes.toInt(customer.getRow()[10:14]) + + assert customer != None + assert c_id != None + + byte_arr = customer.getValue(self.col_fam1,COLUMN_NAME["C_FIRST"]) + customer_first = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = customer.getValue(self.col_fam1,COLUMN_NAME["C_MIDDLE"]) + customer_middle = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = customer.getValue(self.col_fam1,COLUMN_NAME["C_LAST"]) + customer_last = Bytes.toString(byte_arr, 0, len(byte_arr)) + customer_balance = Bytes.toFloat(customer.getValue(self.col_fam1,COLUMN_NAME["C_BALANCE"])) + customer = [c_id, customer_first, customer_middle, customer_last, customer_balance] + + ## getLastOrder + getOrderRes = [ ] + start_row = self.getRowKey([w_id, d_id]) + s = Scan(start_row, self.getLexNextRowKey(start_row)) + s.addColumn(self.col_fam1, COLUMN_NAME["O_CARRIER_ID"]) + s.addColumn(self.col_fam1, COLUMN_NAME["O_ENTRY_D"]) + s.addColumn(self.col_fam1, COLUMN_NAME["O_C_ID"]) + scanner = self.order_tbl.getScanner(s) + for res in scanner: + current_o_c_id = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["O_C_ID"])) + if current_o_c_id == c_id: + getOrderRes.append(res) + ## FOR + scanner.close() + + getOrderRes.sort(self.compareOrderByOID) + order = None + if len(getOrderRes)>0 : + order = getOrderRes[0] + + orderLines = [ ] + if order != None: + order_o_id = Bytes.toInt(order.getRow()[10:14]) + byte_arr = order.getValue(self.col_fam1,COLUMN_NAME["O_CARRIER_ID"]) + order_o_carrier_id = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = order.getValue(self.col_fam1,COLUMN_NAME["O_ENTRY_D"]) + order_o_entry_d = Bytes.toString(byte_arr, 0, len(byte_arr)) + order = [order_o_id, order_o_carrier_id, order_o_entry_d] + ## getOrderLines + start_row = self.getRowKey([w_id, d_id, order_o_id]) + s = Scan(start_row, self.getLexNextRowKey(start_row)) + s.addColumn(self.col_fam1, COLUMN_NAME["OL_SUPPLY_W_ID"]) + s.addColumn(self.col_fam1, COLUMN_NAME["OL_I_ID"]) + s.addColumn(self.col_fam1, COLUMN_NAME["OL_QUANTITY"]) + s.addColumn(self.col_fam1, COLUMN_NAME["OL_AMOUNT"]) + s.addColumn(self.col_fam1, COLUMN_NAME["OL_DELIVERY_D"]) + scanner = self.order_line_tbl.getScanner(s) + for res in scanner: + current_ol_supply_w_id = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["OL_SUPPLY_W_ID"])) + current_ol_i_id = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["OL_I_ID"])) + current_ol_quantity = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["OL_QUANTITY"])) + current_ol_amount = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["OL_AMOUNT"])) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["OL_DELIVERY_D"]) + current_ol_delivery_d = Bytes.toString(byte_arr, 0, len(byte_arr)) + current_order_line = [current_ol_supply_w_id, current_ol_i_id, current_ol_quantity, current_ol_amount, current_ol_delivery_d] + orderLines.append(current_order_line) + ## FOR + scanner.close() + + else: + orderLines = [ ] + + return [ customer, order, orderLines ] + + ## ============================================== + ## doPayment + ## ============================================== + + def doPayment(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + h_amount = params["h_amount"] + c_w_id = params["c_w_id"] + c_d_id = params["c_d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + h_date = params["h_date"] + + if c_id != None: + ## getCustomerByCustomerId + row_key = self.getRowKey([w_id, d_id, c_id]) + get = Get(row_key) + get.addFamily(self.col_fam1) + res = self.customer_tbl.get(get) + current_c_id = Bytes.toInt(res.getRow()[10:14]) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_FIRST"]) + current_c_first = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_MIDDLE"]) + current_c_middle = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_LAST"]) + current_c_last = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STREET_1"]) + current_c_street_1 = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STREET_2"]) + current_c_street_2 = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_CITY"]) + current_c_city = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STATE"]) + current_c_state = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_ZIP"]) + current_c_zip = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_PHONE"]) + current_c_phone = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_SINCE"]) + current_c_since = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_CREDIT"]) + current_c_credit = Bytes.toString(byte_arr, 0, len(byte_arr)) + current_c_credit_lim = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_CREDIT_LIM"])) + current_c_discount = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_DISCOUNT"])) + current_c_balance = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_BALANCE"])) + current_c_ytd_payment = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_YTD_PAYMENT"])) + current_c_payment_cnt = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["C_PAYMENT_CNT"])) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_DATA"]) + current_c_data = Bytes.toString(byte_arr, 0, len(byte_arr)) + customer = [current_c_id, current_c_first, current_c_middle, current_c_last, current_c_street_1, current_c_street_2, current_c_city, current_c_state, current_c_zip, current_c_phone, current_c_since, current_c_credit, current_c_credit_lim, current_c_discount, current_c_balance, current_c_ytd_payment, current_c_payment_cnt, current_c_data] + else: + ## getCustomersByLastName + start_row = self.getRowKey([w_id, d_id]) + s = Scan(start_row, self.getLexNextRowKey(start_row)) + s.addFamily(self.col_fam1) + scanner = self.customer_tbl.getScanner(s) + all_customers = [ ] + for res in scanner: + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_LAST"]) + current_c_last = Bytes.toString(byte_arr, 0, len(byte_arr)) + if current_c_last == c_last: + current_c_id = Bytes.toInt(res.getRow()[10:14]) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_FIRST"]) + current_c_first = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_MIDDLE"]) + current_c_middle = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_LAST"]) + current_c_last = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STREET_1"]) + current_c_street_1 = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STREET_2"]) + current_c_street_2 = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_CITY"]) + current_c_city = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STATE"]) + current_c_state = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_ZIP"]) + current_c_zip = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_PHONE"]) + current_c_phone = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_SINCE"]) + current_c_since = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_CREDIT"]) + current_c_credit = Bytes.toString(byte_arr, 0, len(byte_arr)) + current_c_credit_lim = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_CREDIT_LIM"])) + current_c_discount = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_DISCOUNT"])) + current_c_balance = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_BALANCE"])) + current_c_ytd_payment = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_YTD_PAYMENT"])) + current_c_payment_cnt = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["C_PAYMENT_CNT"])) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_DATA"]) + current_c_data = Bytes.toString(byte_arr, 0, len(byte_arr)) + + current_customer = [current_c_id, current_c_first, current_c_middle, current_c_last, current_c_street_1, current_c_street_2, current_c_city, current_c_state, current_c_zip, current_c_phone, current_c_since, current_c_credit, current_c_credit_lim, current_c_discount, current_c_balance, current_c_ytd_payment, current_c_payment_cnt, current_c_data] + all_customers.append(current_customer) + ## FOR + scanner.close() + + all_customers.sort(self.compareCustomerArrByFirst) + assert len(all_customers) > 0 + namecnt = len(all_customers) + index = (namecnt-1)/2 + customer = all_customers[index] + c_id = customer[0] + assert len(customer) > 0 + c_balance = customer[14] - h_amount + c_ytd_payment = customer[15] + h_amount + c_payment_cnt = customer[16] + 1 + c_data = customer[17] + + ## getWarehouse + row_key = self.getRowKey([w_id]) + get = Get(row_key) + get.addFamily(self.col_fam1) + get.addFamily(self.col_fam3) + res = self.warehouse_tbl.get(get) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_NAME"]) + current_w_name = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_STREET_1"]) + current_w_street_1 = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_STREET_2"]) + current_w_street_2 = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_CITY"]) + current_w_city = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_STATE"]) + current_w_state = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_ZIP"]) + current_w_zip = Bytes.toString(byte_arr, 0, len(byte_arr)) + ## pre-fetch w_ytd for updateWarehouseBalance + current_w_ytd = Bytes.toFloat(res.getValue(self.col_fam3,COLUMN_NAME["W_YTD"])) + warehouse = [current_w_name, current_w_street_1, current_w_street_2, current_w_city, current_w_state, current_w_zip] + + ## getDistrict + row_key = self.getRowKey([w_id, d_id]) + get = Get(row_key) + get.addFamily(self.col_fam2) + get.addFamily(self.col_fam3) + res = self.district_tbl.get(get) + byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_NAME"]) + current_d_name = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_STREET_1"]) + current_d_street_1 = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_STREET_2"]) + current_d_street_2 = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_CITY"]) + current_d_city = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_STATE"]) + current_d_state = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_ZIP"]) + current_d_zip = Bytes.toString(byte_arr, 0, len(byte_arr)) + ## pre-fetch d_ytd for updateDistrictBalance + current_d_ytd = Bytes.toFloat(res.getValue(self.col_fam3,COLUMN_NAME["D_YTD"])) + district = [current_d_name, current_d_street_1, current_d_street_2, current_d_city, current_d_state, current_d_zip] + + ## updateWarehouseBalance + row_key = self.getRowKey([w_id]) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam3, COLUMN_NAME["W_YTD"], Bytes.toBytes(Float(current_w_ytd + h_amount))) + self.warehouse_tbl.put(put) + + ## updateDistrictBalance + row_key = self.getRowKey([w_id, d_id]) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam3, COLUMN_NAME["D_YTD"], Bytes.toBytes(Float(current_d_ytd + h_amount))) + self.district_tbl.put(put) + + ## Customer Credit Information + if customer[11] == constants.BAD_CREDIT: + newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) + c_data = (newData + "|" + c_data) + if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] + ## updateBCCustomer + row_key = self.getRowKey([c_w_id, c_d_id, c_id]) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["C_BALANCE"], Bytes.toBytes(Float(c_balance))) + put.add(self.col_fam1, COLUMN_NAME["C_YTD_PAYMENT"], Bytes.toBytes(Float(c_ytd_payment))) + put.add(self.col_fam1, COLUMN_NAME["C_PAYMENT_CNT"], Bytes.toBytes(Integer(c_payment_cnt))) + put.add(self.col_fam1, COLUMN_NAME["C_DATA"], Bytes.toBytes(String(c_data))) + self.customer_tbl.put(put) + else: + c_data = "" + ## updateGCCustomer + row_key = self.getRowKey([c_w_id, c_d_id, c_id]) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["C_BALANCE"], Bytes.toBytes(Float(c_balance))) + put.add(self.col_fam1, COLUMN_NAME["C_YTD_PAYMENT"], Bytes.toBytes(Float(c_ytd_payment))) + put.add(self.col_fam1, COLUMN_NAME["C_PAYMENT_CNT"], Bytes.toBytes(Integer(c_payment_cnt))) + self.customer_tbl.put(put) + + # Concatenate w_name, four spaces, d_name + h_data = "%s %s" % (warehouse[0], district[0]) + # Create the history record + ## insertHistory + row_key = Bytes.toBytes(str(uuid.uuid1())) + put = Put(row_key) + put.setWriteToWAL(WRITE_TO_WAL) + put.add(self.col_fam1, COLUMN_NAME["H_C_ID"], Bytes.toBytes(Integer(c_id))) + put.add(self.col_fam1, COLUMN_NAME["H_C_D_ID"], Bytes.toBytes(Integer(c_d_id))) + put.add(self.col_fam1, COLUMN_NAME["H_C_W_ID"], Bytes.toBytes(Integer(c_w_id))) + put.add(self.col_fam1, COLUMN_NAME["H_D_ID"], Bytes.toBytes(Integer(d_id))) + put.add(self.col_fam1, COLUMN_NAME["H_W_ID"], Bytes.toBytes(Integer(w_id))) + put.add(self.col_fam1, COLUMN_NAME["H_DATE"], Bytes.toBytes(String(str(h_date)))) + put.add(self.col_fam1, COLUMN_NAME["H_AMOUNT"], Bytes.toBytes(Float(h_amount))) + put.add(self.col_fam1, COLUMN_NAME["H_DATA"], Bytes.toBytes(String(h_data))) + self.history_tbl.put(put) + + # TPC-C 2.5.3.3: Must display the following fields: + # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, + # D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, + # C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, + # C_DISCOUNT, C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = "BC"), + # H_AMOUNT, and H_DATE. + + # Hand back all the warehouse, district, and customer data + return [ warehouse, district, customer ] + + ## ============================================== + ## doStockLevel + ## ============================================== + def doStockLevel(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + threshold = params["threshold"] + + ## getOId + row_key = self.getRowKey([w_id, d_id]) + get = Get(row_key) + get.addColumn(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"]) + res = self.district_tbl.get(get) + assert res.getRow() is not None + o_id = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"])) + + ## getStockCount + ol_i_id_set = set() + start_row = self.getRowKey([w_id, d_id]) + s = Scan(start_row, self.getLexNextRowKey(start_row)) + s.addColumn(self.col_fam1, COLUMN_NAME["OL_I_ID"]) + scanner = self.order_line_tbl.getScanner(s) + for res in scanner: + current_row = res.getRow() + ol_o_id = Bytes.toInt(current_row[10:14]) + if (ol_o_id < o_id) and (ol_o_id >= o_id - 20): + current_ol_i_id = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["OL_I_ID"])) + row_key = self.getRowKey([w_id, current_ol_i_id]) + get = Get(row_key) + get.addColumn(self.col_fam1, COLUMN_NAME["S_QUANTITY"]) + resx = self.stock_tbl.get(get) + if resx.getRow() is not None: + current_s_quantity = Bytes.toInt(resx.getValue(self.col_fam1, COLUMN_NAME["S_QUANTITY"])) + if current_s_quantity < threshold: + ol_i_id_set.add(current_ol_i_id) + ## FOR + scanner.close() + + return len(ol_i_id_set) + + ## get a row key which is composite of primary keys + def getRowKey(self,keylist): + byte_arr = [ ] + length = len(keylist) + for i in range(length): + byte_arr.extend(Bytes.toBytes(Integer(keylist[i]))) + if i= 0 and next[j] < 127) or (next[j] >= -128 and next[j] < -1): + next[j] = next[j] + 1 + break + elif next[j] == 127: + next[j] = -128 + break + elif next[j] == -1: + next[j] = 0 + ## FOR + return next + + + ## compare function used in OrderStatus transaction: ASCE + def compareCustomerByFirst(self, i, j): + byte_arr = i.getValue(self.col_fam1,COLUMN_NAME["C_FIRST"]) + first_i = Bytes.toString(byte_arr, 0, len(byte_arr)) + byte_arr = j.getValue(self.col_fam1,COLUMN_NAME["C_FIRST"]) + first_j = Bytes.toString(byte_arr, 0, len(byte_arr)) + return cmp(first_i,first_j) + + ## compare function used in OrderStatus transaction: DESC + def compareOrderByOID(self, i, j): + oid_i = Bytes.toInt(i.getRow()[10:14]) + oid_j = Bytes.toInt(j.getRow()[10:14]) + return -cmp(oid_i,oid_j) + + ## compare function used in Payment transaction: ASCE + def compareCustomerArrByFirst(self, i, j): + first_i = i[1] + first_j = i[1] + return cmp(first_i,first_j) diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/membasedriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/membasedriver.py new file mode 100644 index 00000000..11317573 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/membasedriver.py @@ -0,0 +1,932 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Sunil Mallya, Silvia Zuffi +# http://www.cs.brown.edu/~sunilmallya/ +# http://www.cs.brown.edu/~zuffi/ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +from __future__ import with_statement + +import os,time +import logging +import commands +import memcache +from pprint import pprint,pformat + +import constants +from abstractdriver import * +MAX_CUSTOMER_ID = 3000 +MAX_ORDER_ID = 2999 + +# Appended with number of columns that needs to be used as the key +TABLE_COLUMNS = { + constants.TABLENAME_ITEM: [ + "I_ID", # INTEGER + "I_IM_ID", # INTEGER + "I_NAME", # VARCHAR + "I_PRICE", # FLOAT + "I_DATA", # VARCHAR + 1 + ], + constants.TABLENAME_WAREHOUSE: [ + "W_ID", # SMALLINT + "W_NAME", # VARCHAR + "W_STREET_1", # VARCHAR + "W_STREET_2", # VARCHAR + "W_CITY", # VARCHAR + "W_STATE", # VARCHAR + "W_ZIP", # VARCHAR + "W_TAX", # FLOAT + "W_YTD", # FLOAT + 1 + ], + constants.TABLENAME_DISTRICT: [ + "D_ID", # TINYINT + "D_W_ID", # SMALLINT + "D_NAME", # VARCHAR + "D_STREET_1", # VARCHAR + "D_STREET_2", # VARCHAR + "D_CITY", # VARCHAR + "D_STATE", # VARCHAR + "D_ZIP", # VARCHAR + "D_TAX", # FLOAT + "D_YTD", # FLOAT + "D_NEXT_O_ID", # INT + 2 + ], + constants.TABLENAME_CUSTOMER: [ + "C_ID", # INTEGER + "C_D_ID", # TINYINT + "C_W_ID", # SMALLINT + "C_FIRST", # VARCHAR + "C_MIDDLE", # VARCHAR + "C_LAST", # VARCHAR + "C_STREET_1", # VARCHAR + "C_STREET_2", # VARCHAR + "C_CITY", # VARCHAR + "C_STATE", # VARCHAR + "C_ZIP", # VARCHAR + "C_PHONE", # VARCHAR + "C_SINCE", # TIMESTAMP + "C_CREDIT", # VARCHAR + "C_CREDIT_LIM", # FLOAT + "C_DISCOUNT", # FLOAT + "C_BALANCE", # FLOAT + "C_YTD_PAYMENT", # FLOAT + "C_PAYMENT_CNT", # INTEGER + "C_DELIVERY_CNT", # INTEGER + "C_DATA", # VARCHAR + 3 + ], + constants.TABLENAME_STOCK: [ + "S_I_ID", # INTEGER + "S_W_ID", # SMALLINT + "S_QUANTITY", # INTEGER + "S_DIST_01", # VARCHAR + "S_DIST_02", # VARCHAR + "S_DIST_03", # VARCHAR + "S_DIST_04", # VARCHAR + "S_DIST_05", # VARCHAR + "S_DIST_06", # VARCHAR + "S_DIST_07", # VARCHAR + "S_DIST_08", # VARCHAR + "S_DIST_09", # VARCHAR + "S_DIST_10", # VARCHAR + "S_YTD", # INTEGER + "S_ORDER_CNT", # INTEGER + "S_REMOTE_CNT", # INTEGER + "S_DATA", # VARCHAR + 2 + ], + constants.TABLENAME_ORDERS: [ + "O_ID", # INTEGER + "O_C_ID", # INTEGER + "O_D_ID", # TINYINT + "O_W_ID", # SMALLINT + "O_ENTRY_D", # TIMESTAMP + "O_CARRIER_ID", # INTEGER + "O_OL_CNT", # INTEGER + "O_ALL_LOCAL", # INTEGER + 4 + ], + "ORDERS_ID": [ + "O_C_ID", # INTEGER + "O_D_ID", # TINYINT + "O_W_ID", # SMALLINT + "O_ID", # VARCHAR + 3 + ], + constants.TABLENAME_NEW_ORDER: [ + "NO_O_ID", # INTEGER + "NO_D_ID", # TINYINT + "NO_W_ID", # SMALLINT + 3 + ], + constants.TABLENAME_ORDER_LINE: [ + "OL_O_ID", # INTEGER + "OL_D_ID", # TINYINT + "OL_W_ID", # SMALLINT + "OL_NUMBER", # INTEGER + "OL_I_ID", # INTEGER + "OL_SUPPLY_W_ID", # SMALLINT + "OL_DELIVERY_D", # TIMESTAMP + "OL_QUANTITY", # INTEGER + "OL_AMOUNT", # FLOAT + "OL_DIST_INFO", # VARCHAR + 3 + ], + constants.TABLENAME_HISTORY: [ + "H_C_ID", # INTEGER + "H_C_D_ID", # TINYINT + "H_C_W_ID", # SMALLINT + "H_D_ID", # TINYINT + "H_W_ID", # SMALLINT + "H_DATE", # TIMESTAMP + "H_AMOUNT", # FLOAT + "H_DATA", # VARCHAR + 5 + ], +} + +def irange(sequence): + return zip(range(len(sequence)),sequence) + +def filter_table(sec_keys,table_values,table_name): + + columns = TABLE_COLUMNS[table_name] + return_list=[] + sec_key_indices = [] # [index, value] + for sec_key_name in sec_keys: + if sec_key_name[0]!= None: + index = columns.index(sec_key_name[0]) + sec_key_indices.append([index,sec_key_name[1]]) + + for tvalue in table_values: + copy_flag = 1 + for sec_key_tuple in sec_key_indices: + if tvalue[sec_key_tuple[0]] != sec_key_tuple[1]: + copy_flag =0 + + if copy_flag ==1: + return_list.append(tvalue) + + return return_list + + +def return_columns_single_record(extract_columns,table_values,table_name): + columns = TABLE_COLUMNS[table_name] + record =[] + for c in extract_columns: + c_index = columns.index(c) + record.append(table_values[c_index]) + + return record + +def return_columns(extract_columns,table_values,table_name): + #c contains the name of the columns to be extracted + columns = TABLE_COLUMNS[table_name] + + return_table=[] + for tvalue in table_values: + insert_tuple=[] + for c in extract_columns: + c_index = columns.index(c) + insert_tuple.append(tvalue[c_index]) + return_table.append(insert_tuple) + + return return_table + + +## ============================================== +## MembaseDriver +## ============================================== +class MembaseDriver(AbstractDriver): + DEFAULT_CONFIG = { + "host": ("The hostname to membase", "localhost" ), + "port": ("The port number to membase", 11211 ), + "name": ("Collection name", "tpcc"), + } + #This may be similar to the memcache configuration + + def __init__(self, ddl): + super(MembaseDriver, self).__init__("membase", ddl) + self.database = None + self.conn = None + self.load_time = 0 + + ## ---------------------------------------------- + ## makeDefaultConfig + ## ---------------------------------------------- + def makeDefaultConfig(self): + return MembaseDriver.DEFAULT_CONFIG + + ## ---------------------------------------------- + ## loadConfig + ## ---------------------------------------------- + def loadConfig(self, config): + for key in MembaseDriver.DEFAULT_CONFIG.keys(): + assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) + + connection_string = config['host'] +":"+ str(config['port']) + conn_list = [] ; conn_list.append(connection_string) + + self.conn = memcache.Client(conn_list) + #self.database = self.conn[str(config['name'])] + + ## ---------------------------------------------- + ## loadTuples into a csv file + ## ../py-tpcc/src/pytpcc/insert_data.csv + ## ---------------------------------------------- + + def loadTuples_to_csv(self, tableName, tuples): + + if len(tuples) == 0: return + f = open("insert_data.csv",'a') + for tuple_element in tuples: + primary_key = tuple_element[0] + key = tableName +"_"+ str(primary_key) + value = tuple_element[1:] + value = str(value) + value = value.strip('['); value = value.strip(']'); + f.write(key+','+value+"\n") + f.close() + + ## ---------------------------------------------- + ## loadTuples + ## ---------------------------------------------- + + def loadTuples(self, tableName, tuples): + if len(tuples) == 0: + print "NO DATA" + return + temp_max_id = self.conn.get(tableName+"_max_pkey") + if temp_max_id == None: + self.conn.set(tableName+"_max_pkey",0) + temp_max_id = 0 + + columns = TABLE_COLUMNS[tableName] + key_ids = columns[-1] + + key_val_pairs={} + start_time = time.time() + for tuple_element in tuples: + primary_key = tuple_element[0] + if temp_max_id < primary_key: + temp_max_id = primary_key + + key_postfix ="" + for idx in range(0,key_ids): + key_postfix = key_postfix + "_" + str(tuple_element[idx]) + + key = tableName + key_postfix + value = tuple_element + key_val_pairs[key] = value + #tmp_dict={} + #tmp_dict[key]=value + #key_val_pairs.append(tmp_dict) + + #self.conn.set(key,value) + + """ + if tableName == "ORDERS": + okey = "ORDERS_ID_"+str(tuple_element[1])+"_"+str(tuple_element[2])+"_"+str(tuple_element[3]) + ovalue = str(tuple_element[0])+":" + ret = self.conn.append(okey, ovalue) + if ret == False: + self.conn.set(okey, ovalue) + """ + if tableName == "ORDERS": + okey = "ORDERS_ID_"+str(tuple_element[1])+"_"+str(tuple_element[2])+"_"+str(tuple_element[3]) + ovalue = self.conn.get(okey) + if ovalue == None: + ovalue = str(tuple_element[0])+":" + else: + ovalue = ovalue + str(tuple_element[0])+":" + key_val_pairs[okey] = ovalue + + #Use multi_set to send all the keys to vbucket + self.conn.set_multi(key_val_pairs) + + #Define the implicit Schema + end_time = time.time() + self.load_time += (end_time - start_time) + self.conn.replace(tableName+"_max_pkey",temp_max_id) + print "[",tableName,"]"," Current Load Time: ",self.load_time + logging.debug("Loaded %d tuples for tableName %s" % (len(tuples), tableName)) + return + + ## ---------------------------------------------- + ## loadFinish + ## ---------------------------------------------- + + def loadFinish(self): + logging.info("Commiting changes to database") + ## ---------------------------------------------- + ## doOrderStatus + ## ---------------------------------------------- + + + ## ---------------------------------------------- + ## doDelivery + ## ---------------------------------------------- + def doDelivery(self, params): + + w_id = params["w_id"] + o_carrier_id = params["o_carrier_id"] + ol_delivery_d = params["o_carrier_id"] + + result = [ ] + ol_total =0 + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): + + #------------------------------------------------------------------------------------- + # The row in the NEW-ORDER table with matching NO_W_ID (equals W_ID) and NO_D_ID (equals D_ID) + # and with the lowest NO_O_ID value is selected. + #------------------------------------------------------------------------------------- + + for idx in range(1, MAX_ORDER_ID+1): + new_order = self.conn.get("NEW_ORDER_"+str(idx)+"_"+str(d_id)+"_"+str(w_id)) + if new_order != None: + break + + if new_order == None: + continue + + assert len(new_order) > 0 + columns = TABLE_COLUMNS["NEW_ORDER"] + o_id = new_order[ columns.index("NO_O_ID") ] + + #------------------------------------------------------------------------------------- + # The selected row in the NEW-ORDER table is deleted. + #------------------------------------------------------------------------------------- + + self.conn.delete("NEW_ORDER_"+str(o_id)+"_"+str(d_id)+"_"+str(w_id)) + + + #------------------------------------------------------------------------------------- + # The row in the ORDER table with matching O_W_ID (equals W_ ID), O_D_ID (equals D_ID), + # and O_ID (equals NO_O_ID) is selected, O_C_ID, the customer number, is retrieved, + # and O_CARRIER_ID is updated. + #------------------------------------------------------------------------------------- + + # getCId + for idx in range(1, MAX_CUSTOMER_ID+1): + order = self.conn.get("ORDERS_"+str(o_id)+"_"+str(idx)+"_"+str(d_id)+"_"+str(w_id)) + if order != None: + c_id = idx; + break + + + + # "updateOrders" + updated_order = self.update_single_record("O_CARRIER_ID", o_carrier_id, order,"ORDERS"); + self.conn.set("ORDERS_"+str(o_id)+"_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id),updated_order); + + #------------------------------------------------------------------------------------- + # All rows in the ORDER-LINE table with matching OL_W_ID (equals O_W_ID), OL_D_ID (equals O_D_ID), + # and OL_O_ID (equals O_ID) are selected. All OL_DELIVERY_D, the delivery dates, are updated to + # the current system time as returned by the operating system and the sum of all OL_AMOUNT is retrieved. + #------------------------------------------------------------------------------------- + + order_line = self.conn.get("ORDER_LINE_"+str(o_id)+"_"+str(d_id)+"_"+str(w_id)) + updated_order_line = self.update_single_record("OL_DELIVERY_D", ol_delivery_d, order_line,"ORDER_LINE") + self.conn.set("ORDER_LINE_"+str(o_id)+"_"+str(d_id)+"_"+str(w_id), updated_order_line) + + columns = TABLE_COLUMNS["ORDER_LINE"] + selected_order_line = order_line[ columns.index("OL_AMOUNT") ] + + ol_total += selected_order_line + + #------------------------------------------------------------------------------------- + # The row in the CUSTOMER table with matching C_W_ID (equals W_ID), C_D_ID (equals D_ID), + # and C_ID (equals O_C_ID) is selected and C_BALANCE is increased by the sum of all order-line + # amounts (OL_AMOUNT) previously retrieved. C_DELIVERY_CNT is incremented by 1. + #------------------------------------------------------------------------------------- + + # "updateCustomer": + customer = self.conn.get("CUSTOMER_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id)) + updated_customer = self.increment_field("C_BALANCE", ol_total, customer,"CUSTOMER") + updated_customer = self.increment_field("C_DELIVERY_CNT", 1, updated_customer,"CUSTOMER") + self.conn.set("CUSTOMER_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id), updated_customer); + + result.append((d_id, o_id)) + return result + + + ## ---------------------------------------------- + ## doNewOrder + ## ---------------------------------------------- + def doNewOrder(self, params): + def __getItem(items,item_id): + for index,item in irange(items): + if item[0] == item_id: + return index + + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + i_ids = params["i_ids"] + i_w_ids = params["i_w_ids"] + i_qtys = params["i_qtys"] + + o_entry_d = params["o_entry_d"] + s_dist_col = "S_DIST_%02d" % d_id + + assert len(i_ids) > 0 + assert len(i_ids) == len(i_w_ids) + assert len(i_ids) == len(i_qtys) + + items = [] + for item_index in i_ids: + item_key = "ITEM_"+str(item_index) + new_item = self.conn.get(item_key) + if new_item != None: + items.append(new_item) + + #------------------------------------------------------------------------------------- + # For each item on the order: + # The row in the ITEM table with matching I_ID (equals OL_I_ID) is selected and I_PRICE, + # the price of the item, I_NAME, the name of the item, and I_DATA are retrieved. + # If I_ID has an unused value, a "not-found" condition is signaled, + # resulting in a rollback of the database transaction. + #------------------------------------------------------------------------------------- + + if len(items) != len(i_ids): + return + + total = 1 + item_data = [] + + #------------------------------------------------------------------------------------- + + # The row in the WAREHOUSE table with matching W_ID is selected and W_TAX, the warehouse tax rate, is retrieved. + #------------------------------------------------------------------------------------- + warehouse = self.conn.get("WAREHOUSE_"+str(w_id)) + assert warehouse + columns = TABLE_COLUMNS["WAREHOUSE"] + w_tax = warehouse[ columns.index("W_TAX") ] + + #------------------------------------------------------------------------------------- + # The row in the DISTRICT table with matching D_W_ID and D_ ID is selected, D_TAX, the district tax rate, + # is retrieved, and D_NEXT_O_ID, the next available order number for the district, + # is retrieved and incremented by one. + #------------------------------------------------------------------------------------- + district = self.conn.get("DISTRICT_"+str(d_id)+"_"+str(w_id)) + assert district + selected_district = return_columns_single_record(["D_TAX", "D_NEXT_O_ID"], district, "DISTRICT") + d_tax = selected_district[0] + d_next_o_id = selected_district[1] + + updated_district = self.increment_field("D_NEXT_O_ID", 1, district, "DISTRICT"); + self.conn.set("DISTRICT_"+str(d_id)+"_"+str(w_id), updated_district); + + #------------------------------------------------------------------------------------- + # The row in the CUSTOMER table with matching C_W_ID, C_D_ID, and C_ID is selected and C_DISCOUNT, + # the customer's discount rate, C_LAST, the customer's last name, and C_CREDIT, the customer's credit status, + # are retrieved. + #------------------------------------------------------------------------------------- + customer = self.conn.get("CUSTOMER_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id)) + assert customer + selected_customer = return_columns_single_record(["C_DISCOUNT", "C_LAST", "C_CREDIT"], customer, "CUSTOMER") + c_discount = selected_customer[0] + + #------------------------------------------------------------------------------------- + # A new row is inserted into both the NEW-ORDER table and the ORDER table to reflect the creation + # of the new order. O_CARRIER_ID is set to a null value. If the order includes only home order-lines, + # then O_ALL_LOCAL is set to 1, otherwise O_ALL_LOCAL is set to 0. + # The number of items, O_OL_CNT, is computed to match ol_cnt. + #------------------------------------------------------------------------------------- + # from mongodb driver + all_local = (not i_w_ids or [w_id] * len(i_w_ids) == i_w_ids) + + o_all_local = 0 + if all_local: + o_all_local = 1; + ol_cnt = len(i_ids) + o_carrier_id = constants.NULL_CARRIER_ID + value = [o_entry_d, o_carrier_id, ol_cnt, all_local] + self.conn.set("ORDERS_"+str(d_next_o_id)+"_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id), value) + + # update the table that stores the orders id + ret = self.conn.append("ORDERS_ID_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id), str(d_next_o_id)+":") + if ret == None: + self.conn.set("ORDERS_ID_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id), str(d_next_o_id)+":") + + self.conn.set("NEW_ORDERS_"+str(d_next_o_id)+"_"+str(d_id)+"_"+str(w_id),value) + + for idx in i_ids: + + #------------------------------------------------------------------------------------- + # For each O_OL_CNT item on the order: + # The row in the ITEM table with matching I_ID (equals OL_I_ID) is selected and I_PRICE, + # the price of the item, I_NAME, the name of the item, and I_DATA are retrieved. + # If I_ID has an unused value, a "not-found" condition is signaled, + # resulting in a rollback of the database transaction. + #------------------------------------------------------------------------------------- + + i = __getItem(items,idx) + item = self.conn.get("ITEM_"+str(idx)); + if item == None: + return + + selected_item = return_columns_single_record(["I_PRICE", "I_NAME", "I_DATA"], item, "ITEM"); + i_price = selected_item[0] + i_name = selected_item[1] + i_data = selected_item[2] + + + #------------------------------------------------------------------------------------- + # The row in the STOCK table with matching S_I_ID (equals OL_I_ID) and S_W_ID (equals OL_SUPPLY_W_ID) + # is selected. S_QUANTITY, the quantity in stock, S_DIST_xx, where xx represents the district number, + # and S_DATA are retrieved. If the retrieved value for S_QUANTITY exceeds OL_QUANTITY by 10 or more, + # then S_QUANTITY is decreased by OL_QUANTITY; otherwise S_QUANTITY is updated to (S_QUANTITY - OL_QUANTITY)+91. + # S_YTD is increased by OL_QUANTITY and S_ORDER_CNT is incremented by 1. If the order-line is remote, then + # S_REMOTE_CNT is incremented by 1. + #------------------------------------------------------------------------------------- + + ol_supply_w_id = i_w_ids[i] + stock = self.conn.get("STOCK_"+str(idx)+"_"+str(ol_supply_w_id)) + assert stock + + selected_stock = return_columns_single_record(["S_QUANTITY", "S_DATA", "S_YTD", "S_ORDER_CNT", "S_REMOTE_CNT", s_dist_col ], stock, "STOCK") + s_quantity = selected_stock[0] + s_data = selected_stock[1] + ol_quantity = i_qtys[i] + if(s_quantity > (ol_quantity+10)): + s_quantity = s_quantity - ol_quantity + else: + s_quantity = (s_quantity - ol_quantity)+91 + + updated_stock = self.update_single_record("S_QUANTITY", s_quantity, stock,"STOCK"); + updated_stock = self.increment_field("S_YTD", ol_quantity, updated_stock, "STOCK") + updated_stock = self.increment_field("S_ORDER_CNT", 1, updated_stock, "STOCK") + if o_all_local == 1: + updated_stock = self.increment_field("S_REMOTE_CNT", 1, updated_stock, "STOCK") + + self.conn.set("STOCK_"+str(idx)+"_"+str(ol_supply_w_id), updated_stock); + + if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: + brand_generic = 'B' + else: + brand_generic = 'G' + item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_quantity) ) + + total *= (1 - c_discount) * (1 + w_tax + d_tax) + misc = [ (w_tax, d_tax, d_next_o_id, total) ] + + return [ customer, misc, item_data ] + + + ## ---------------------------------------------- + ## doOrderStatus + def doOrderStatus(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + + + assert w_id, pformat(params) + assert d_id, pformat(params) + + #------------------------------------------------------------------------------------- + # Case 1, the customer is selected based on customer number: the row in the CUSTOMER table with matching + # C_W_ID, C_D_ID, and C_ID is selected and C_BALANCE, C_FIRST, C_MIDDLE, and C_LAST are retrieved. + #------------------------------------------------------------------------------------- + + if c_id != None: + customer = self.conn.get("CUSTOMER_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id)) + + #------------------------------------------------------------------------------------- + # Case 2, the customer is selected based on customer last name: all rows in the CUSTOMER table with + # matching C_W_ID, C_D_ID and C_LAST are selected sorted by C_FIRST in ascending order. + # Let n be the number of rows selected. C_BALANCE, C_FIRST, C_MIDDLE, and C_LAST are retrieved from + # the row at position n/2 rounded up in the sorted set of selected rows from the CUSTOMER table. + #------------------------------------------------------------------------------------- + else: + all_customers= [] + for idx in range(1, MAX_CUSTOMER_ID+1): + customer_record = self.conn.get("CUSTOMER_"+str(idx)+"_"+str(d_id)+"_"+str(w_id)) + all_customers.append(customer_record) + filtered_customer_table = filter_table([["C_LAST",c_last]],all_customers,"CUSTOMER") + ordered_customer_map = self.order_by("C_FIRST", filtered_customer_table, "CUSTOMER"); + + # Get the midpoint customer's id + namecnt = len(filtered_customer_table) + assert namecnt > 0 + index = ordered_customer_map[(namecnt-1)/2][0] + customer = filtered_customer_table[index] + + columns = TABLE_COLUMNS["CUSTOMER"] + c_index = columns.index('C_ID') + c_id = customer[c_index] + + selected_customer = return_columns_single_record(["C_BALANCE", "C_FIRST", "C_MIDDLE", "C_LAST"], customer, "CUSTOMER") + + #------------------------------------------------------------------------------------- + # The row in the ORDER table with matching O_W_ID (equals C_W_ID), O_D_ID (equals C_D_ID), O_C_ID (equals C_ID), + # and with the largest existing O_ID, is selected. This is the most recent order placed by that customer. + # O_ID, O_ENTRY_D, and O_CARRIER_ID are retrieved. + #-------------------------------------------------------------------------------------• + + # read all the orders id from table + all_orders_id_str = self.conn.get("ORDERS_ID_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id)) + assert all_orders_id_str + all_orders_id_array = all_orders_id_str.split(":"); + o_id = 0; + + # the last element is always empty because we append : at the end + for j in range(len(all_orders_id_array)-1): + o_str = all_orders_id_array[j] + o = int(o_str) + if o > o_id: + o_id = o + order = self.conn.get("ORDERS_"+str(o_id)+"_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id)) + + + + #------------------------------------------------------------------------------------- + # All rows in the ORDER-LINE table with matching OL_W_ID (equals O_W_ID), OL_D_ID (equals O_D_ID), + # and OL_O_ID (equals O_ID) are selected and the corresponding sets of OL_I_ID, OL_SUPPLY_W_ID, + # OL_QUANTITY, OL_AMOUNT, and OL_DELIVERY_D are retrieved. + #------------------------------------------------------------------------------------- + + order_lines = self.conn.get("ORDER_LINE_"+str(o_id)+"_"+str(d_id)+"_"+str(w_id)) + filter_order_lines = return_columns_single_record(["OL_SUPPLY_W_ID", "OL_I_ID", "OL_QUANTITY", "OL_AMOUNT", "OL_DELIVERY_D"],order_lines, "ORDER_LINE") + return [ selected_customer, order, filter_order_lines ] + + + ## ---------------------------------------------- + ## doPayment + ## ---------------------------------------------- + def doPayment(self, params): + #print "PAYMENT" + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_w_id = params["c_w_id"] + c_d_id = params["c_d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + h_amount = params["h_amount"]; + h_date = params["h_date"] + # if w_id = c_w_id the transaction is "home", otherwise "remote" + + + #------------------------------------------------------------------------------------- + # The row in the WAREHOUSE table with matching W_ID is selected. + # W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, and W_ZIP are retrieved + # and W_YTD, the warehouse's year-to-date balance, is increased by H_ AMOUNT. + #------------------------------------------------------------------------------------- + + warehouse = self.conn.get("WAREHOUSE_"+str(w_id)) + updated_warehouse = self.increment_field("W_YTD", h_amount, warehouse,"WAREHOUSE") + self.conn.set("WAREHOUSE_"+str(w_id), updated_warehouse); + selected_warehouse = return_columns_single_record(["W_NAME", "W_STREET_1", "W_STREET_2", "W_CITY", "W_STATE", "W_ZIP"], updated_warehouse, "WAREHOUSE") + + #------------------------------------------------------------------------------------- + # The row in the DISTRICT table with matching D_W_ID and D_ID is selected. + # D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, and D_ZIP are retrieved and D_YTD, + # the district's year-to-date balance, is increased by H_AMOUNT. + #------------------------------------------------------------------------------------- + + district = self.conn.get("DISTRICT_"+str(d_id)+"_"+str(w_id)) + selected_district = return_columns_single_record(["D_NAME", "D_STREET_1", "D_STREET_2", "D_CITY", "D_STATE", "D_ZIP"], district, "DISTRICT") + updated_district = self.increment_field("D_YTD", h_amount, district,"DISTRICT") + self.conn.set("DISTRICT_"+str(d_id)+"_"+str(w_id),updated_district) + + #------------------------------------------------------------------------------------- + # Case 1, the customer is selected based on customer number: the row in the CUSTOMER table with + # matching C_W_ID, C_D_ID and C_ID is selected. C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, + # C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, and C_BALANCE are retrieved. + # C_BALANCE is decreased by H_AMOUNT. C_YTD_PAYMENT is increased by H_AMOUNT. C_PAYMENT_CNT is incremented by 1. + #------------------------------------------------------------------------------------- + + if c_id != None: + customer = self.conn.get("CUSTOMER_"+str(c_id)+"_"+str(c_d_id)+"_"+str(c_w_id)) + selected_customer = return_columns_single_record(["C_ID", "C_FIRST", "C_MIDDLE", "C_LAST", "C_STREET_1", "C_STREET_2", "C_CITY", "C_STATE", "C_ZIP", "C_PHONE", "C_SINCE", "C_CREDIT", "C_CREDIT_LIM", "C_DISCOUNT", "C_BALANCE", "C_YTD_PAYMENT", "C_PAYMENT_CNT", "C_DATA"], customer, "CUSTOMER") + #c_balance = selected_customer[14] + #c_ytd_payment = selected_customer[15] + #c_payment_cnt = selected_customer[16] + #c_data = selected_customer[17] + + updated_customer = self.increment_field("C_BALANCE", -1 * h_amount, customer, "CUSTOMER") + updated_customer = self.increment_field("C_YTD_PAYMENT", h_amount, updated_customer, "CUSTOMER") + updated_customer = self.increment_field("C_PAYMENT_CNT", 1, updated_customer, "CUSTOMER") + self.conn.set("CUSTOMER_"+str(c_id)+"_"+str(c_d_id)+"_"+str(c_w_id), updated_customer) + + #------------------------------------------------------------------------------------- + # Case 2, the customer is selected based on customer last name: all rows in the CUSTOMER table with matching + # C_W_ID, C_D_ID and C_LAST are selected sorted by C_FIRST in ascending order. Let n be the number of rows selected. + # C_ID, C_FIRST, C_MIDDLE, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, + # C_DISCOUNT, and C_BALANCE are retrieved from the row at position (n/2 rounded up to the next integer) + # in the sorted set of selected rows from the CUSTOMER table. C_BALANCE is decreased by H_AMOUNT. + # C_YTD_PAYMENT is increased by H_AMOUNT. C_PAYMENT_CNT is incremented by 1. + #------------------------------------------------------------------------------------- + + else: + + all_customers= [] + for idx in range(1, MAX_CUSTOMER_ID+1): + customer_record = self.conn.get("CUSTOMER_"+str(idx)+"_"+str(c_d_id)+"_"+str(c_w_id)) + if customer_record != None: + all_customers.append(customer_record) + + filter_customer_table = filter_table([["C_LAST",c_last]],all_customers,"CUSTOMER") + ordered_customer_map = self.order_by("C_FIRST", filter_customer_table, "CUSTOMER"); + + # Get the midpoint customer's id + namecnt = len(filter_customer_table) + assert namecnt > 0 + index = ordered_customer_map[(namecnt-1)/2][0] + customer = filter_customer_table[index] + + columns = TABLE_COLUMNS["CUSTOMER"] + c_index = columns.index('C_ID') + + c_id = customer[c_index] + + selected_customer = return_columns_single_record(["C_ID", "C_FIRST", "C_MIDDLE", "C_LAST", "C_STREET_1", "C_STREET_2", "C_CITY", "C_STATE", "C_ZIP", "C_PHONE", "C_SINCE", "C_CREDIT", "C_CREDIT_LIM", "C_DISCOUNT", "C_BALANCE", "C_YTD_PAYMENT", "C_PAYMENT_CNT", "C_DATA"], customer, "CUSTOMER") + + updated_customer = self.increment_field("C_BALANCE", -1* h_amount, customer, "CUSTOMER") + updated_customer = self.increment_field("C_YTD_PAYMENT", h_amount, updated_customer, "CUSTOMER") + updated_customer = self.increment_field("C_PAYMENT_CNT", 1, updated_customer, "CUSTOMER") + self.conn.set("CUSTOMER_"+str(c_id)+"_"+str(c_d_id)+"_"+str(c_w_id), updated_customer) + + assert len(customer) > 0 + c_data = selected_customer[17] + #------------------------------------------------------------------------------------- + # If the value of C_CREDIT is equal to "BC", then C_DATA is also retrieved from the selected customer and + # the following history information: C_ID, C_D_ID, C_W_ID, D_ID, W_ID, and H_AMOUNT, are inserted at the + # left of the C_DATA field by shifting the existing content of C_DATA to the right by an equal number of + # bytes and by discarding the bytes that are shifted out of the right side of the C_DATA field. + # The content of the C_DATA field never exceeds 500 characters. The selected customer is updated with + # the new C_DATA field. + #------------------------------------------------------------------------------------- + columns = TABLE_COLUMNS["CUSTOMER"] + credit_index = columns.index("C_CREDIT") + + if(customer[credit_index] == constants.BAD_CREDIT): + newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) + c_data = (newData + "|" + c_data) + if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] + + updated_customer = self.update_single_record("C_DATA", c_data, updated_customer,"CUSTOMER") + + #------------------------------------------------------------------------------------- + # H_DATA is built by concatenating W_NAME and D_NAME separated by 4 spaces. + # A new row is inserted into the HISTORY table with H_C_ID = C_ID, H_C_D_ID = C_D_ID, + # H_C_W_ID = C_W_ID, H_D_ID = D_ID, and H_W_ID = W_ID. + #------------------------------------------------------------------------------------- + columns = TABLE_COLUMNS["WAREHOUSE"] + w_index = columns.index("W_NAME") + columns = TABLE_COLUMNS["DISTRICT"] + d_index = columns.index("D_NAME") + + h_data = "%s %s" % (warehouse[w_index], district[d_index]) + value = [ d_id, w_id, h_date, h_amount, h_data ] + self.conn.set("HISTORY_"+str(c_id)+"_"+str(c_d_id)+"_"+str(c_w_id)+"_"+str(d_id)+"_"+str(w_id),value) + return [ selected_warehouse, selected_district, updated_customer ] + + ## ---------------------------------------------- + ## doStockLevel + ## ---------------------------------------------- + def doStockLevel(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + threshold = params["threshold"] + + #------------------------------------------------------------------------------------- + # The row in the DISTRICT table with matching D_W_ID and D_ID is selected and D_NEXT_O_ID is retrieved. + #------------------------------------------------------------------------------------- + + district = self.conn.get("DISTRICT_"+str(d_id)+"_"+str(w_id)) + selected_district = return_columns_single_record(["D_NEXT_O_ID"], district, "DISTRICT") + next_o_id = selected_district[0] + #------------------------------------------------------------------------------------- + # All rows in the ORDER-LINE table with matching OL_W_ID (equals W_ID), OL_D_ID (equals D_ID), + # and OL_O_ID (lower than D_NEXT_O_ID and greater than or equal to D_NEXT_O_ID minus 20) are selected. + # They are the items for 20 recent orders of the district. + #------------------------------------------------------------------------------------- + + columns = TABLE_COLUMNS["ORDER_LINE"] + idx_O_ID = columns.index("OL_O_ID") + + # read the customers for the district and warehouse + """ + all_customers_idx = [] + for idx in range(1, MAX_CUSTOMER_ID+1): + customer_record = self.conn.get("CUSTOMER_"+str(idx)+"_"+str(d_id)+"_"+str(w_id)) + if customer_record != None: + all_customers_idx.append(idx) + """ + + # read all the orders for the customers + all_orders = [] + for idx in range(1,MAX_CUSTOMER_ID+1): + order_ids_str = self.conn.get("ORDERS_ID_"+str(idx)+"_"+str(d_id)+"_"+str(w_id)) + if order_ids_str != None: + order_ids = order_ids_str.split(":"); + for j in range(len(order_ids)-1): + o_id = int(order_ids[j]) + order_record = self.conn.get("ORDER_LINE_"+str(o_id)+"_"+str(d_id)+"_"+str(w_id)) + if order_record != None: + if order_record[idx_O_ID] < next_o_id or order_record[idx_O_ID] >= (next_o_id - 20): + all_orders.append(order_record) + + assert all_orders + order_lines_i = return_columns(["OL_O_ID"], all_orders, "ORDER_LINE") + + #------------------------------------------------------------------------------------- + # All rows in the STOCK table with matching S_I_ID (equals OL_I_ID) and S_W_ID (equals W_ID) + # from the list of distinct item numbers and with S_QUANTITY lower than threshold are counted (giving low_stock). + #------------------------------------------------------------------------------------- + + all_stock = [] + for idx in order_lines_i: + idx = idx[0] + stock = self.conn.get("STOCK_"+str(idx)+"_"+str(w_id)) + if stock != None: + quantity = return_columns_single_record(["S_QUANTITY"], stock, "STOCK") + if quantity[0] < threshold : + all_stock.append(stock) + + assert all_stock + if len(all_stock) ==1: + stock_i = return_columns_single_record(["S_I_ID"],all_stock,"STOCK") + else: + stock_i = return_columns(["S_I_ID"], all_stock, "STOCK") + + stock_i = self.count_distinct_values(stock_i) + return [ stock_i ] + + ## ---------------------------------------------- + ## Update single record + ## ---------------------------------------------- + + def update_single_record(self,field, value, record, table_name): + columns = TABLE_COLUMNS[table_name] + field_index = columns.index(field) + record[field_index] = value + return record + + + ## ---------------------------------------------- + ## increment_field + ## ---------------------------------------------- + def increment_field(self,field, value, record, table_name): + columns = TABLE_COLUMNS[table_name] + field_index = columns.index(field) + + record[field_index] += value + return record + + ## ---------------------------------------------- + ## count_distinct_values + ## ---------------------------------------------- + def count_distinct_values(self,values): + value_map={} + for val in values: + value_map[val[0]]=1 + + return len(value_map) + + ## ---------------------------------------------- + ## order_by + ## ---------------------------------------------- + #order_by("C_FIRST", filtered_customer_table, "CUSTOMER"); + + def order_by(self,field,records,table_name): + columns = TABLE_COLUMNS[table_name] + field_index = columns.index(field) + #filter_records = return_columns([field],records) + orderby_map={} + for idx,tvalue in irange(records): + field_value = tvalue[field_index] + orderby_map[idx] = field_value + + import operator + sorted_index_list = sorted(orderby_map.items(), key=operator.itemgetter(1),reverse=False) + + return sorted_index_list + #store index, value & then sort... diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/mongodbdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/mongodbdriver.py new file mode 100644 index 00000000..d4a73a82 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/mongodbdriver.py @@ -0,0 +1,809 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Original Java Version: +# Copyright (C) 2008 +# Evan Jones +# Massachusetts Institute of Technology +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +from __future__ import with_statement + +import os +import sys +import logging +import pymongo +from pprint import pprint,pformat + +import constants +from abstractdriver import * + +TABLE_COLUMNS = { + constants.TABLENAME_ITEM: [ + "I_ID", # INTEGER + "I_IM_ID", # INTEGER + "I_NAME", # VARCHAR + "I_PRICE", # FLOAT + "I_DATA", # VARCHAR + ], + constants.TABLENAME_WAREHOUSE: [ + "W_ID", # SMALLINT + "W_NAME", # VARCHAR + "W_STREET_1", # VARCHAR + "W_STREET_2", # VARCHAR + "W_CITY", # VARCHAR + "W_STATE", # VARCHAR + "W_ZIP", # VARCHAR + "W_TAX", # FLOAT + "W_YTD", # FLOAT + ], + constants.TABLENAME_DISTRICT: [ + "D_ID", # TINYINT + "D_W_ID", # SMALLINT + "D_NAME", # VARCHAR + "D_STREET_1", # VARCHAR + "D_STREET_2", # VARCHAR + "D_CITY", # VARCHAR + "D_STATE", # VARCHAR + "D_ZIP", # VARCHAR + "D_TAX", # FLOAT + "D_YTD", # FLOAT + "D_NEXT_O_ID", # INT + ], + constants.TABLENAME_CUSTOMER: [ + "C_ID", # INTEGER + "C_D_ID", # TINYINT + "C_W_ID", # SMALLINT + "C_FIRST", # VARCHAR + "C_MIDDLE", # VARCHAR + "C_LAST", # VARCHAR + "C_STREET_1", # VARCHAR + "C_STREET_2", # VARCHAR + "C_CITY", # VARCHAR + "C_STATE", # VARCHAR + "C_ZIP", # VARCHAR + "C_PHONE", # VARCHAR + "C_SINCE", # TIMESTAMP + "C_CREDIT", # VARCHAR + "C_CREDIT_LIM", # FLOAT + "C_DISCOUNT", # FLOAT + "C_BALANCE", # FLOAT + "C_YTD_PAYMENT", # FLOAT + "C_PAYMENT_CNT", # INTEGER + "C_DELIVERY_CNT", # INTEGER + "C_DATA", # VARCHAR + ], + constants.TABLENAME_STOCK: [ + "S_I_ID", # INTEGER + "S_W_ID", # SMALLINT + "S_QUANTITY", # INTEGER + "S_DIST_01", # VARCHAR + "S_DIST_02", # VARCHAR + "S_DIST_03", # VARCHAR + "S_DIST_04", # VARCHAR + "S_DIST_05", # VARCHAR + "S_DIST_06", # VARCHAR + "S_DIST_07", # VARCHAR + "S_DIST_08", # VARCHAR + "S_DIST_09", # VARCHAR + "S_DIST_10", # VARCHAR + "S_YTD", # INTEGER + "S_ORDER_CNT", # INTEGER + "S_REMOTE_CNT", # INTEGER + "S_DATA", # VARCHAR + ], + constants.TABLENAME_ORDERS: [ + "O_ID", # INTEGER + "O_C_ID", # INTEGER + "O_D_ID", # TINYINT + "O_W_ID", # SMALLINT + "O_ENTRY_D", # TIMESTAMP + "O_CARRIER_ID", # INTEGER + "O_OL_CNT", # INTEGER + "O_ALL_LOCAL", # INTEGER + ], + constants.TABLENAME_NEW_ORDER: [ + "NO_O_ID", # INTEGER + "NO_D_ID", # TINYINT + "NO_W_ID", # SMALLINT + ], + constants.TABLENAME_ORDER_LINE: [ + "OL_O_ID", # INTEGER + "OL_D_ID", # TINYINT + "OL_W_ID", # SMALLINT + "OL_NUMBER", # INTEGER + "OL_I_ID", # INTEGER + "OL_SUPPLY_W_ID", # SMALLINT + "OL_DELIVERY_D", # TIMESTAMP + "OL_QUANTITY", # INTEGER + "OL_AMOUNT", # FLOAT + "OL_DIST_INFO", # VARCHAR + ], + constants.TABLENAME_HISTORY: [ + "H_C_ID", # INTEGER + "H_C_D_ID", # TINYINT + "H_C_W_ID", # SMALLINT + "H_D_ID", # TINYINT + "H_W_ID", # SMALLINT + "H_DATE", # TIMESTAMP + "H_AMOUNT", # FLOAT + "H_DATA", # VARCHAR + ], +} +TABLE_INDEXES = { + constants.TABLENAME_ITEM: [ + "I_ID", + ], + constants.TABLENAME_WAREHOUSE: [ + "W_ID", + ], + constants.TABLENAME_DISTRICT: [ + "D_ID", + "D_W_ID", + ], + constants.TABLENAME_CUSTOMER: [ + "C_ID", + "C_D_ID", + "C_W_ID", + ], + constants.TABLENAME_STOCK: [ + "S_I_ID", + "S_W_ID", + ], + constants.TABLENAME_ORDERS: [ + "O_ID", + "O_D_ID", + "O_W_ID", + "O_C_ID", + ], + constants.TABLENAME_NEW_ORDER: [ + "NO_O_ID", + "NO_D_ID", + "NO_W_ID", + ], + constants.TABLENAME_ORDER_LINE: [ + "OL_O_ID", + "OL_D_ID", + "OL_W_ID", + ], +} + +## ============================================== +## MongodbDriver +## ============================================== +class MongodbDriver(AbstractDriver): + DEFAULT_CONFIG = { + "host": ("The hostname to mongod", "localhost" ), + "port": ("The port number to mongod", 27017 ), + "name": ("Collection name", "tpcc"), + "denormalize": ("If set to true, then the CUSTOMER data will be denormalized into a single document", True), + } + DENORMALIZED_TABLES = [ + constants.TABLENAME_CUSTOMER, + constants.TABLENAME_ORDERS, + constants.TABLENAME_ORDER_LINE, + constants.TABLENAME_HISTORY, + ] + + + def __init__(self, ddl): + super(MongodbDriver, self).__init__("mongodb", ddl) + self.database = None + self.conn = None + self.denormalize = False + self.w_customers = { } + self.w_orders = { } + + ## Create member mapping to collections + for name in constants.ALL_TABLES: + self.__dict__[name.lower()] = None + + ## ---------------------------------------------- + ## makeDefaultConfig + ## ---------------------------------------------- + def makeDefaultConfig(self): + return MongodbDriver.DEFAULT_CONFIG + + ## ---------------------------------------------- + ## loadConfig + ## ---------------------------------------------- + def loadConfig(self, config): + for key in MongodbDriver.DEFAULT_CONFIG.keys(): + assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) + + self.conn = pymongo.MongoClient(config['host'], int(config['port'])) + self.database = self.conn[str(config['name'])] + self.denormalize = config['denormalize'] + if self.denormalize: logging.debug("Using denormalized data model") + + if config["reset"]: + logging.debug("Deleting database '%s'" % self.database.name) + for name in constants.ALL_TABLES: + if name in self.database.collection_names(): + self.database.drop_collection(name) + logging.debug("Dropped collection %s" % name) + ## IF + + ## Setup! + load_indexes = ('execute' in config and not config['execute']) and \ + ('load' in config and not config['load']) + for name in constants.ALL_TABLES: + if self.denormalize and name in MongodbDriver.DENORMALIZED_TABLES[1:]: continue + + ## Create member mapping to collections + self.__dict__[name.lower()] = self.database[name] + + ## Create Indexes + if load_indexes and name in TABLE_INDEXES and \ + (self.denormalize or (self.denormalize == False and not name in MongodbDriver.DENORMALIZED_TABLES[1:])): + logging.debug("Creating index for %s" % name) + for index in TABLE_INDEXES[name]: + self.database[name].create_index(index) + ## FOR + + ## ---------------------------------------------- + ## loadTuples + ## ---------------------------------------------- + def loadTuples(self, tableName, tuples): + if len(tuples) == 0: return + logging.debug("Loading %d tuples for tableName %s" % (len(tuples), tableName)) + + assert tableName in TABLE_COLUMNS, "Unexpected table %s" % tableName + columns = TABLE_COLUMNS[tableName] + num_columns = range(len(columns)) + + tuple_dicts = [ ] + + ## We want to combine all of a CUSTOMER's ORDERS, ORDER_LINE, and HISTORY records + ## into a single document + if self.denormalize and tableName in MongodbDriver.DENORMALIZED_TABLES: + ## If this is the CUSTOMER table, then we'll just store the record locally for now + if tableName == constants.TABLENAME_CUSTOMER: + for t in tuples: + key = tuple(t[:3]) # C_ID, D_ID, W_ID + self.w_customers[key] = dict(map(lambda i: (columns[i], t[i]), num_columns)) + ## FOR + + ## If this is an ORDER_LINE record, then we need to stick it inside of the + ## right ORDERS record + elif tableName == constants.TABLENAME_ORDER_LINE: + for t in tuples: + o_key = tuple(t[:3]) # O_ID, O_D_ID, O_W_ID + (c_key, o_idx) = self.w_orders[o_key] + c = self.w_customers[c_key] + assert o_idx >= 0 + assert o_idx < len(c[constants.TABLENAME_ORDERS]) + o = c[constants.TABLENAME_ORDERS][o_idx] + if not tableName in o: o[tableName] = [ ] + o[tableName].append(dict(map(lambda i: (columns[i], t[i]), num_columns[4:]))) + ## FOR + + ## Otherwise we have to find the CUSTOMER record for the other tables + ## and append ourselves to them + else: + if tableName == constants.TABLENAME_ORDERS: + key_start = 1 + cols = num_columns[0:1] + num_columns[4:] # Removes O_C_ID, O_D_ID, O_W_ID + else: + key_start = 0 + cols = num_columns[3:] # Removes H_C_ID, H_C_D_ID, H_C_W_ID + + for t in tuples: + c_key = tuple(t[key_start:key_start+3]) # C_ID, D_ID, W_ID + assert c_key in self.w_customers, "Customer Key: %s\nAll Keys:\n%s" % (str(c_key), "\n".join(map(str, sorted(self.w_customers.keys())))) + c = self.w_customers[c_key] + + if not tableName in c: c[tableName] = [ ] + c[tableName].append(dict(map(lambda i: (columns[i], t[i]), cols))) + + ## Since ORDER_LINE doesn't have a C_ID, we have to store a reference to + ## this ORDERS record so that we can look it up later + if tableName == constants.TABLENAME_ORDERS: + o_key = (t[0], t[2], t[3]) # O_ID, O_D_ID, O_W_ID + self.w_orders[o_key] = (c_key, len(c[tableName])-1) # CUSTOMER, ORDER IDX + ## FOR + ## IF + + ## Otherwise just shove the tuples straight to the target collection + else: + for t in tuples: + tuple_dicts.append(dict(map(lambda i: (columns[i], t[i]), num_columns))) + ## FOR + self.database[tableName].insert(tuple_dicts) + ## IF + + return + + ## ---------------------------------------------- + ## loadFinishDistrict + ## ---------------------------------------------- + def loadFinishDistrict(self, w_id, d_id): + if self.denormalize: + logging.debug("Pushing %d denormalized CUSTOMER records for WAREHOUSE %d DISTRICT %d into MongoDB" % (len(self.w_customers), w_id, d_id)) + self.database[constants.TABLENAME_CUSTOMER].insert(self.w_customers.values()) + self.w_customers.clear() + self.w_orders.clear() + ## IF + + ## ---------------------------------------------- + ## loadFinish + ## ---------------------------------------------- + def loadFinish(self): + logging.info("Finished loading tables") + if logging.getLogger().isEnabledFor(logging.DEBUG): + for name in constants.ALL_TABLES: + if self.denormalize and name in MongodbDriver.DENORMALIZED_TABLES[1:]: continue + logging.debug("%-12s%d records" % (name+":", self.database[name].count())) + ## IF + + ## ---------------------------------------------- + ## doDelivery + ## ---------------------------------------------- + def doDelivery(self, params): + w_id = params["w_id"] + o_carrier_id = params["o_carrier_id"] + ol_delivery_d = params["ol_delivery_d"] + + result = [ ] + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): + ## getNewOrder + no = self.new_order.find_one({"NO_D_ID": d_id, "NO_W_ID": w_id}, {"NO_O_ID": 1}) + if no == None: + ## No orders for this district: skip it. Note: This must be reported if > 1% + continue + assert len(no) > 0 + o_id = no["NO_O_ID"] + + if self.denormalize: + ## getCId + c = self.customer.find_one({"ORDERS.O_ID": o_id, "C_D_ID": d_id, "C_W_ID": w_id}, {"C_ID": 1, "ORDERS.O_ID": 1, "ORDERS.ORDER_LINE": 1}) + assert c != None, "No customer record [O_ID=%d, D_ID=%d, W_ID=%d]" % (o_id, d_id, w_id) + c_id = c["C_ID"] + + ## sumOLAmount + updateOrderLine + ol_total = 0 + for o in c["ORDERS"]: + if o["O_ID"] == o_id: + orderLines = o["ORDER_LINE"] + for ol in orderLines: + ol_total += ol["OL_AMOUNT"] + ## We have to do this here because we can't update the nested array atomically + ol["OL_DELIVERY_D"] = ol_delivery_d + break + ## FOR + + if ol_total == 0: + pprint(params) + pprint(no) + pprint(c) + sys.exit(1) + + ## updateOrders + updateCustomer + self.customer.update({"_id": c['_id'], "ORDERS.O_ID": o_id}, {"$set": {"ORDERS.$.O_CARRIER_ID": o_carrier_id, "ORDERS.$.ORDER_LINE": orderLines}, "$inc": {"C_BALANCE": ol_total}}, multi=False) + + else: + ## getCId + o = self.orders.find_one({"O_ID": o_id, "O_D_ID": d_id, "O_W_ID": w_id}, {"O_C_ID": 1}) + assert o != None + c_id = o["O_C_ID"] + + ## sumOLAmount + orderLines = self.order_line.find({"OL_O_ID": o_id, "OL_D_ID": d_id, "OL_W_ID": w_id}, {"OL_AMOUNT": 1}) + assert orderLines != None + ol_total = sum([ol["OL_AMOUNT"] for ol in orderLines]) + + ## updateOrders + self.orders.update(o, {"$set": {"O_CARRIER_ID": o_carrier_id}}, multi=False) + + ## updateOrderLine + self.order_line.update({"OL_O_ID": o_id, "OL_D_ID": d_id, "OL_W_ID": w_id}, {"$set": {"OL_DELIVERY_D": ol_delivery_d}}, multi=True) + + ## updateCustomer + self.customer.update({"C_ID": c_id, "C_D_ID": d_id, "C_W_ID": w_id}, {"$inc": {"C_BALANCE": ol_total}}) + ## IF + + ## deleteNewOrder + self.new_order.remove({"_id": no['_id']}) + + # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) + # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure + # them out + # If there are no order lines, SUM returns null. There should always be order lines. + assert ol_total != None, "ol_total is NULL: there are no order lines. This should not happen" + assert ol_total > 0.0 + + result.append((d_id, o_id)) + ## FOR + return result + + ## ---------------------------------------------- + ## doNewOrder + ## ---------------------------------------------- + def doNewOrder(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + o_entry_d = params["o_entry_d"] + i_ids = params["i_ids"] + i_w_ids = params["i_w_ids"] + i_qtys = params["i_qtys"] + s_dist_col = "S_DIST_%02d" % d_id + + assert len(i_ids) > 0 + assert len(i_ids) == len(i_w_ids) + assert len(i_ids) == len(i_qtys) + + ## http://stackoverflow.com/q/3844931/ + all_local = (not i_w_ids or [w_id] * len(i_w_ids) == i_w_ids) + + items = self.item.find({"I_ID": {"$in": i_ids}}, {"I_ID": 1, "I_PRICE": 1, "I_NAME": 1, "I_DATA": 1}) + ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. + ## Note that this will happen with 1% of transactions on purpose. + if items.count() != len(i_ids): + ## TODO Abort here! + return + ## IF + + ## ---------------- + ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER + ## ---------------- + + # getWarehouseTaxRate + w = self.warehouse.find_one({"W_ID": w_id}, {"W_TAX": 1}) + assert w + w_tax = w["W_TAX"] + + # getDistrict + d = self.district.find_one({"D_ID": d_id, "D_W_ID": w_id}, {"D_TAX": 1, "D_NEXT_O_ID": 1}) + assert d + d_tax = d["D_TAX"] + d_next_o_id = d["D_NEXT_O_ID"] + + # incrementNextOrderId + # HACK: This is not transactionally safe! + self.district.update(d, {"$inc": {"D_NEXT_O_ID": 1}}, multi=False) + + # getCustomer + c = self.customer.find_one({"C_ID": c_id, "C_D_ID": d_id, "C_W_ID": w_id}, {"C_DISCOUNT": 1, "C_LAST": 1, "C_CREDIT": 1}) + assert c + c_discount = c["C_DISCOUNT"] + + ## ---------------- + ## Insert Order Information + ## ---------------- + ol_cnt = len(i_ids) + o_carrier_id = constants.NULL_CARRIER_ID + + # createNewOrder + self.new_order.insert({"NO_O_ID": d_next_o_id, "NO_D_ID": d_id, "NO_W_ID": w_id}) + + o = {"O_ID": d_next_o_id, "O_ENTRY_D": o_entry_d, "O_CARRIER_ID": o_carrier_id, "O_OL_CNT": ol_cnt, "O_ALL_LOCAL": all_local} + if self.denormalize: + o[constants.TABLENAME_ORDER_LINE] = [ ] + else: + o["O_D_ID"] = d_id + o["O_W_ID"] = w_id + o["O_C_ID"] = c_id + + # createOrder + self.orders.insert(o) + + ## ---------------- + ## OPTIMIZATION: + ## If all of the items are at the same warehouse, then we'll issue a single + ## request to get their information + ## ---------------- + stockInfos = None + if all_local and False: + # getStockInfo + allStocks = self.stock.find({"S_I_ID": {"$in": i_ids}, "S_W_ID": w_id}, {"S_I_ID": 1, "S_QUANTITY": 1, "S_DATA": 1, "S_YTD": 1, "S_ORDER_CNT": 1, "S_REMOTE_CNT": 1, s_dist_col: 1}) + assert allStocks.count() == ol_cnt + stockInfos = { } + for si in allStocks: + stockInfos["S_I_ID"] = si # HACK + ## IF + + ## ---------------- + ## Insert Order Item Information + ## ---------------- + item_data = [ ] + total = 0 + for i in range(ol_cnt): + ol_number = i + 1 + ol_supply_w_id = i_w_ids[i] + ol_i_id = i_ids[i] + ol_quantity = i_qtys[i] + + itemInfo = items[i] + i_name = itemInfo["I_NAME"] + i_data = itemInfo["I_DATA"] + i_price = itemInfo["I_PRICE"] + + # getStockInfo + if all_local and stockInfos != None: + si = stockInfos[ol_i_id] + assert si["S_I_ID"] == ol_i_id, "S_I_ID should be %d\n%s" % (ol_i_id, pformat(si)) + else: + si = self.stock.find_one({"S_I_ID": ol_i_id, "S_W_ID": w_id}, {"S_I_ID": 1, "S_QUANTITY": 1, "S_DATA": 1, "S_YTD": 1, "S_ORDER_CNT": 1, "S_REMOTE_CNT": 1, s_dist_col: 1}) + assert si, "Failed to find S_I_ID: %d\n%s" % (ol_i_id, pformat(itemInfo)) + + s_quantity = si["S_QUANTITY"] + s_ytd = si["S_YTD"] + s_order_cnt = si["S_ORDER_CNT"] + s_remote_cnt = si["S_REMOTE_CNT"] + s_data = si["S_DATA"] + s_dist_xx = si[s_dist_col] # Fetches data from the s_dist_[d_id] column + + ## Update stock + s_ytd += ol_quantity + if s_quantity >= ol_quantity + 10: + s_quantity = s_quantity - ol_quantity + else: + s_quantity = s_quantity + 91 - ol_quantity + s_order_cnt += 1 + + if ol_supply_w_id != w_id: s_remote_cnt += 1 + + # updateStock + self.stock.update(si, {"$set": {"S_QUANTITY": s_quantity, "S_YTD": s_ytd, "S_ORDER_CNT": s_order_cnt, "S_REMOTE_CNT": s_remote_cnt}}) + + if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: + brand_generic = 'B' + else: + brand_generic = 'G' + ## Transaction profile states to use "ol_quantity * i_price" + ol_amount = ol_quantity * i_price + total += ol_amount + + ol = {"OL_O_ID": d_next_o_id, "OL_NUMBER": ol_number, "OL_I_ID": ol_i_id, "OL_SUPPLY_W_ID": ol_supply_w_id, "OL_DELIVERY_D": o_entry_d, "OL_QUANTITY": ol_quantity, "OL_AMOUNT": ol_amount, "OL_DIST_INFO": s_dist_xx} + + if self.denormalize: + # createOrderLine + o[constants.TABLENAME_ORDER_LINE].append(ol) + else: + ol["OL_D_ID"] = d_id + ol["OL_W_ID"] = w_id + + # createOrderLine + self.order_line.insert(ol) + ## IF + + ## Add the info to be returned + item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) + ## FOR + + ## Adjust the total for the discount + #print "c_discount:", c_discount, type(c_discount) + #print "w_tax:", w_tax, type(w_tax) + #print "d_tax:", d_tax, type(d_tax) + total *= (1 - c_discount) * (1 + w_tax + d_tax) + + # createOrder + self.customer.update({"_id": c["_id"]}, {"$push": {"ORDERS": o}}) + + ## Pack up values the client is missing (see TPC-C 2.4.3.5) + misc = [ (w_tax, d_tax, d_next_o_id, total) ] + + return [ c, misc, item_data ] + + ## ---------------------------------------------- + ## doOrderStatus + ## ---------------------------------------------- + def doOrderStatus(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + + assert w_id, pformat(params) + assert d_id, pformat(params) + + search_fields = {"C_W_ID": w_id, "C_D_ID": d_id} + return_fields = {"C_ID": 1, "C_FIRST": 1, "C_MIDDLE": 1, "C_LAST": 1, "C_BALANCE": 1} + if self.denormalize: + for f in ['O_ID', 'O_CARRIER_ID', 'O_ENTRY_D']: + return_fields["%s.%s" % (constants.TABLENAME_ORDERS, f)] = 1 + for f in ['OL_SUPPLY_W_ID', 'OL_I_ID', 'OL_QUANTITY']: + return_fields["%s.%s.%s" % (constants.TABLENAME_ORDERS, constants.TABLENAME_ORDER_LINE, f)] = 1 + ## IF + + if c_id != None: + # getCustomerByCustomerId + search_fields["C_ID"] = c_id + c = self.customer.find_one(search_fields, return_fields) + assert c + + else: + # getCustomersByLastName + # Get the midpoint customer's id + search_fields['C_LAST'] = c_last + + all_customers = self.customer.find(search_fields, return_fields) + namecnt = all_customers.count() + assert namecnt > 0 + index = (namecnt-1)/2 + c = all_customers[index] + c_id = c["C_ID"] + assert len(c) > 0 + assert c_id != None + + orderLines = [ ] + order = None + + if self.denormalize: + # getLastOrder + if constants.TABLENAME_ORDERS in c: + order = c[constants.TABLENAME_ORDERS][-1] + # getOrderLines + orderLines = order[constants.TABLENAME_ORDER_LINE] + else: + # getLastOrder + order = self.orders.find({"O_W_ID": w_id, "O_D_ID": d_id, "O_C_ID": c_id}, {"O_ID": 1, "O_CARRIER_ID": 1, "O_ENTRY_D": 1}).sort("O_ID", direction=pymongo.DESCENDING).limit(1)[0] + o_id = order["O_ID"] + + if order: + # getOrderLines + orderLines = self.order_line.find({"OL_W_ID": w_id, "OL_D_ID": d_id, "OL_O_ID": o_id}, {"OL_SUPPLY_W_ID": 1, "OL_I_ID": 1, "OL_QUANTITY": 1, "OL_AMOUNT": 1, "OL_DELIVERY_D": 1}) + ## IF + + + return [ c, order, orderLines ] + + ## ---------------------------------------------- + ## doPayment + ## ---------------------------------------------- + def doPayment(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + h_amount = params["h_amount"] + c_w_id = params["c_w_id"] + c_d_id = params["c_d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + h_date = params["h_date"] + + search_fields = {"C_W_ID": w_id, "C_D_ID": d_id} + return_fields = {"C_BALANCE": 0, "C_YTD_PAYMENT": 0, "C_PAYMENT_CNT": 0} + + if c_id != None: + # getCustomerByCustomerId + search_fields["C_ID"] = c_id + c = self.customer.find_one(search_fields, return_fields) + assert c + + else: + # getCustomersByLastName + # Get the midpoint customer's id + search_fields['C_LAST'] = c_last + all_customers = self.customer.find(search_fields, return_fields) + namecnt = all_customers.count() + assert namecnt > 0 + index = (namecnt-1)/2 + c = all_customers[index] + c_id = c["C_ID"] + assert len(c) > 0 + assert c_id != None + + if c_id != None: + # getCustomerByCustomerId + c = self.customer.find_one({"C_W_ID": w_id, "C_D_ID": d_id, "C_ID": c_id}) + else: + # getCustomersByLastName + # Get the midpoint customer's id + all_customers = self.customer.find({"C_W_ID": w_id, "C_D_ID": d_id, "C_LAST": c_last}) + namecnt = all_customers.count() + assert namecnt > 0 + index = (namecnt-1)/2 + c = all_customers[index] + c_id = c["C_ID"] + assert len(c) > 0 + assert c_id != None + c_data = c["C_DATA"] + + # getWarehouse + w = self.warehouse.find_one({"W_ID": w_id}, {"W_NAME": 1, "W_STREET_1": 1, "W_STREET_2": 1, "W_CITY": 1, "W_STATE": 1, "W_ZIP": 1}) + assert w + + # updateWarehouseBalance + self.warehouse.update({"_id": w["_id"]}, {"$inc": {"W_YTD": h_amount}}) + + # getDistrict + d = self.district.find_one({"D_W_ID": w_id, "D_ID": d_id}, {"D_NAME": 1, "D_STREET_1": 1, "D_STREET_2": 1, "D_CITY": 1, "D_STATE": 1, "D_ZIP": 1}) + assert d + + # updateDistrictBalance + self.district.update({"_id": d["_id"]}, {"$inc": {"D_YTD": h_amount}}) + + # Build CUSTOMER update command + customer_update = {"$inc": {"C_BALANCE": h_amount*-1, "C_YTD_PAYMENT": h_amount, "C_PAYMENT_CNT": 1}} + + # Customer Credit Information + if c["C_CREDIT"] == constants.BAD_CREDIT: + newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) + c_data = (newData + "|" + c_data) + if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] + customer_update["$set"] = {"C_DATA": c_data} + ## IF + + # Concatenate w_name, four spaces, d_name + h_data = "%s %s" % (w["W_NAME"], d["D_NAME"]) + h = {"H_D_ID": d_id, "H_W_ID": w_id, "H_DATE": h_date, "H_AMOUNT": h_amount, "H_DATA": h_data} + if self.denormalize: + # insertHistory + updateCustomer + customer_update["$push"] = {constants.TABLENAME_HISTORY: h} + self.customer.update({"_id": c["_id"]}, customer_update) + else: + # updateCustomer + self.customer.update({"_id": c["_id"]}, customer_update) + + # insertHistory + self.history.insert(h) + + # TPC-C 2.5.3.3: Must display the following fields: + # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, + # D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, + # C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, + # C_DISCOUNT, C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = "BC"), + # H_AMOUNT, and H_DATE. + + # Hand back all the warehouse, district, and customer data + return [ w, d, c ] + + ## ---------------------------------------------- + ## doStockLevel + ## ---------------------------------------------- + def doStockLevel(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + threshold = params["threshold"] + + # getOId + d = self.district.find_one({"D_W_ID": w_id, "D_ID": d_id}, {"D_NEXT_O_ID": 1}) + assert d + o_id = d["D_NEXT_O_ID"] + + # getStockCount + # Outer Table: ORDER_LINE + # Inner Table: STOCK + if self.denormalize: + c = self.customer.find({"C_W_ID": w_id, "C_D_ID": d_id, "ORDERS.O_ID": {"$lt": o_id, "$gte": o_id-20}}, {"ORDERS.ORDER_LINE.OL_I_ID": 1}) + assert c + orderLines = [ ] + for ol in c: + assert "ORDER_LINE" in ol["ORDERS"][0] + orderLines.extend(ol["ORDERS"][0]["ORDER_LINE"]) + else: + orderLines = self.order_line.find({"OL_W_ID": w_id, "OL_D_ID": d_id, "OL_O_ID": {"$lt": o_id, "$gte": o_id-20}}, {"OL_I_ID": 1}) + + assert orderLines + ol_ids = set() + for ol in orderLines: + ol_ids.add(ol["OL_I_ID"]) + ## FOR + result = self.stock.find({"S_W_ID": w_id, "S_I_ID": {"$in": list(ol_ids)}, "S_QUANTITY": {"$lt": threshold}}).count() + + return int(result) + +## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/redisdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/redisdriver.py new file mode 100644 index 00000000..48108fdd --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/redisdriver.py @@ -0,0 +1,1663 @@ +import os, redis, time, sys +from datetime import datetime +from pprint import pprint,pformat +from abstractdriver import * + +#---------------------------------------------------------------------------- +# Redis TPC-C Driver +# +# @author Christopher Keith +# @author James Tavares +#---------------------------------------------------------------------------- +class RedisDriver(AbstractDriver): + + KEY_SEPARATOR = ':' + + TABLES = { + 'WAREHOUSE' : { + 'columns' : + [ + 'W_ID', + 'W_NAME', + 'W_STREET_1', + 'W_STREET_2', + 'W_CITY', + 'W_STATE', + 'W_ZIP', + 'W_TAX', + 'W_YTD', + ], + 'map' : + { + 'W_ID' : 0, + 'W_NAME' : 1, + 'W_STREET_1' : 2, + 'W_STREET_2' : 3, + 'W_CITY' : 4, + 'W_STATE' : 5, + 'W_ZIP' : 6, + 'W_TAX' : 7, + 'W_YTD' : 8, + }, + 'primary_key' : ['W_ID'], + 'indexes' : [ ], + }, + 'DISTRICT' : { + 'columns' : + [ + 'D_ID', + 'D_W_ID', + 'D_NAME', + 'D_STREET_1', + 'D_STREET_2', + 'D_CITY', + 'D_STATE', + 'D_ZIP', + 'D_TAX', + 'D_YTD', + 'D_NEXT_O_ID', + ], + 'map' : + { + 'D_ID' : 0, + 'D_W_ID' : 1, + 'D_NAME' : 2, + 'D_STREET_1' : 3, + 'D_STREET_2' : 4, + 'D_CITY' : 5, + 'D_STATE' : 6, + 'D_ZIP' : 7, + 'D_TAX' : 8, + 'D_YTD' : 9, + 'D_NEXT_O_ID' : 10, + }, + 'primary_key' : ['D_W_ID', 'D_ID'], + 'indexes' : [ ], + }, + 'ITEM' : { + 'columns' : + [ + 'I_ID', + 'I_IM_ID', + 'I_NAME', + 'I_PRICE', + 'I_DATA', + ], + 'map' : + { + 'I_ID' : 0, + 'I_IM_ID' : 1, + 'I_NAME' : 2, + 'I_PRICE' : 3, + 'I_DATA' : 4, + }, + 'primary_key' : ['I_ID'], + 'indexes' : [ ] + }, + 'CUSTOMER' : { + 'columns' : + [ + 'C_ID', + 'C_D_ID', + 'C_W_ID', + 'C_FIRST', + 'C_MIDDLE', + 'C_LAST', + 'C_STREET_1', + 'C_STREET_2', + 'C_CITY', + 'C_ZIP', + 'C_PHONE', + 'C_SINCE', + 'C_CREDIT', + 'C_CREDIT_LIM', + 'C_DISCOUNT', + 'C_BALANCE', + 'C_YTD_PAYMENT', + 'C_PAYMENT_CNT', + 'C_DELIVERY_CNT', + 'C_DATA', + ], + 'map' : + { + 'C_ID' : 0, + 'C_D_ID' : 1, + 'C_W_ID' : 2, + 'C_FIRST' : 3, + 'C_MIDDLE' : 4, + 'C_LAST' : 5, + 'C_STREET_1' : 6, + 'C_STREET_2' : 7, + 'C_CITY' : 8, + 'C_ZIP' : 9, + 'C_PHONE' : 10, + 'C_SINCE' : 11, + 'C_CREDIT' : 12, + 'C_CREDIT_LIM' : 13, + 'C_DISCOUNT' : 14, + 'C_BALANCE' : 15, + 'C_YTD_PAYMENT' : 16, + 'C_PAYMENT_CNT' : 17, + 'C_DELIVERY_CNT' : 18, + 'C_DATA' : 19, + }, + 'primary_key' : ['C_W_ID', 'C_D_ID', 'C_ID'], + 'indexes' : [ ], + }, + 'HISTORY' : { + 'columns' : + [ + 'H_C_ID', + 'H_C_D_ID', + 'H_C_W_ID', + 'H_D_ID', + 'H_W_ID', + 'H_DATE', + 'H_AMOUNT', + 'H_DATA', + ], + 'map' : + { + 'H_C_ID' : 0, + 'H_C_D_ID' : 1, + 'H_C_W_ID' : 2, + 'H_D_ID' : 3, + 'H_W_ID' : 4, + 'H_DATE' : 5, + 'H_AMOUNT' : 6, + 'H_DATA' : 7, + }, + 'primary_key' : [ ], + 'indexes' : [ ], + }, + 'STOCK' : { + 'columns' : + [ + 'S_I_ID', + 'S_W_ID', + 'S_QUANTITY', + 'S_DIST_01', + 'S_DIST_02', + 'S_DIST_03', + 'S_DIST_04', + 'S_DIST_05', + 'S_DIST_06', + 'S_DIST_07', + 'S_DIST_08', + 'S_DIST_09', + 'S_DIST_10', + 'S_YTD', + 'S_ORDER_CNT', + 'S_REMOTE_CNT', + 'S_DATA', + ], + 'map' : + { + 'S_I_ID' : 0, + 'S_W_ID' : 1, + 'S_QUANTITY' : 2, + 'S_DIST_01' : 3, + 'S_DIST_02' : 4, + 'S_DIST_03' : 5, + 'S_DIST_04' : 6, + 'S_DIST_05' : 7, + 'S_DIST_06' : 8, + 'S_DIST_07' : 9, + 'S_DIST_08' : 10, + 'S_DIST_09' : 11, + 'S_DIST_10' : 12, + 'S_YTD' : 13, + 'S_ORDER_CNT' : 14, + 'S_REMOTE_CNT' : 15, + 'S_DATA' : 16, + }, + 'primary_key' : ['S_W_ID', 'S_I_ID'], + 'indexes' : [ ], + }, + 'ORDERS' : { + 'columns' : + [ + 'O_ID', + 'O_D_ID', + 'O_W_ID', + 'O_C_ID', + 'O_ENTRY_D', + 'O_CARRIER_ID', + 'O_OL_CNT', + 'O_ALL_LOCAL', + ], + 'map' : + { + 'O_ID' : 0, + 'O_D_ID' : 1, + 'O_W_ID' : 2, + 'O_C_ID' : 3, + 'O_ENTRY_D' : 4, + 'O_CARRIER_ID' : 5, + 'O_OL_CNT' : 6, + 'O_ALL_LOCAL' : 7, + }, + 'primary_key' : ['O_ID', 'O_C_ID', 'O_D_ID', 'O_W_ID'], + 'indexes' : [ ], + }, + 'NEW_ORDER' : { + 'columns' : + [ + 'NO_O_ID', + 'NO_D_ID', + 'NO_W_ID', + ], + 'map' : + { + 'NO_O_ID' : 0, + 'NO_D_ID' : 1, + 'NO_W_ID' : 2, + }, + 'primary_key' : ['NO_D_ID', 'NO_W_ID', 'NO_O_ID'], + 'indexes' : [ ], + }, + 'ORDER_LINE' : { + 'columns' : + [ + 'OL_O_ID', + 'OL_D_ID', + 'OL_W_ID', + 'OL_NUMBER', + 'OL_I_ID', + 'OL_SUPPLY_W_ID', + 'OL_DELIVERY_D', + 'OL_QUANTITY', + 'OL_AMOUNT', + 'OL_DIST_INFO', + ], + 'map' : + { + 'OL_O_ID' : 0, + 'OL_D_ID' : 1, + 'OL_W_ID' : 2, + 'OL_NUMBER' : 3, + 'OL_I_ID' : 4, + 'OL_SUPPLY_W_ID' : 5, + 'OL_DELIVERY_D' : 6, + 'OL_QUANTITY' : 7, + 'OL_AMOUNT' : 8, + 'OL_DIST_INFO' : 9, + }, + 'primary_key' : ['OL_W_ID', 'OL_D_ID', 'OL_O_ID', 'OL_NUMBER'], + 'indexes' : [ ], + } + } + + DEFAULT_CONFIG = { + 'databases' : ("List of Redis Hosts", "127.0.0.1:6379"), + 'host-info' : ("Show information about hosts", 'Verbose'), + 'debug-load' : ("Show Loading Information", 'None'), + 'debug-delivery' : ("Show Delivery Performance", 'None'), + 'debug-new-order' : ("Show New Order Performance", 'None'), + 'debug-order-status' : ("Show Order Status Performance", 'None'), + 'debug-payment' : ("Show Payment Performance", 'None'), + 'debug-stock-level' : ("Show Stock Level Performance", 'None'), + } + + #------------------------------------------------------------------------ + # Class constructor + # + # @param string ddl (Data Definintion Language) + #------------------------------------------------------------------------ + def __init__(self, ddl) : + super(RedisDriver, self).__init__("redis", ddl) + self.databases = [ ] + self.t0 = 0 + self.db_count = 0 + self.metadata = None + self.r_pipes = [ ] + self.r_sizes = [ ] + self.w_pipes = [ ] + self.w_sizes = [ ] + self.debug = { + 'load' : 'None', + 'delivery' : 'None', + 'new-order' : 'None', + 'order-status' : 'None', + 'payment' : 'None', + 'stock-level' : 'None', + } + self.hosts = [ ] + # End __init__() + + #------------------------------------------------------------------------ + # Execute TPC-C Delivery Transaction + # + # @param dictionary params (transaction parameters) + # { + # "w_id" : value, + # "o_carrier_id" : value, + # "ol_delivery_d" : value, + # } + #------------------------------------------------------------------------ + def doDelivery(self, params) : + if self.debug['delivery'] != 'None' : + print 'TXN DELIVERY STARTING ------------------' + tt = time.time() + if self.debug['delivery'] == 'Verbose' : + t0 = tt + + # Initialize input parameters + w_id = params["w_id"] + o_carrier_id = params["o_carrier_id"] + ol_delivery_d = params["ol_delivery_d"] + + # Setup Redis pipelining + node = self.shard(w_id) + rdr = self.r_pipes[node] + wtr = self.w_pipes[node] + + # Initialize result set + result = [ ] + + #------------------------- + # Initialize Data Holders + #------------------------- + order_key = [ ] + ol_total = [ ] + customer_key = [ ] + ol_counts = [ ] + no_o_id = [ ] + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : + order_key.append(None) + ol_total.append(0) + customer_key.append(None) + ol_counts.append(0) + + #--------------------- + # Get New Order Query + #--------------------- + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : + cursor = d_id - 1 + # Get set of possible new order ids + index_key = self.safeKey([d_id, w_id]) + rdr.srandmember('NEW_ORDER.INDEXES.GETNEWORDER.' + index_key) + id_set = rdr.execute() + + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : + cursor = d_id - 1 + if id_set[cursor] == None : + rdr.get('NULL_VALUE') + else : + rdr.hget('NEW_ORDER.' + str(id_set[cursor]), 'NO_O_ID') + no_o_id = rdr.execute() + + if self.debug['delivery'] == 'Verbose' : + print 'New Order Query: ', time.time() - t0 + t0 = time.time() + + #----------------------- + # Get Customer ID Query + #----------------------- + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : + cursor = d_id - 1 + if no_o_id[cursor] == None : + order_key.insert(cursor, 'NO_KEY') + else : + order_key.insert( + cursor, + self.safeKey([w_id, d_id, no_o_id[0]]) + ) + rdr.hget('ORDERS.' + order_key[cursor], 'O_C_ID') + c_id = rdr.execute() + + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : + cursor = d_id - 1 + if no_o_id[cursor] == None or c_id[cursor] == None : + si_key = 'NO_KEY' + else : + si_key = self.safeKey([no_o_id[cursor], w_id, d_id]) + rdr.smembers('ORDER_LINE.INDEXES.SUMOLAMOUNT.' + si_key) + ol_ids = rdr.execute() + + if self.debug['delivery'] == 'Verbose' : + print 'Get Customer ID Query:', time.time() - t0 + t0 = time.time() + + #----------------------------- + # Sum Order Line Amount Query + #----------------------------- + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : + cursor = d_id - 1 + if no_o_id[cursor] == None or c_id[cursor] == None : + rdr.get('NULL_VALUE') + else : + for i in ol_ids[cursor] : + rdr.hget('ORDER_LINE.' + str(i), 'OL_AMOUNT') + ol_counts[cursor] += 1 + + pipe_results = rdr.execute() + index = 0 + counter = 0 + + for ol_amount in pipe_results : + counter += 1 + if counter > ol_counts[index] : + index += 1 + counter = 0 + elif ol_amount != None : + ol_total[index] += float(ol_amount) + + if self.debug['delivery'] == 'Verbose' : + print 'Sum Order Line Query:', time.time() - t0 + t0 = time.time() + + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : + cursor = d_id - 1 + if no_o_id[cursor] == None or c_id[cursor] == None : + ## No orders for this district: skip it. + ## Note: This must be reported if > 1% + continue + + #------------------------ + # Delete New Order Query + #------------------------ + no_key = self.safeKey([d_id, w_id, no_o_id[cursor]]) + no_si_key = self.safeKey([d_id, w_id]) + wtr.delete('NEW_ORDER.' + no_key) + wtr.srem('NEW_ORDER.IDS', no_key) + wtr.srem('NEW_ORDER.INDEXES.GETNEWORDER.' + no_si_key, no_key) + + if self.debug['delivery'] == 'Verbose' : + print 'Delete New Order Query:', time.time() - t0 + t0 = time.time() + + #--------------------- + # Update Orders Query + #--------------------- + wtr.hset( + 'ORDERS.' + order_key[cursor], + 'W_CARRIER_ID', + o_carrier_id + ) + + if self.debug['delivery'] == 'Verbose' : + print 'Update Orders Query:', time.time() - t0 + t0 = time.time() + + #------------------------- + # Update Order Line Query + #------------------------- + for i in ol_ids[cursor] : + wtr.hset( + 'ORDER_LINE.' + str(i), + 'OL_DELIVERY_D', + ol_delivery_d + ) + + if self.debug['delivery'] == 'Verbose' : + print 'Update Order Line Query:', time.time() - t0 + t0 = time.time() + + #----------------------- + # Update Customer Query + #----------------------- + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : + cursor = d_id - 1 + if no_o_id[cursor] == None or c_id[cursor] == None : + rdr.get('NULL_VALUE') + customer_key.insert(cursor, 'NO_KEY') + else : + customer_key.insert( + cursor, + self.safeKey([w_id, d_id, c_id[cursor]]) + ) + rdr.hget('CUSTOMER.' + customer_key[cursor], 'C_BALANCE') + old_balance = rdr.execute() + + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : + cursor = d_id - 1 + if no_o_id[cursor] == None or c_id[cursor] == None : + continue + else : + new_balance = float(old_balance[cursor]) + float(ol_total[cursor]) + wtr.hset( + 'CUSTOMER.' + customer_key[cursor], + 'C_BALANCE', + new_balance + ) + result.append((d_id, no_o_id[cursor])) + wtr.execute() + + if self.debug['delivery'] == 'Verbose' : + print 'Update Customer Query:', time.time() - t0 + if self.debug['delivery'] != 'None' : + print 'TXN DELIVERY:', time.time() - tt + + return result + # End doDelivery() + + #------------------------------------------------------------------------ + # Execute TPC-C New Order Transaction + # + # @param dictionary params (transaction parameters) + # { + # 'w_id' : value, + # 'd_id' : value, + # 'c_id' : value, + # 'o_entry_d' : value, + # 'i_ids' : value, + # 'i_w_ids' : value, + # 'i_qtys' : value, + # } + #------------------------------------------------------------------------ + def doNewOrder(self, params) : + if self.debug['new-order'] != 'None' : + print 'TXN NEW ORDER STARTING -----------------' + tt = time.time() + if self.debug['new-order'] == 'Verbose' : + t0 = tt + + # Initialize transaction parameters + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + o_entry_d = params["o_entry_d"] + i_ids = params["i_ids"] + i_w_ids = params["i_w_ids"] + i_qtys = params["i_qtys"] + + # Setup Redis pipelining + node = self.shard(w_id) + rdr = self.r_pipes[node] + wtr = self.w_pipes[node] + + # Validate transaction parameters + assert len(i_ids) > 0 + assert len(i_ids) == len(i_w_ids) + assert len(i_ids) == len(i_qtys) + + # Check if all items are local + all_local = True + items = [ ] + pipe_results = [ ] + for i in range(len(i_ids)): + all_local = all_local and i_w_ids[i] == w_id + rdr.hgetall('ITEM.' + str(i_ids[i])) + pipe_results = rdr.execute() + + for pr in pipe_results : + if len(pr) > 0 : + if pr['I_PRICE'] == None and pr['I_NAME'] == None and pr['I_DATA'] == None : + result = [ ] + items.append([ + pr['I_PRICE'], + pr['I_NAME'], + pr['I_DATA'], + ]) + else : + items.append([]) + + assert len(items) == len(i_ids) + + ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. + ## Note that this will happen with 1% of transactions on purpose. + for item in items : + if len(item) == 0 : + return + + #------------------------------------------------------------ + # Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER + #------------------------------------------------------------ + district_key = self.safeKey([w_id, d_id]) + customer_key = self.safeKey([w_id, d_id, c_id]) + + #------------------------------ + # Get Warehouse Tax Rate Query + #------------------------------ + rdr.hget('WAREHOUSE.' + str(w_id), 'W_TAX') + + if self.debug['new-order'] == 'Verbose' : + print 'Get Warehouse Tax Rate Query:', time.time() - t0 + t0 = time.time() + + #-------------------- + # Get District Query + #-------------------- + rdr.hgetall('DISTRICT.' + district_key) + + if self.debug['new-order'] == 'Verbose' : + print 'Get District Query:', time.time() - t0 + t0 = time.time() + + #-------------------- + # Get Customer Query + #-------------------- + rdr.hgetall('CUSTOMER.' + customer_key) + rdr_results = rdr.execute() + + w_tax = float(rdr_results[0]) + d_tax = float(rdr_results[1]['D_TAX']) + d_next_o_id = rdr_results[1]['D_NEXT_O_ID'] + customer_info = rdr_results[2] + c_discount = float(rdr_results[2]['C_DISCOUNT']) + + if self.debug['new-order'] == 'Verbose' : + print 'Get Customer Query:', time.time() - t0 + t0 = time.time() + + #-------------------------- + # Insert Order Information + #-------------------------- + ol_cnt = len(i_ids) + o_carrier_id = constants.NULL_CARRIER_ID + order_key = self.safeKey([w_id, d_id, d_next_o_id]) + new_order_key = self.safeKey([d_next_o_id, w_id, d_id]) + + #------------------------------- + # Increment Next Order ID Query + #------------------------------- + wtr.hincrby('DISTRICT.' + district_key, 'D_NEXT_O_ID') + + if self.debug['new-order'] == 'Verbose' : + print 'Increment Next Order ID Query:', time.time() - t0 + t0 = time.time() + + #-------------------- + # Create Order Query + #-------------------- + wtr.sadd('ORDERS.IDS', order_key) + wtr.hmset( + 'ORDERS.' + order_key, + { + 'O_ID' : d_next_o_id, + 'O_D_ID' : d_id, + 'O_W_ID' : w_id, + 'O_C_ID' : c_id, + 'O_ENTRY_D' : o_entry_d, + 'O_CARRIER_ID' : o_carrier_id, + 'O_OL_CNT' : ol_cnt, + 'O_ALL_LOCAL' : all_local + } + ) + + if self.debug['new-order'] == 'Verbose' : + print 'Create Order Query:', time.time() - t0 + t0 = time.time() + + # Add index for Order searching + si_key = self.safeKey([w_id, d_id, c_id]) + wtr.sadd('ORDERS.INDEXES.ORDERSEARCH', si_key) + + #------------------------ + # Create New Order Query + #------------------------ + wtr.sadd('NEW_ORDER.IDS', new_order_key) + wtr.hmset( + 'NEW_ORDER.' + new_order_key, + { + 'NO_O_ID' : d_next_o_id, + 'NO_D_ID' : d_id, + 'NO_W_ID' : w_id, + } + ) + + # Add index for New Order Searching + si_key = self.safeKey([d_id, w_id]) + wtr.sadd('NEW_ORDER.INDEXES.GETNEWORDER', si_key) + + if self.debug['new-order'] == 'Verbose' : + print 'Create New Order Query:', time.time() - t0 + t0 = time.time() + + #------------------------------- + # Insert Order Item Information + #------------------------------- + item_data = [ ] + total = 0 + + ol_number = [ ] + ol_quantity = [ ] + ol_supply_w_id = [ ] + ol_i_id = [ ] + i_name = [ ] + i_price = [ ] + i_data = [ ] + stock_key = [ ] + stock_info = [ ] + + for i in range(len(i_ids)) : + ol_number.append(i + 1) + ol_supply_w_id.append(i_w_ids[i]) + ol_i_id.append(i_ids[i]) + ol_quantity.append(i_qtys[i]) + + itemInfo = items[i] + i_name.append(itemInfo[1]) + i_data.append(itemInfo[2]) + i_price.append(float(itemInfo[0])) + + #----------------------------- + # Get Stock Information Query + #----------------------------- + stock_key.append(self.safeKey([ol_supply_w_id[i], ol_i_id[i]])) + rdr.hgetall('STOCK.' + stock_key[i]) + stock_info = rdr.execute() + + for index, si in enumerate(stock_info) : + if len(si) == 0 : + continue + s_quantity = float(si['S_QUANTITY']) + s_ytd = float(si['S_YTD']) + s_order_cnt = float(si['S_ORDER_CNT']) + s_remote_cnt = float(si['S_REMOTE_CNT']) + s_data = si['S_DATA'] + if len(str(d_id)) == 1 : + s_dist_xx = si['S_DIST_0' + str(d_id)] + else : + s_dist_xx = si['S_DIST_' + str(d_id)] + + if self.debug['new-order'] == 'Verbose' : + print 'Get Stock Information Query:', time.time() - t0 + t0 = time.time() + + #-------------------- + # Update Stock Query + #-------------------- + s_ytd += ol_quantity[index] + if s_quantity >= ol_quantity[index] + 10 : + s_quantity = s_quantity - ol_quantity[index] + else : + s_quantity = s_quantity + 91 - ol_quantity[index] + s_order_cnt += 1 + + if ol_supply_w_id[index] != w_id : s_remote_cnt += 1 + + wtr.hmset( + 'STOCK.' + stock_key[index], + { + 'S_QUANTITY' : s_quantity, + 'S_YTD' : s_ytd, + 'S_ORDER_CNT' : s_order_cnt, + 'S_REMOTE_CNT' : s_remote_cnt + } + ) + + if i_data[index].find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: + brand_generic = 'B' + else: + brand_generic = 'G' + + ## Transaction profile states to use "ol_quantity * i_price" + ol_amount = ol_quantity[index] * i_price[index] + total += ol_amount + + if self.debug['new-order'] == 'Verbose' : + print 'Update Stock Query:', time.time() - t0 + t0 = time.time() + + #------------------------- + # Create Order Line Query + #------------------------- + order_line_key = self.safeKey([w_id, d_id,d_next_o_id, ol_number]) + si_key = self.safeKey([d_next_o_id, d_id, w_id]) + wtr.sadd('ORDER_LINE.IDS', order_line_key) + wtr.hmset( + 'ORDER_LINE.' + order_line_key, + { + 'OL_O_ID' : d_next_o_id, + 'OL_D_ID' : d_id, + 'OL_W_ID' : w_id, + 'OL_NUMBER' : ol_number[index], + 'OL_I_ID' : ol_i_id[index], + 'OL_SUPPLY_W_ID' : ol_supply_w_id[index], + 'OL_DELIVERY_D' : o_entry_d, + 'OL_QUANTITY' : ol_quantity[index], + 'OL_AMOUNT' : ol_amount, + 'OL_DISTRICT_INFO' : s_dist_xx + } + ) + + # Create index for Order Line Searching + wtr.sadd( + 'ORDER_LINE.INDEXES.SUMOLAMOUNT.' + si_key, + order_line_key + ) + + if self.debug['new-order'] == 'Verbose' : + print 'Create Order Line Query:', time.time() - t0 + t0 = time.time() + + ## Add the info to be returned + item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) + ## End for i in range(len(i_ids)) : + + ## Commit! + wtr.execute() + + ## Adjust the total for the discount + total *= (1 - c_discount) * (1 + w_tax + d_tax) + + ## Pack up values the client is missing (see TPC-C 2.4.3.5) + misc = [ (w_tax, d_tax, d_next_o_id, total) ] + + if self.debug['new-order'] != 'None' : + print 'TXN NEW ORDER:', time.time() - tt + return [ customer_info, misc, item_data ] + + #------------------------------------------------------------------------ + # Execute TPC-C Do Order Status transaction + # + # @param dictionary params (transaction parameters) + # { + # 'w_id' : value, + # 'd_id' : value, + # 'c_id' : value, + # 'c_last' : value, + # } + #------------------------------------------------------------------------ + def doOrderStatus(self, params) : + if self.debug['order-status'] != 'None' : + print 'TXN ORDER STATUS STARTING --------------' + tt = time.time() + if self.debug['order-status'] == 'Verbose' : + t0 = tt + + # Initialize transactions parameters + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + + # Initialize Redis pipelining + node = self.shard(w_id) + rdr = self.databases[node].pipeline(False) + wtr = self.databases[node].pipeline(True) + + # Validate transaction parameters + assert w_id, pformat(params) + assert d_id, pformat(params) + + if c_id != None: + #----------------------------------- + # Get Customer By Customer ID Query + #----------------------------------- + customer_key = self.safeKey([w_id, d_id, c_id]) + rdr.hgetall('CUSTOMER.' + customer_key) + results = rdr.execute(); + customer = results[0] + + if self.debug['order-status'] == 'Verbose' : + print 'Get Customer By Customer ID Query:', time.time() - t0 + t0 = time.time() + else: + #---------------------------------- + # Get Customers By Last Name Query + #---------------------------------- + si_key = self.safeKey([w_id, d_id, c_last]) + rdr.smembers('CUSTOMER.INDEXES.NAMESEARCH.' + si_key) + results = rdr.execute(); + customer_id_set = results[0] + + customer_ids = [ ] + for customer_id in customer_id_set : + rdr.hgetall('CUSTOMER.' + str(customer_id)) + customer_ids.append(str(customer_id)) + + customers = [ ] + unsorted_customers = rdr.execute() + customers.append(unsorted_customers.pop()) + for cust in unsorted_customers : + for index in range(len(customers)) : + if cust['C_FIRST'] < customers[index]['C_FIRST'] : + customers.insert(index, cust) + continue + + assert len(customers) > 0 + + namecnt = len(customers) + index = (namecnt - 1)/2 + customer = customers[index] + customer_key = self.safeKey([ + customer['C_W_ID'], + customer['C_D_ID'], + customer['C_ID'] + ]) + c_id = customer['C_ID'] + + if self.debug['order-status'] == 'Verbose' : + print 'Get Customers By Last Name Query:', time.time() - t0 + t0 = time.time() + assert len(customer) > 0 + assert c_id != None + + #---------------------- + # Get Last Order Query + #---------------------- + search_key = self.safeKey([w_id, d_id, c_id]) + rdr.smembers('ORDERS.INDEXES.ORDERSEARCH.' + search_key) + results = rdr.execute() + order_id_set = results[0] + + if self.debug['order-status'] == 'Verbose' : + print 'Get Last Order Query:', time.time() - t0 + t0 = time.time() + + order = [ ] + if len(order_id_set) > 0 : + order_ids = sorted(list(order_id_set)) + order_key = str(order_ids[0]) + rdr.hgetall('ORDERS.' + order_key) + + result = rdr.execute() + order = [ + result[0]['O_ID'], + result[0]['O_CARRIER_ID'], + result[0]['O_ENTRY_D'], + ] + + #----------------------- + # Get Order Lines Query + #----------------------- + search_key = self.safeKey([order[0], d_id, w_id]) + rdr.smembers('ORDER_LINE.INDEXES.SUMOLAMOUNT.' + search_key) + results = rdr.execute() + line_ids = results[0] + + for line_id in line_ids : + rdr.hgetall('ORDER_LINE.' + str(line_id)) + + orderLines = [ ] + results = rdr.execute() + for r in results : + orderLines.append([ + r['OL_SUPPLY_W_ID'], + r['OL_I_ID'], + r['OL_QUANTITY'], + r['OL_AMOUNT'], + r['OL_DELIVERY_D'] + ]) + + if self.debug['order-status'] == 'Verbose' : + print 'Get Order Lines Query:', time.time() - t0 + else : + orderLines = [ ] + + if self.debug['order-status'] != 'None' : + print 'TXN ORDER STATUS:', time.time() - tt + + return [ customer, order, orderLines ] + + #------------------------------------------------------------------------ + # Execute TPC-C Do Payement Transaction + # + # @param dictionary params (transaction parameters) + # { + # 'w_id' : value, + # 'd_id' : value, + # 'h_amount' : value, + # 'c_w_id' : value, + # 'c_d_id' : value, + # 'c_id' : value, + # 'c_last' : value, + # 'h_date' : value, + # } + #------------------------------------------------------------------------ + def doPayment(self, params) : + if self.debug['payment'] != 'None' : + print 'TXN PAYMENT STARTING -------------------' + tt = time.time() + if self.debug['payment'] == 'Verbose' : + t0 = tt + + # Initialize transaction properties + w_id = params["w_id"] + d_id = params["d_id"] + h_amount = params["h_amount"] + c_w_id = params["c_w_id"] + c_d_id = params["c_d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + h_date = params["h_date"] + + # Initialize Redis pipeline + node = self.shard(w_id) + rdr = self.r_pipes[node] + wtr = self.w_pipes[node] + + t0 = time.time() + if c_id != None: + #-------------------------- + # Get Customer By ID Query + #-------------------------- + customer_key = self.safeKey([w_id, d_id, c_id]) + rdr.hgetall('CUSTOMER.' + customer_key) + results = rdr.execute() + customer = results[0] + + if self.debug['payment'] == 'Verbose' : + print 'Get Customer By ID Query:', time.time() - t0 + t0 = time.time() + else: + #---------------------------------- + # Get Customers By Last Name Query + #---------------------------------- + si_key = self.safeKey([w_id, d_id, c_last]) + rdr.smembers('CUSTOMER.INDEXES.NAMESEARCH.' + si_key) + results = rdr.execute() + customer_id_set = results[0] + + customer_ids = [ ] + for customer_id in customer_id_set : + rdr.hgetall('CUSTOMER.' + str(customer_id)) + customer_ids.append(str(customer_id)) + + customers = [ ] + unsorted_customers = rdr.execute() + customers.append(unsorted_customers.pop()) + for cust in unsorted_customers : + for index in range(len(customers)) : + if cust['C_FIRST'] < customers[index]['C_FIRST'] : + customers.insert(index, cust) + continue + + assert len(customers) > 0 + + namecnt = len(customers) + index = (namecnt - 1)/2 + customer = customers[index] + customer_key = self.safeKey([ + customer['C_W_ID'], + customer['C_D_ID'], + customer['C_ID'] + ]) + c_id = customer['C_ID'] + + if self.debug['payment'] == 'Verbose' : + print 'Get Customers By Last Name Query:', time.time() - t0 + t0 = time.time() + + assert len(customer) > 0 + assert c_id != None + + c_balance = float(customer['C_BALANCE']) - h_amount + c_ytd_payment = float(customer['C_YTD_PAYMENT']) + h_amount + c_payment_cnt = float(customer['C_PAYMENT_CNT']) + 1 + c_data = customer['C_DATA'] + + #--------------------- + # Get Warehouse Query + #--------------------- + rdr.hgetall('WAREHOUSE.' + str(w_id)) + + if self.debug['payment'] == 'Verbose' : + print 'Get Warehouse Query:', time.time() - t0 + t0 = time.time() + + #-------------------- + # Get District Query + #-------------------- + district_key = self.safeKey([w_id, d_id]) + rdr.hgetall('DISTRICT.' + district_key) + warehouse, district = rdr.execute() + + if self.debug['payment'] == 'Verbose' : + print 'Get District Query:', time.time() - t0 + t0 = time.time() + + #-------------------------------- + # Update Warehouse Balance Query + #-------------------------------- + wtr.set( + 'WAREHOUSE.' + str(w_id) + '.W_YTD', + float(warehouse['W_YTD']) + h_amount + ) + + if self.debug['payment'] == 'Verbose' : + print 'Update Warehouse Query:', time.time() - t0 + t0 = time.time() + + #------------------------------- + # Update District Balance Query + #------------------------------- + wtr.set( + 'DISTRICT.' + district_key + '.D_YTD', + float(district['D_YTD']) + h_amount + ) + + if self.debug['payment'] == 'Verbose' : + print 'Update District Balance Query:', time.time() - t0 + t0 = time.time() + + if customer['C_CREDIT'] == constants.BAD_CREDIT: + #---------------------------------- + # Update Bad Credit Customer Query + #---------------------------------- + newData = " ".join( + map( + str, + [c_id, c_d_id, c_w_id, d_id, w_id, h_amount] + ) + ) + + c_data = (newData + "|" + c_data) + if len(c_data) > constants.MAX_C_DATA : + c_data = c_data[:constants.MAX_C_DATA] + + wtr.hmset( + 'CUSTOMER.' + customer_key, + { + 'C_BALANCE' : c_balance, + 'C_YTD_PAYMENT' : c_ytd_payment, + 'C_PAYMENT_CNT' : c_payment_cnt, + 'C_DATA' : c_data, + } + ) + + if self.debug['payment'] == 'Verbose' : + print 'Update Bad Credit Customer Query:', time.time() - t0 + t0 = time.time() + else: + #----------------------------------- + # Update Good Credit Customer Query + #----------------------------------- + wtr.hmset( + 'CUSTOMER.' + customer_key, + { + 'C_BALANCE' : c_balance, + 'C_YTD_PAYMENT' : c_ytd_payment, + 'C_PAYMENT_CNT' : c_payment_cnt, + 'C_DATA' : '', + } + ) + if self.debug['payment'] == 'Verbose' : + print 'Update Good Credit Customer Query:', time.time() - t0 + t0 = time.time() + + wtr.execute() + + # Concatenate w_name, four spaces, d_name + h_data = "%s %s" % (warehouse['W_NAME'], district['D_NAME']) + + #---------------------- + # Insert History Query + #---------------------- + next_score = self.metadata.get('HISTORY.next_score') + self.metadata.incr('HISTORY.next_score') + + history_key = self.safeKey(next_score) + wtr.hmset( + 'HISTORY.' + history_key, + { + 'H_C_ID' : c_id, + 'H_C_D_ID' : c_d_id, + 'H_C_W_ID' : c_w_id, + 'H_D_ID' : d_id, + 'H_W_ID' : w_id, + 'H_DATE' : h_date, + 'H_AMOUNT' : h_amount, + 'H_DATA' : h_data, + } + ) + + if self.debug['payment'] == 'Verbose' : + print 'Insert History Query:', time.time() - t0 + t0 = time.time() + + wtr.execute() + + + # TPC-C 2.5.3.3: Must display the following fields: + # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, + # W_STATE, W_ZIP, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, + # C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, + # C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, + # C_BALANCE, the first 200 characters of C_DATA + # (only if C_CREDIT = "BC"), H_AMOUNT, and H_DATE. + + # Hand back all the warehouse, district, and customer data + + if self.debug['payment'] != 'None' : + print 'TXN PAYMENT:', time.time() - tt + + return [ warehouse, district, customer ] + + #------------------------------------------------------------------------ + # Execute TPC-C Stock Level Transaction + # + # @param dictionary params (transaction parameters) + # { + # 'w_id' : value, + # 'd_id' : value, + # 'threshold : value, + # } + #------------------------------------------------------------------------ + def doStockLevel(self, params) : + if self.debug['order-status'] != 'None' : + print 'TXN STOCK LEVEL STARTING ---------------' + tt = time.time() + if self.debug['stock-level'] == 'Verbose' : + t0 = tt + + # Initialize transaction parameters + w_id = params["w_id"] + d_id = params["d_id"] + threshold = params["threshold"] + + # Setup Redis pipelining + node = self.shard(w_id) + rdr = self.r_pipes[node] + wtr = self.w_pipes[node] + + #-------------------- + # Get Order ID Query + #-------------------- + district_key = self.safeKey([w_id, d_id]) + rdr.hget('DISTRICT.' + district_key, 'D_NEXT_O_ID') + results = rdr.execute() + o_id = results[0] + + if self.debug['stock-level'] == 'Verbose' : + print 'Get Order ID Query:', time.time() - t0 + t0 = time.time() + + #----------------------- + # Get Stock Count Query + #----------------------- + stock_counts = {} + si_key = self.safeKey([o_id, d_id, w_id]) + rdr.smembers('ORDER_LINE.INDEXES.SUMOLAMOUNT.' + si_key) + results = rdr.execute() + line_ids = results[0] + + for line_id in line_ids : + rdr.hgetall('ORDER_LINE.' + str(line_id)) + order_lines = rdr.execute() + + for line['OL_I_ID'] in order_lines : + stock_key = self.safeKey([w_id, line]) + rdr.hget('STOCK.' + stock_key, 'S_QUANTITY') + stocks = rdr.execute() + + for index in range(len(order_lines)) : + if int(order_lines[index]['OL_I_ID']) < int(o_id) and int(order_lines[index]['OL_I_ID']) > int(o_id) - 20 and float(stocks[index]) < threshold : + stock_counts[order_lines[index]['OL_I_ID']] = order_lines[index]['OL_I_ID'] + + if self.debug['stock-level'] == 'Verbose' : + print 'Get Stock Count Query:', time.time() - t0 + if self.debug['stock-level'] != 'None' : + print 'TXN STOCK LEVEL:', time.time() - tt + + return len(stock_counts) + + #------------------------------------------------------------------------ + # Load the specified configuration for Redis TPC-C run + # + # @param dictionary config (configuration options) + #------------------------------------------------------------------------ + def loadConfig(self, config) : + for key in RedisDriver.DEFAULT_CONFIG.keys() : + assert key in config, "Missing parameter '%s' in the %s configuration" % (key, self.name) + + hosts = config["databases"].split() + first = True + c_num = 0 + for host in hosts : + db, port_str = host.split(':') + port = int(port_str) + print 'Connectiong to host %s on port %s' % (db, port) + self.databases.append(redis.Redis(host=db, port=port, db=0)) + print str(self.databases[c_num].ping()) + self.r_pipes.append(self.databases[c_num].pipeline(False)) + self.r_sizes.append(0) + self.w_pipes.append(self.databases[c_num].pipeline(True)) + self.w_sizes.append(0) + if (first) : + first = False + self.metadata = redis.Redis(host=db, port=port, db=0) + c_num += 1 + self.db_count += 1 + + # Reset Databases if required + if config['reset'] : + for db in self.databases : + db.flushall() + + # Process Debugging Levels + self.debug['load'] = config['debug-load'] + self.debug['delivery'] = config['debug-delivery'] + self.debug['new-order'] = config['debug-new-order'] + self.debug['order-status'] = config['debug-order-status'] + self.debug['payment'] = config['debug-payment'] + self.debug['stock-level'] = config['debug-stock-level'] + + if config['host-info'] != 'None' : + print 'TPC-C Benchmark Running on Redis with %s nodes' % (len(hosts)) + if config['host-info'] == 'Verbose' : + for host in hosts : + db, port = host.split(':') + print 'Host: %s | Port: %s' % (db, port) + # End loadConfig() + + #------------------------------------------------------------------------ + # Post-processing function for data loading + #------------------------------------------------------------------------ + def loadFinish(self) : + # Check to see if pipelines need to be flushed + for index in range(len(self.w_pipes)) : + if self.w_sizes[index] > 0 : + if self.debug['load'] != 'None' : + print index, + self.w_pipes[index].execute() + self.w_sizes[index] = 0 + + elapsed = time.time() - self.t0 + print '' + print 'Loading Complete: ' + str(elapsed) + ' elapsed' + + # Store Metadata + for table, next in self.next_scores.items() : + self.metadata.set(table + '.next_score', next) + # End loadFinish() + + #------------------------------------------------------------------------ + # Pre-pocessing function for data loading + #------------------------------------------------------------------------ + def loadStart(self) : + if self.debug['load'] != 'None': + print 'Starting data load' + self.t0 = time.time() + self.next_scores = { + 'WAREHOUSE' : 1, + 'DISTRICT' : 1, + 'ITEM' : 1, + 'CUSTOMER' : 1, + 'HISTORY' : 1, + 'STOCK' : 1, + 'ORDERS' : 1, + 'NEW_ORDER' : 1, + 'ORDER_LINE' : 1, + } + # End loadStart() + + #------------------------------------------------------------------------ + # Load tuples into a table for TPC-C benchmarking + # + # @param string table name + # @param list of tuples corresponding to table schema + #------------------------------------------------------------------------ + def loadTuples(self, tableName, tuples) : + + # Instantiate Column-mapping + column_map = self.TABLES[tableName]['map'] + if self.debug['load'] == 'Verbose' : + print tableName, + + for record in tuples : + # Determine at which node to store this data + node = 'ALL' + if tableName == 'WAREHOUSE' : + node = self.shard(record[column_map['W_ID']]) + key = self.safeKey([record[0]]) + self.w_pipes[node].sadd('WAREHOUSE.IDS', key) + self.w_pipes[node].hmset( + 'WAREHOUSE.' + key, + { + 'W_ID' : record[0], + 'W_NAME' : record[1], + 'W_STREET_1' : record[2], + 'W_STREET_2' : record[3], + 'W_CITY' : record[4], + 'W_STATE' : record[5], + 'W_ZIP' : record[6], + 'W_TAX' : record[7], + 'W_YTD' : record[8], + } + ) + self.w_sizes[node] += 2 + elif tableName == 'DISTRICT' : + node = self.shard(record[column_map['D_W_ID']]) + key = self.safeKey([record[1], record[0]]) + self.w_pipes[node].sadd('DISTRICT.IDS', key) + self.w_pipes[node].hmset( + 'DISTRICT.' + key, + { + 'D_ID' : record[0], + 'D_W_ID' : record[1], + 'D_NAME' : record[2], + 'D_STREET_1' : record[3], + 'D_STREET_2' : record[4], + 'D_CITY' : record[5], + 'D_STATE' : record[6], + 'D_ZIP' : record[7], + 'D_TAX' : record[8], + 'D_YTD' : record[9], + 'D_NEXT_O_ID' : record[10], + } + ) + self.w_sizes[node] += 2 + elif tableName == 'CUSTOMER' : + node = self.shard(record[column_map['C_W_ID']]) + key = self.safeKey([record[2], record[1], record[0]]) + self.w_pipes[node].sadd('CUSTOMER.IDS', key) + self.w_pipes[node].hmset( + 'CUSTOMER.' + key, + { + 'C_ID' : record[0], + 'C_D_ID' : record[1], + 'C_W_ID' : record[2], + 'C_FIRST' : record[3], + 'C_MIDDLE' : record[4], + 'C_LAST' : record[5], + 'C_STREET_1' : record[6], + 'C_STREET_2' : record[7], + 'C_CITY' : record[8], + 'C_ZIP' : record[9], + 'C_PHONE' : record[10], + 'C_SINCE' : record[11], + 'C_CREDIT' : record[12], + 'C_CREDIT_LIM' : record[13], + 'C_DISCOUNT' : record[14], + 'C_BALANCE' : record[15], + 'C_YTD_PAYMENT' : record[16], + 'C_PAYMENT_CNT' : record[17], + 'C_DELIVERY_CNT' : record[18], + 'C_DATA' : record[19], + } + ) + + # Add Special Index for Customer Table + index_key = self.safeKey([record[2], record[1], record[5]]) + self.w_pipes[node].sadd( + 'CUSTOMER.INDEXES.NAMESEARCH.' + index_key, + key + ) + self.w_sizes[node] += 3 + elif tableName == 'HISTORY' : + node = self.shard(record[column_map['H_W_ID']]) + key = self.safeKey([self.next_scores['HISTORY']]) + self.w_pipes[node].sadd('HISTORY.IDS', key) + self.w_pipes[node].hmset( + 'HISTORY.' + key, + { + 'H_C_ID' : record[0], + 'H_C_D_ID' : record[1], + 'H_C_W_ID' : record[2], + 'H_D_ID' : record[3], + 'H_W_ID' : record[4], + 'H_DATE' : record[5], + 'H_AMOUNT' : record[6], + 'H_DATA' : record[7], + } + ) + self.w_sizes[node] += 2 + elif tableName == 'STOCK' : + node = self.shard(record[column_map['S_W_ID']]) + key = self.safeKey([record[1], record[0]]) + self.w_pipes[node].sadd('STOCK.IDS', key) + self.w_pipes[node].hmset( + 'STOCK.' + key, + { + 'S_I_ID' : record[0], + 'S_W_ID' : record[1], + 'S_QUANTITY' : record[2], + 'S_DIST_01' : record[3], + 'S_DIST_02' : record[4], + 'S_DIST_03' : record[5], + 'S_DIST_04' : record[6], + 'S_DIST_05' : record[7], + 'S_DIST_06' : record[8], + 'S_DIST_07' : record[9], + 'S_DIST_08' : record[10], + 'S_DIST_09' : record[11], + 'S_DIST_10' : record[12], + 'S_YTD' : record[13], + 'S_ORDER_CNT' : record[14], + 'S_REMOTE_CNT' : record[15], + 'S_DATA' : record[16], + + } + ) + self.w_sizes[node] += 2 + elif tableName == 'ORDERS' : + node = self.shard(record[column_map['O_W_ID']]) + key = self.safeKey([ + record[0], + record[3], + record[1], + record[2] + ]) + self.w_pipes[node].sadd('ORDER.IDS', key) + self.w_pipes[node].hmset( + 'ORDER.' + key, + { + 'O_ID' : record[0], + 'O_D_ID' : record[1], + 'O_W_ID' : record[2], + 'O_C_ID' : record[3], + 'O_ENTRY_D' : record[4], + 'O_CARRIER_ID' : record[5], + 'O_OL_CNT' : record[6], + 'O_ALL_LOCAL' : record[7], + } + ) + + # Add Special Index for Order Table + index_key = self.safeKey([record[2], record[1], record[3]]) + self.w_pipes[node].sadd( + 'ORDERS.INDEXES.ORDERSEARCH.' + index_key, + key + ) + self.w_sizes[node] += 3 + elif tableName == 'NEW_ORDER' : + node = self.shard(record[column_map['NO_W_ID']]) + key = self.safeKey([record[1], record[2], record[0]]) + self.w_pipes[node].sadd('NEW_ORDER.IDS', key) + self.w_pipes[node].hmset( + 'NEW_ORDER.' + key, + { + 'NO_O_ID' : record[0], + 'NO_D_ID' : record[1], + 'NO_W_ID' : record[2], + } + ) + + # Add Special Index for New Order Table + index_key = self.safeKey([record[1], record[2]]) + self.w_pipes[node].sadd( + 'NEW_ORDER.INDEXES.GETNEWORDER.' + index_key, + key + ) + self.w_sizes[node] += 3 + elif tableName == 'ORDER_LINE' : + node = self.shard(record[column_map['OL_W_ID']]) + key = self.safeKey([ + record[2], + record[1], + record[0], + record[3] + ]) + self.w_pipes[node].sadd('ORDER_LINE.IDS', key) + self.w_pipes[node].hmset( + 'ORDER_LINE.' + key, + { + 'OL_O_ID' : record[0], + 'OL_D_ID' : record[1], + 'OL_W_ID' : record[2], + 'OL_NUMBER' : record[3], + 'OL_I_ID' : record[4], + 'OL_SUPPLY_W_ID' : record[5], + 'OL_DELIVERY_D' : record[6], + 'OL_QUANTITY' : record[7], + 'OL_AMOUNT' : record[8], + 'OL_DIST_INFO' : record[9], + } + ) + + # Add Special Index for Order Line Table + index_key = self.safeKey([record[0], record[1], record[2]]) + self.w_pipes[node].sadd( + 'ORDER_LINE.INDEXES.SUMOLAMOUNT.' + index_key, + key + ) + self.w_sizes[node] += 3 + elif tableName == 'ITEMS' : + key = self.safeKey([record[0]]); + pi = 0 + for pipe in self.w_pipes : + pipe.sadd('ITEM.IDS', key) + pipe.hmset( + 'ITEM.' + key, + { + 'I_ID' : record[0], + 'I_IM_ID' : record[1], + 'I_NAME' : record[2], + 'I_PRICE' : record[3], + 'I_DATA' : record[4], + } + ) + self.w_sizes[pi] += 2 + pass + self.next_scores[tableName] += 1 + + #print key, + # Check to see if pipelines need to be flushed + for index in range(len(self.w_pipes)) : + if self.w_sizes[index] > 10000 : + if self.debug['load'] != 'None' : + print index, + sys.stdout.flush() + self.w_pipes[index].execute() + self.w_sizes[index] = 0 + + if self.debug['load'] == 'Verbose' : + print '' + # End loadTuples() + + #------------------------------------------------------------------------ + # Return default configuration when none is specified via command line + # + # @return dictionary configuration parameters + #------------------------------------------------------------------------ + def makeDefaultConfig(self) : + return self.DEFAULT_CONFIG + # End makeDefaultConfig() + + #------------------------------------------------------------------------ + # Create a safe key for Redis by removing invalid characters from + # input list + # + # @param list keys + #------------------------------------------------------------------------ + def safeKey(self, keys) : + new_keys = [] + for k in keys : + new_keys.append(str(k)) + return self.KEY_SEPARATOR.join(new_keys + ).replace('\n', '').replace(' ','') + # End safeKey() + + #------------------------------------------------------------------------ + # Sharding Method for determine which not to access + # + # @param string w_id warehouse id + # @return int + #------------------------------------------------------------------------ + def shard(self, w_id) : + return int(w_id) % self.db_count + # End shard() \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/scalarisdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/scalarisdriver.py new file mode 100644 index 00000000..662f978e --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/scalarisdriver.py @@ -0,0 +1,1039 @@ +''' +Created on May 2, 2011 + +@author: Irina Calciu and Alex Gillmor + +Scalaris Driver for CS227 TPCC Benchmark +''' + +from abstractdriver import * + +import os, logging, commands, constants + +from collections import defaultdict + +from api.Scalaris import JSONConnection, Transaction, TransactionSingleOp, NotFoundException + + +#Table Definitions +TABLE_COLUMNS = { + constants.TABLENAME_ITEM: [ + "I_ID", # INTEGER + "I_IM_ID", # INTEGER + "I_NAME", # VARCHAR + "I_PRICE", # FLOAT + "I_DATA", # VARCHAR + ], + constants.TABLENAME_WAREHOUSE: [ + "W_ID", # SMALLINT + "W_NAME", # VARCHAR + "W_STREET_1", # VARCHAR + "W_STREET_2", # VARCHAR + "W_CITY", # VARCHAR + "W_STATE", # VARCHAR + "W_ZIP", # VARCHAR + "W_TAX", # FLOAT + "W_YTD", # FLOAT + ], + constants.TABLENAME_DISTRICT: [ + "D_ID", # TINYINT + "D_W_ID", # SMALLINT + "D_NAME", # VARCHAR + "D_STREET_1", # VARCHAR + "D_STREET_2", # VARCHAR + "D_CITY", # VARCHAR + "D_STATE", # VARCHAR + "D_ZIP", # VARCHAR + "D_TAX", # FLOAT + "D_YTD", # FLOAT + "D_NEXT_O_ID", # INT + ], + constants.TABLENAME_CUSTOMER: [ + "C_ID", # INTEGER + "C_D_ID", # TINYINT + "C_W_ID", # SMALLINT + "C_FIRST", # VARCHAR + "C_MIDDLE", # VARCHAR + "C_LAST", # VARCHAR + "C_STREET_1", # VARCHAR + "C_STREET_2", # VARCHAR + "C_CITY", # VARCHAR + "C_STATE", # VARCHAR + "C_ZIP", # VARCHAR + "C_PHONE", # VARCHAR + "C_SINCE", # TIMESTAMP + "C_CREDIT", # VARCHAR + "C_CREDIT_LIM", # FLOAT + "C_DISCOUNT", # FLOAT + "C_BALANCE", # FLOAT + "C_YTD_PAYMENT", # FLOAT + "C_PAYMENT_CNT", # INTEGER + "C_DELIVERY_CNT", # INTEGER + "C_DATA", # VARCHAR + ], + constants.TABLENAME_STOCK: [ + "S_I_ID", # INTEGER + "S_W_ID", # SMALLINT + "S_QUANTITY", # INTEGER + "S_DIST_01", # VARCHAR + "S_DIST_02", # VARCHAR + "S_DIST_03", # VARCHAR + "S_DIST_04", # VARCHAR + "S_DIST_05", # VARCHAR + "S_DIST_06", # VARCHAR + "S_DIST_07", # VARCHAR + "S_DIST_08", # VARCHAR + "S_DIST_09", # VARCHAR + "S_DIST_10", # VARCHAR + "S_YTD", # INTEGER + "S_ORDER_CNT", # INTEGER + "S_REMOTE_CNT", # INTEGER + "S_DATA", # VARCHAR + ], + constants.TABLENAME_ORDERS: [ + "O_ID", # INTEGER + "O_D_ID", # TINYINT + "O_W_ID", # SMALLINT + "O_C_ID", # INTEGER + "O_ENTRY_D", # TIMESTAMP + "O_CARRIER_ID", # INTEGER + "O_OL_CNT", # INTEGER + "O_ALL_LOCAL", # INTEGER + ], + constants.TABLENAME_NEW_ORDER: [ + "NO_O_ID", # INTEGER + "NO_D_ID", # TINYINT + "NO_W_ID", # SMALLINT + ], + constants.TABLENAME_ORDER_LINE: [ + "OL_O_ID", # INTEGER + "OL_D_ID", # TINYINT + "OL_W_ID", # SMALLINT + "OL_NUMBER", # INTEGER + "OL_I_ID", # INTEGER + "OL_SUPPLY_W_ID", # SMALLINT + "OL_DELIVERY_D", # TIMESTAMP + "OL_QUANTITY", # INTEGER + "OL_AMOUNT", # FLOAT + "OL_DIST_INFO", # VARCHAR + ], + constants.TABLENAME_HISTORY: [ + "H_C_ID", # INTEGER + "H_C_D_ID", # TINYINT + "H_C_W_ID", # SMALLINT + "H_D_ID", # TINYINT + "H_W_ID", # SMALLINT + "H_DATE", # TIMESTAMP + "H_AMOUNT", # FLOAT + "H_DATA", # VARCHAR + ], +} +TABLE_INDEXES = { + constants.TABLENAME_ITEM: [ + "I_ID", + ], + constants.TABLENAME_WAREHOUSE: [ + "W_ID", + ], + constants.TABLENAME_DISTRICT: [ + "D_ID", + "D_W_ID", + ], + constants.TABLENAME_CUSTOMER: [ + "C_ID", + "C_D_ID", + "C_W_ID", + ], + constants.TABLENAME_STOCK: [ + "S_I_ID", + "S_W_ID", + ], + constants.TABLENAME_ORDERS: [ + "O_ID", + "O_D_ID", + "O_W_ID", + "O_C_ID", + ], + constants.TABLENAME_NEW_ORDER: [ + "NO_O_ID", + "NO_D_ID", + "NO_W_ID", + ], + constants.TABLENAME_ORDER_LINE: [ + "OL_O_ID", + "OL_D_ID", + "OL_W_ID", + ], +} + +def createPrimaryKey(tableName, id, obj): + ''' + Helper method to create normalized primary keys + ''' + if tableName == constants.TABLENAME_ITEM: + return '%s.%s' % (constants.TABLENAME_ITEM, id) + elif tableName == constants.TABLENAME_WAREHOUSE: + return '%s.%s' %(constants.TABLENAME_WAREHOUSE, id) + elif tableName == constants.TABLENAME_DISTRICT: + return '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, obj['D_W_ID'], constants.TABLENAME_DISTRICT, id) + elif tableName == constants.TABLENAME_CUSTOMER: + return '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, obj['C_W_ID'], \ + constants.TABLENAME_DISTRICT, obj['C_D_ID'], constants.TABLENAME_CUSTOMER, id) + elif tableName == constants.TABLENAME_ORDERS: + return '%s.%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, obj['O_W_ID'], \ + constants.TABLENAME_DISTRICT, obj['O_D_ID'], constants.TABLENAME_CUSTOMER, obj['O_C_ID'], \ + constants.TABLENAME_ORDERS, id) + else: + return '%s.%s' % (tableName,id) + + + + + +class ScalarisDriver(AbstractDriver): + ''' + Scalaris Driver for CS227 TPCC benchmark + ''' + + DEFAULT_CONFIG = { + "database": ("The path to the main Scalaris Node", "http://localhost:8000" ), + } + + def __init__(self, ddl): + ''' + init + ''' + super(ScalarisDriver, self).__init__("scalaris", ddl) + + ## ---------------------------------------------- + ## makeDefaultConfig + ## ---------------------------------------------- + def makeDefaultConfig(self): + return ScalarisDriver.DEFAULT_CONFIG + + ## ---------------------------------------------- + ## loadConfig + ## ---------------------------------------------- + def loadConfig(self, config): + for key in ScalarisDriver.DEFAULT_CONFIG.keys(): + assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) + + self.database = str(config["database"]) + + + self.conn = JSONConnection(url=self.database) + + #self.tran = self.transactionFactory() + #single op is much faster for nearly all operations + self.tran = TransactionSingleOp(self.conn) + + def transactionFactory(self): + ''' + Transaction object factory method + ''' + return Transaction(self.conn) + + def loadTuples(self, tableName, tuples): + s = set([constants.TABLENAME_HISTORY, \ + constants.TABLENAME_NEW_ORDER, \ + constants.TABLENAME_ORDER_LINE,\ + constants.TABLENAME_STOCK]) + + if tableName in s: + self.loadComplexTuples(tableName,tuples) + else: + self.loadSimpleTuples(tableName, tuples) + + if tableName == constants.TABLENAME_ORDERS: + self.loadOrderCustomer(tableName, tuples) + if tableName == constants.TABLENAME_CUSTOMER: + self.loadWarehouseDistrictCustomers(tableName, tuples) + + def loadHistory(self,tuples): + ''' + Specialized method for history table. History is stored based on customer info. + ''' + history_d=defaultdict(lambda : defaultdict(lambda : defaultdict(list))) + tableDef = TABLE_COLUMNS[constants.TABLENAME_HISTORY] + for tuple in tuples: + history = dict(zip(tableDef,tuple)) + w_id = history["H_C_W_ID"] + d_id = history['H_C_D_ID'] + c_id = history["H_C_ID"] + + history_d[w_id][d_id][c_id].append(history) + + for w in history_d.keys(): + for d in history_d[w].keys(): + for o in history_d[w][d].keys(): + history_key = '%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ + constants.TABLENAME_DISTRICT, d, \ + constants.TABLENAME_CUSTOMER, o,\ + constants.TABLENAME_HISTORY) + self.tran.write(history_key, history_d[w][d][o]) + + def loadStock(self,tuples): + ''' + Specialized method for laoding stock + ''' + + #need to reestablish because some large values cause problems with scalaris ws + self.conn = JSONConnection(url=self.database) + self.tran = TransactionSingleOp(self.conn) + + stock_d = defaultdict(defaultdict) + tableDef = TABLE_COLUMNS[constants.TABLENAME_STOCK] + stock_idx = defaultdict(list) + + for tuple in tuples: + stock = dict(zip(tableDef,tuple)) + tuple_short = [tuple[0], tuple[2]] + stock_short = dict(zip(['S_I_ID', 'S_QUANTITY'],tuple_short)) + s_w_id = stock['S_W_ID'] + s_i_id = stock['S_I_ID'] + stock_d[s_w_id][s_i_id] = stock + stock_idx[s_w_id].append(stock_short) + + for s in stock_d.keys(): + s_key = '%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, s, constants.TABLENAME_STOCK) + print "key %s" % s_key + print "value %s" % stock_idx[s] + self.tran.write(s_key, stock_idx[s]) + + for w in stock_d.keys(): + for i in stock_d[w].keys(): + s_key = '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w, constants.TABLENAME_STOCK, i) + self.tran.write(s_key, stock_d[s][i]) + + + + def loadOrderLine(self,tuples): + ''' + Load order line objects: + WAREHOUSE.W_ID.DISTRICT.D_ID.ORDERLINE.O_ID -> OrderLine Object + WAREHOUSE.W_ID.DISTRICT.D_ID.ORDERLINE -> List of Order Ids for that District + ''' + ol_d=defaultdict(lambda : defaultdict(lambda : defaultdict(list))) + order_ids = defaultdict(lambda : defaultdict(list)) + tableDef = TABLE_COLUMNS[constants.TABLENAME_ORDER_LINE] + for tuple in tuples: + no = dict(zip(tableDef,tuple)) + w_id = no["OL_W_ID"] + d_id = no['OL_D_ID'] + o_id = no["OL_O_ID"] + + ol_d[w_id][d_id][o_id].append(no) + order_ids[w_id][d_id].append(str(o_id)) + + for w in ol_d.keys(): + for d in ol_d[w].keys(): + for o in ol_d[w][d].keys(): + ol_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ + constants.TABLENAME_DISTRICT, d, \ + constants.TABLENAME_ORDER_LINE, o) + self.tran.write(ol_key, ol_d[w][d][o]) + for w in order_ids.keys(): + for d in order_ids[w].keys(): + no_key = '%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ + constants.TABLENAME_DISTRICT, d, \ + constants.TABLENAME_ORDER_LINE) + self.tran.write(no_key, order_ids[w][d]) + + def loadNewOrder(self,tuples): + ''' + Load new order objects: + WAREHOUSE.W_ID.DISTRICT.D_ID.NEW_ORDER.O_ID -> New Order Object + WAREHOUSE.W_ID.DISTRICT.D_ID.NEW_ORDER -> List of Order Ids for that new_order + ''' + no_d=defaultdict(lambda : defaultdict(lambda : defaultdict(list))) + order_ids = defaultdict(lambda : defaultdict(list)) + tableDef = TABLE_COLUMNS[constants.TABLENAME_NEW_ORDER] + for tuple in tuples: + no = dict(zip(tableDef,tuple)) + w_id = no["NO_W_ID"] + d_id = no['NO_D_ID'] + o_id = no["NO_O_ID"] + + no_d[w_id][d_id][o_id].append(no) + order_ids[w_id][d_id].append(str(o_id)) + + for w in no_d.keys(): + for d in no_d[w].keys(): + for o in no_d[w][d].keys(): + no_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ + constants.TABLENAME_DISTRICT, d, \ + constants.TABLENAME_NEW_ORDER, o) + self.tran.write(no_key, no_d[w][d][o]) + + for w in order_ids.keys(): + for d in order_ids[w].keys(): + no_key = '%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ + constants.TABLENAME_DISTRICT, d, \ + constants.TABLENAME_NEW_ORDER) + self.tran.write(no_key, order_ids[w][d]) + + def loadComplexTuples(self, tableName, tuples): + ''' + Dispatching method for tuples that need secondary/advanced indexing + ''' + if tableName == constants.TABLENAME_ORDER_LINE: + self.loadOrderLine(tuples) + if tableName == constants.TABLENAME_NEW_ORDER: + self.loadNewOrder(tuples) + if tableName == constants.TABLENAME_HISTORY: + self.loadHistory(tuples) + if tableName == constants.TABLENAME_STOCK: + self.loadStock(tuples) + #self.tran.commit() + + + def loadOrderCustomer(self, tableName, tuples): + ''' + Load order objects: + WAREHOUSE.W_ID.DISTRICT.D_ID.ORDERS.O_ID -> Order Object + WAREHOUSE.W_ID.DISTRICT.D_ID.ORDERS-> List of Order Ids for that district + ''' + + #order case + tableDef = TABLE_COLUMNS[tableName] + o_d=defaultdict(lambda : defaultdict(lambda : defaultdict(list))) + + + for tuple in tuples: + value = dict(zip(tableDef,tuple)) + o_id = value['O_ID'] + c_id = value['O_C_ID'] + d_id = value['O_D_ID'] + w_id = value['O_W_ID'] + + oc_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE,w_id, constants.TABLENAME_DISTRICT,d_id, constants.TABLENAME_ORDERS,o_id) + self.tran.write(oc_key, c_id) + + o_d[w_id][d_id][c_id].append(str(o_id)) + + for k1 in o_d.keys(): + for k2 in o_d[k1].keys(): + for k3 in o_d[k1][k2].keys(): + orders_key = '%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, k1, constants.TABLENAME_DISTRICT, k2, \ + constants.TABLENAME_CUSTOMER, k3, constants.TABLENAME_ORDERS) + self.tran.write(orders_key,o_d[k1][k2][k3]) + + + def loadWarehouseDistrictCustomers(self, tableName, tuples): + ''' + Load warehouse district customers: + WAREHOUSE.W_ID.DISTRICT.CUSTOMERS -> List of Customer IDs + ''' + + #customers + custs = defaultdict(lambda : defaultdict(list)) + tableDef = TABLE_COLUMNS[tableName] + for tuple in tuples: + value = dict(zip(tableDef,tuple)) + + c_last = value['C_LAST'] + c_id = value['C_ID'] + c_idx = {} + c_idx['C_LAST'] = c_last + c_idx['C_ID'] = c_id + d_id = value['C_D_ID'] + w_id = value['C_W_ID'] + custs[w_id][d_id].append(c_idx) + + for w in custs.keys(): + for d in custs[w].keys(): + custs_key = '%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ + constants.TABLENAME_DISTRICT, d, \ + 'CUSTOMERS') + self.tran.write(custs_key, custs[w][d]) + + + + + def loadSimpleTuples(self, tableName, tuples): + """Load a list of tuples into the target table""" + + self.conn = JSONConnection(url=self.database) + self.tran = TransactionSingleOp(self.conn) + + if len(tuples) == 0: return + + idx =0 + + tableDef = TABLE_COLUMNS[tableName] + + for tuple in tuples: + pId= tuple[0] + value = dict(zip(tableDef[1:],tuple[1:])) + primaryKey = createPrimaryKey(tableName, pId, value) + self.tran.write(primaryKey, value) + #self.tran.commit() + if idx == 500: + print '%s %s' % (tableName, primaryKey) + idx = 0 +# self.tran.commit() + idx+=1 + + #self.tran.commit() + + def loadFinish(self): + logging.info("Commiting changes to database") + #self.tran.commit() + + def executeStart(self): + self.conn = JSONConnection(url=self.database) + + #self.tran = self.transactionFactory() + self.tran = TransactionSingleOp(self.conn) + def executeFinish(self): + """Callback after the execution phase finishes""" +# self.tran.commit() + self.tran.closeConnection() + #self.tran.commit() + + ## ---------------------------------------------- + ## doDelivery + ## ---------------------------------------------- + def doDelivery(self, params): + w_id = params["w_id"] + o_carrier_id = params["o_carrier_id"] + ol_delivery_d = params["ol_delivery_d"] + + result = [ ] + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): + ## getNewOrder + ## WAREHOUSE.W_ID.DISTRICT.D_ID.NEW_ORDERS -> List of New Orders + no_key = '%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_NEW_ORDER) + + newOrders = None + newOrders = self.tran.read(no_key) #we expect a list of new order ifd + if newOrders == None: + ## No orders for this district: skip it. Note: This must be reported if > 1% + continue + assert len(newOrders) > 0 + + #just ids + no_o_id = min(newOrders) + newOrders.remove(no_o_id) + no_o_id = int(no_o_id) + #new order objects + + ## getCId + ## WAREHOUSE.W_ID.DISTRICT.D_ID.ORDER.ORDER_ID = List of Customers + customer = None + c_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ + constants.TABLENAME_DISTRICT, d_id, \ + constants.TABLENAME_ORDERS, no_o_id) + + customer = self.tran.read(c_key) + assert customer != None + c_id = customer + + ## sumOLAmount + ## WAREHOUSE.W_ID.DISTRICT.D_ID.ORDER_LINE.ORDER_ID -> List of OrderLine Objects or list of OL_NUMBERS + orderLines = [] + ol_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ + constants.TABLENAME_DISTRICT, d_id, \ + constants.TABLENAME_ORDER_LINE, no_o_id) + + orderLines = self.tran.read(ol_key) + + ol_total = 0.0 + for ol in orderLines: + ol_total += ol["OL_AMOUNT"] + ol['OL_DELIVERY_D'] = ol_delivery_d + ## FOR + + ## deleteNewOrder + #self.new_order.remove({"NO_D_ID": d_id, "NO_W_ID": w_id, "NO_O_ID": no_o_id}) + no_del_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_NEW_ORDER, no_o_id) + self.tran.write(no_del_key, None) + self.tran.write(no_key,newOrders) + + ## updateOrders + order_key = '%s.%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_CUSTOMER, c_id, constants.TABLENAME_ORDERS, no_o_id) + order = self.tran.read(order_key) + order['O_CARRIER_ID'] = o_carrier_id + self.tran.write(order_key, order) + #self.orders.update({"O_ID": no_o_id, "O_D_ID": d_id, "O_W_ID": w_id}, {"$set": {"O_CARRIER_ID": o_carrier_id}}, multi=False) + + ## updateOrderLine + #self.order_line.update({"OL_O_ID": no_o_id, "OL_D_ID": d_id, "OL_W_ID": w_id}, {"$set": {"OL_DELIVERY_D": ol_delivery_d}}, multi=False) + self.tran.write(ol_key, orderLines) + + + # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) + # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure + # them out + # If there are no order lines, SUM returns null. There should always be order lines. + assert ol_total != None, "ol_total is NULL: there are no order lines. This should not happen" + assert ol_total > 0.0 + + ## updateCustomer + customer_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ + constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_CUSTOMER, c_id) + + customer = self.tran.read(customer_key) + customer["C_BALANCE"]=customer["C_BALANCE"]+ol_total + + #self.customer.update({"C_ID": c_id, "C_D_ID": d_id, "C_W_ID": w_id}, {"$inc": {"C_BALANCE": ol_total}}) + + result.append((d_id, no_o_id)) + ## FOR + return result + + ## ---------------------------------------------- + ## doNewOrder + ## ---------------------------------------------- + def doNewOrder(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + o_entry_d = params["o_entry_d"] + i_ids = params["i_ids"] + i_w_ids = params["i_w_ids"] + i_qtys = params["i_qtys"] + s_dist_col = "S_DIST_%02d" % d_id + + assert len(i_ids) > 0 + assert len(i_ids) == len(i_w_ids) + assert len(i_ids) == len(i_qtys) + + ## http://stackoverflow.com/q/3844931/ + all_local = (not i_w_ids or [w_id] * len(i_w_ids) == i_w_ids) + + ## GET ALL ITEMS WITH + ## ITEM.I_ID + items = [] + for id in i_ids: + i_key = '%s.%s' % (constants.TABLENAME_ITEM, id) + item = self.tran.read(i_key) + if item: + items.append(item) + + ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. + ## Note that this will happen with 1% of transactions on purpose. + if len(items) != len(i_ids): + ## TODO Abort here! + return + ## IF + + ## ---------------- + ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER + ## ---------------- + + # getWarehouseTaxRate + ## WAREHOUSE.W_ID -> warehouse object + w_key = '%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id) + w = self.tran.read(w_key) + assert w + w_tax = w["W_TAX"] + + # getDistrict + ## WAREHOUSE.W_ID.DISTRICT.D_ID -> district object + + d_key = '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id) + d = self.tran.read(d_key) + assert d + d_tax = d["D_TAX"] + d_next_o_id = d["D_NEXT_O_ID"] + + + # incrementNextOrderId + d["D_NEXT_O_ID"] = d["D_NEXT_O_ID"] + 1 + self.tran.write(d_key, d); + + + # getCustomer + ## WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMER.C_ID -> Customer Object + c_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ + constants.TABLENAME_CUSTOMER, c_id) + c = self.tran.read(c_key) + assert c + c_discount = c["C_DISCOUNT"] + + ## ---------------- + ## Insert Order Information + ## ---------------- + ol_cnt = len(i_ids) + o_carrier_id = constants.NULL_CARRIER_ID + + # createOrder + ## write to order object to WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMER.C_ID.ORDER.O_ID + o_key = '%s.%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ + constants.TABLENAME_CUSTOMER, c_id, constants.TABLENAME_ORDERS, d_next_o_id) + + order = {#"O_ID": d_next_o_id, + "O_D_ID": d_id, + "O_W_ID": w_id, + "O_C_ID": c_id, + "O_ENTRY_D": o_entry_d, + "O_CARRIER_ID": o_carrier_id, + "O_OL_CNT": ol_cnt, + "O_ALL_LOCAL": all_local} + + self.tran.write(o_key, order) + + + # createNewOrder + + ## write new order object to WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMER.C_ID.ORDER.O_ID.NEW_ORDER.NO_ID + ## newOrder + ## assumption + ## WAREHOUSE.W_ID.DISTRICT.D_ID.NEW_ORDER.ORDER_ID -> List of New Order Objects + new_order = {"NO_O_ID": d_next_o_id, "NO_D_ID": d_id, "NO_W_ID": w_id} + no_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_NEW_ORDER, d_next_o_id) + + try: + new_orders = self.tran.read(no_key) + except NotFoundException: + new_orders = [] + + new_orders.append(new_order) + + self.tran.write(no_key, new_orders) + + #self.new_order.insert({"NO_O_ID": d_next_o_id, "NO_D_ID": d_id, "NO_W_ID": w_id}) + + ## ---------------- + ## OPTIMIZATION: + ## If all of the items are at the same warehouse, then we'll issue a single + ## request to get their information + ## ---------------- + stockInfos = None + if all_local and False: + # getStockInfo + ##WAREHOUSE.W_ID.STOCK -returns-> List of Stock Objects w/ S_I_ID and S_QUANTITY + allStocks_key = '%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_STOCK) + allStocks = self.tran.read(allStocks_key) + + assert len(allStocks) == ol_cnt + stockInfos = { } + for si in allStocks: + stockInfos["S_I_ID"] = si # HACK + ## IF + + ## ---------------- + ## Insert Order Item Information + ## ---------------- + item_data = [ ] + total = 0 + for i in range(ol_cnt): + ol_number = i + 1 + ol_supply_w_id = i_w_ids[i] + ol_i_id = i_ids[i] + ol_quantity = i_qtys[i] + + itemInfo = items[i] + i_name = itemInfo["I_NAME"] + i_data = itemInfo["I_DATA"] + i_price = itemInfo["I_PRICE"] + + pformat = None + # getStockInfo + if all_local and stockInfos != None: + si = stockInfos[ol_i_id] + assert si["S_I_ID"] == ol_i_id, "S_I_ID should be %d\n%s" % (ol_i_id, pformat(si)) + else: + ## WAREHOUSE.W_ID.STOCK.S_ID + + allStocks_key = '%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_STOCK) + allStocks = self.tran.read(allStocks_key) + + for stock in allStocks: + if stock['S_I_ID'] == ol_i_id: + si = stock + break + + assert si, "Failed to find S_I_ID: %d\n%s" % (ol_i_id, pformat(itemInfo)) + + si_key = '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_STOCK, si['S_I_ID']) + stock= self.tran.read(si_key) + + + s_quantity = si["S_QUANTITY"] + s_ytd = stock["S_YTD"] + s_order_cnt = stock["S_ORDER_CNT"] + s_remote_cnt = stock["S_REMOTE_CNT"] + s_data = stock["S_DATA"] + s_dist_xx = stock[s_dist_col] # Fetches data from the s_dist_[d_id] column + + ## Update stock + s_ytd += ol_quantity + if s_quantity >= ol_quantity + 10: + s_quantity = s_quantity - ol_quantity + else: + s_quantity = s_quantity + 91 - ol_quantity + s_order_cnt += 1 + + if ol_supply_w_id != w_id: s_remote_cnt += 1 + + # updateStock + si["S_QUANTITY"] = s_quantity + stock["S_QUANTITY"] = s_quantity + stock["S_YTD"] = s_ytd + stock["S_ORDER_CNT"] = s_order_cnt + stock["S_REMOTE_CNT"] = s_remote_cnt + ## so this should be cools + self.tran.write(allStocks_key, allStocks) + self.tran.write(si_key, stock) + + + if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: + brand_generic = 'B' + else: + brand_generic = 'G' + ## Transaction profile states to use "ol_quantity * i_price" + ol_amount = ol_quantity * i_price + total += ol_amount + + order_line = {"OL_O_ID": d_next_o_id, + "OL_D_ID": d_id, + "OL_W_ID": w_id, + "OL_NUMBER": ol_number, + "OL_I_ID": ol_i_id, + "OL_SUPPLY_W_ID": ol_supply_w_id, + "OL_DELIVERY_D": o_entry_d, + "OL_QUANTITY": ol_quantity, + "OL_AMOUNT": ol_amount, + "OL_DIST_INFO": s_dist_xx} + + ##WAREHOUSE.W_ID.DISTRICT.D_ID.ORDER_LINE.ORDER_ID -> List of OrderLine Objects + ol_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ + constants.TABLENAME_ORDER_LINE, d_next_o_id) + + try: + order_lines = self.tran.read(ol_key) + except NotFoundException: + order_lines = [] + + order_lines.append(order_line) + self.tran.write(ol_key, order_lines) + + # createOrderLine + ## Add the info to be returned + item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) + ## FOR + + ## Adjust the total for the discount + #print "c_discount:", c_discount, type(c_discount) + #print "w_tax:", w_tax, type(w_tax) + #print "d_tax:", d_tax, type(d_tax) + total *= (1 - c_discount) * (1 + w_tax + d_tax) + + ## Pack up values the client is missing (see TPC-C 2.4.3.5) + misc = [ (w_tax, d_tax, d_next_o_id, total) ] + + return [ c, misc, item_data ] + + ## ---------------------------------------------- + ## doOrderStatus + ## ---------------------------------------------- + def doOrderStatus(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + + + if c_id != None: + # getCustomerByCustomerId + cust_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ + constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_CUSTOMER, c_id) + c = self.tran.read(cust_key) + else: + # getCustomersByLastName + # Get the midpoint customer's id + #WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMERS -> List of Customers (or C_ID:C_LAST pairs) + all_customers_key = '%s.%s.%s.%s.CUSTOMERS' % (constants.TABLENAME_WAREHOUSE, w_id, \ + constants.TABLENAME_DISTRICT, d_id) + + all_customers = self.tran.read(all_customers_key) + all_customers = [customer for customer in all_customers if customer['C_LAST']==c_last] + + namecnt = len(all_customers) + assert namecnt > 0 + index = (namecnt-1)/2 + c = all_customers[index] + c_id = c["C_ID"] + cust_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ + constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_CUSTOMER, c_id) + c = self.tran.read(cust_key) + + # getLastOrder + ## WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMER.C_ID.ORDERS - > List of Orders + orders_key = '%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ + constants.TABLENAME_CUSTOMER, c_id, constants.TABLENAME_ORDERS) + orders = self.tran.read(orders_key) + + o_id = max(orders) + + order_key = '%s.%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ + constants.TABLENAME_CUSTOMER, c_id, constants.TABLENAME_ORDERS, o_id) + + + order = self.tran.read(order_key) + + if order: + # getOrderLines + + ## WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMER.ORDER_LINE.O_ID -> List of Orderline Objects + ol_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ + constants.TABLENAME_ORDER_LINE, o_id) + orderLines = self.tran.read(ol_key) + else: + orderLines = [ ] + + return [ c, order, orderLines ] + + + ## ---------------------------------------------- + ## doPayment + ## ---------------------------------------------- + def doPayment(self, params): + + w_id = params["w_id"] + d_id = params["d_id"] + h_amount = params["h_amount"] + c_w_id = params["c_w_id"] + c_d_id = params["c_d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + h_date = params["h_date"] + + if c_id != None: + # getCustomerByCustomerId + cust_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, c_w_id, \ + constants.TABLENAME_DISTRICT, c_d_id, constants.TABLENAME_CUSTOMER, c_id) + c = self.tran.read(cust_key) + else: + # getCustomersByLastName + # Get the midpoint customer's id + #WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMERS -> List of Customers (or C_ID:C_LAST pairs) + all_customers_key = '%s.%s.%s.%s.CUSTOMERS' % (constants.TABLENAME_WAREHOUSE, c_w_id, \ + constants.TABLENAME_DISTRICT, c_d_id) + + all_customers = self.tran.read(all_customers_key) + all_customers = [customer for customer in all_customers if customer['C_LAST']==c_last] + + namecnt = len(all_customers) + assert namecnt > 0 + index = (namecnt-1)/2 + c = all_customers[index] + c_id = c["C_ID"] + cust_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, c_w_id, \ + constants.TABLENAME_DISTRICT, c_d_id, constants.TABLENAME_CUSTOMER, c_id) + c = self.tran.read(cust_key) + + assert len(c) > 0 + assert c_id != None + c["C_BALANCE"] = c["C_BALANCE"] - h_amount + c["C_YTD_PAYMENT"] = c["C_YTD_PAYMENT"] + h_amount + c["C_PAYMENT_CNT"] = c["C_PAYMENT_CNT"] + 1 + c_data = c["C_DATA"] + + # getWarehouse + + ## SCALARIS + ## WAREHOUSE.W_ID -> Warehouse Object + w_key = '%s.%s' % (constants.TABLENAME_WAREHOUSE, c_w_id) + w = self.tran.read(w_key) + assert w + + # getDistrict + ## SCALARIS + ## WAREHOUSE.W_ID.DISTRICT.D_ID - > District Object + d_key = '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ + constants.TABLENAME_DISTRICT, d_id) + d = self.tran.read(d_key) + assert d + + # updateWarehouseBalance + w['W_YTD'] = w['W_YTD'] + h_amount + self.tran.write(w_key, w) + + # updateDistrictBalance + d['D_YTD'] = d['D_YTD'] + h_amount + self.tran.write(d_key, d) + + # Customer Credit Information + if c["C_CREDIT"] == constants.BAD_CREDIT: + newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) + c_data = (newData + "|" + c_data) + if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] + + c['C_DATA']=c_data + + + self.tran.write(cust_key, c) + # Concatenate w_name, four spaces, d_name + h_data = "%s %s" % (w["W_NAME"], d["D_NAME"]) + + # insertHistory + history = {#"H_C_ID": c_id, + #"H_C_D_ID": c_d_id, + #"H_C_W_ID": c_w_id, + "H_D_ID": d_id, + "H_W_ID": w_id, + "H_DATE": h_date, + "H_AMOUNT": h_amount, + "H_DATA": h_data} + + h_key = '%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, c_w_id, \ + constants.TABLENAME_DISTRICT, c_d_id, \ + constants.TABLENAME_CUSTOMER, c_id, + constants.TABLENAME_HISTORY) + histories = self.tran.read(h_key) + if histories == None: + histories = [] + histories.append(history) + self.tran.write(h_key, histories) + + # TPC-C 2.5.3.3: Must display the following fields: + # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, + # D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, + # C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, + # C_DISCOUNT, C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = "BC"), + # H_AMOUNT, and H_DATE. + + # Hand back all the warehouse, district, and customer data + return [ w, d, c ] + + ## ---------------------------------------------- + ## doStockLevel + ## ---------------------------------------------- + def doStockLevel(self, params): + w_id = params["w_id"] + d_id = params["d_id"] + threshold = params["threshold"] + + # getOId + ## WAREHOUSE.W_ID.DISTRICT.D_ID -returns-> District Object w/ attribute: D_NEXT_O_ID + ## + d_key = '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id) + d = self.tran.read(d_key) + assert d + o_id = d["D_NEXT_O_ID"] + + ## WAREHOUSE.W_ID.DISTRICT.D_ID.ORDER_LINE.ORDER_ID -returns-> List of Order Line Objects w/ OL_I_ID + ## + ol_key = '%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ + constants.TABLENAME_DISTRICT, d_id, \ + constants.TABLENAME_ORDER_LINE) + + orderLines = self.tran.read(ol_key) + assert orderLines + ol_ids = set() + for ol in orderLines: + if ol < o_id and ol >= o_id-20: + ol_ids.add(ol) + + + ## WAREHOUSE.W_ID.STOCK -returns-> List of Stock Objects w/ S_I_ID and S_QUANTITY + s_key = '%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ + constants.TABLENAME_STOCK) + stocks = self.tran.read(s_key) + result = len([stock for stock in stocks if stock['S_I_ID'] in ol_ids and stock['S_QUANTITY'] < threshold]) + + return int(result) + diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/sqlitedriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/sqlitedriver.py new file mode 100644 index 00000000..bec71091 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/sqlitedriver.py @@ -0,0 +1,470 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Original Java Version: +# Copyright (C) 2008 +# Evan Jones +# Massachusetts Institute of Technology +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +from __future__ import with_statement + +import os +import sqlite3 +import logging +import commands +from pprint import pprint,pformat + +import constants +from abstractdriver import * + +TXN_QUERIES = { + "DELIVERY": { + "getNewOrder": "SELECT NO_O_ID FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID > -1 LIMIT 1", # + "deleteNewOrder": "DELETE FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID = ?", # d_id, w_id, no_o_id + "getCId": "SELECT O_C_ID FROM ORDERS WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ?", # no_o_id, d_id, w_id + "updateOrders": "UPDATE ORDERS SET O_CARRIER_ID = ? WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ?", # o_carrier_id, no_o_id, d_id, w_id + "updateOrderLine": "UPDATE ORDER_LINE SET OL_DELIVERY_D = ? WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ?", # o_entry_d, no_o_id, d_id, w_id + "sumOLAmount": "SELECT SUM(OL_AMOUNT) FROM ORDER_LINE WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ?", # no_o_id, d_id, w_id + "updateCustomer": "UPDATE CUSTOMER SET C_BALANCE = C_BALANCE + ? WHERE C_ID = ? AND C_D_ID = ? AND C_W_ID = ?", # ol_total, c_id, d_id, w_id + }, + "NEW_ORDER": { + "getWarehouseTaxRate": "SELECT W_TAX FROM WAREHOUSE WHERE W_ID = ?", # w_id + "getDistrict": "SELECT D_TAX, D_NEXT_O_ID FROM DISTRICT WHERE D_ID = ? AND D_W_ID = ?", # d_id, w_id + "incrementNextOrderId": "UPDATE DISTRICT SET D_NEXT_O_ID = ? WHERE D_ID = ? AND D_W_ID = ?", # d_next_o_id, d_id, w_id + "getCustomer": "SELECT C_DISCOUNT, C_LAST, C_CREDIT FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id + "createOrder": "INSERT INTO ORDERS (O_ID, O_D_ID, O_W_ID, O_C_ID, O_ENTRY_D, O_CARRIER_ID, O_OL_CNT, O_ALL_LOCAL) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", # d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, o_ol_cnt, o_all_local + "createNewOrder": "INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, NO_W_ID) VALUES (?, ?, ?)", # o_id, d_id, w_id + "getItemInfo": "SELECT I_PRICE, I_NAME, I_DATA FROM ITEM WHERE I_ID = ?", # ol_i_id + "getStockInfo": "SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DIST_%02d FROM STOCK WHERE S_I_ID = ? AND S_W_ID = ?", # d_id, ol_i_id, ol_supply_w_id + "updateStock": "UPDATE STOCK SET S_QUANTITY = ?, S_YTD = ?, S_ORDER_CNT = ?, S_REMOTE_CNT = ? WHERE S_I_ID = ? AND S_W_ID = ?", # s_quantity, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id + "createOrderLine": "INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", # o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info + }, + + "ORDER_STATUS": { + "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id + "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last + "getLastOrder": "SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS WHERE O_W_ID = ? AND O_D_ID = ? AND O_C_ID = ? ORDER BY O_ID DESC LIMIT 1", # w_id, d_id, c_id + "getOrderLines": "SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID = ?", # w_id, d_id, o_id + }, + + "PAYMENT": { + "getWarehouse": "SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP FROM WAREHOUSE WHERE W_ID = ?", # w_id + "updateWarehouseBalance": "UPDATE WAREHOUSE SET W_YTD = W_YTD + ? WHERE W_ID = ?", # h_amount, w_id + "getDistrict": "SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", # w_id, d_id + "updateDistrictBalance": "UPDATE DISTRICT SET D_YTD = D_YTD + ? WHERE D_W_ID = ? AND D_ID = ?", # h_amount, d_w_id, d_id + "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id + "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last + "updateBCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ?, C_DATA = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id + "updateGCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id + "insertHistory": "INSERT INTO HISTORY VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + }, + + "STOCK_LEVEL": { + "getOId": "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", + "getStockCount": """ + SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK + WHERE OL_W_ID = ? + AND OL_D_ID = ? + AND OL_O_ID < ? + AND OL_O_ID >= ? + AND S_W_ID = ? + AND S_I_ID = OL_I_ID + AND S_QUANTITY < ? + """, + }, +} + + +## ============================================== +## SqliteDriver +## ============================================== +class SqliteDriver(AbstractDriver): + DEFAULT_CONFIG = { + "database": ("The path to the SQLite database", "/tmp/tpcc.db" ), + } + + def __init__(self, ddl): + super(SqliteDriver, self).__init__("sqlite", ddl) + self.database = None + self.conn = None + self.cursor = None + + ## ---------------------------------------------- + ## makeDefaultConfig + ## ---------------------------------------------- + def makeDefaultConfig(self): + return SqliteDriver.DEFAULT_CONFIG + + ## ---------------------------------------------- + ## loadConfig + ## ---------------------------------------------- + def loadConfig(self, config): + for key in SqliteDriver.DEFAULT_CONFIG.keys(): + assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) + + self.database = str(config["database"]) + + if config["reset"] and os.path.exists(self.database): + logging.debug("Deleting database '%s'" % self.database) + os.unlink(self.database) + + if os.path.exists(self.database) == False: + logging.debug("Loading DDL file '%s'" % (self.ddl)) + ## HACK + cmd = "sqlite3 %s < %s" % (self.database, self.ddl) + (result, output) = commands.getstatusoutput(cmd) + assert result == 0, cmd + "\n" + output + ## IF + + self.conn = sqlite3.connect(self.database) + self.cursor = self.conn.cursor() + + ## ---------------------------------------------- + ## loadTuples + ## ---------------------------------------------- + def loadTuples(self, tableName, tuples): + if len(tuples) == 0: return + + p = ["?"]*len(tuples[0]) + sql = "INSERT INTO %s VALUES (%s)" % (tableName, ",".join(p)) + self.cursor.executemany(sql, tuples) + + logging.debug("Loaded %d tuples for tableName %s" % (len(tuples), tableName)) + return + + ## ---------------------------------------------- + ## loadFinish + ## ---------------------------------------------- + def loadFinish(self): + logging.info("Commiting changes to database") + self.conn.commit() + + ## ---------------------------------------------- + ## doDelivery + ## ---------------------------------------------- + def doDelivery(self, params): + q = TXN_QUERIES["DELIVERY"] + + w_id = params["w_id"] + o_carrier_id = params["o_carrier_id"] + ol_delivery_d = params["ol_delivery_d"] + + result = [ ] + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): + self.cursor.execute(q["getNewOrder"], [d_id, w_id]) + newOrder = self.cursor.fetchone() + if newOrder == None: + ## No orders for this district: skip it. Note: This must be reported if > 1% + continue + assert len(newOrder) > 0 + no_o_id = newOrder[0] + + self.cursor.execute(q["getCId"], [no_o_id, d_id, w_id]) + c_id = self.cursor.fetchone()[0] + + self.cursor.execute(q["sumOLAmount"], [no_o_id, d_id, w_id]) + ol_total = self.cursor.fetchone()[0] + + self.cursor.execute(q["deleteNewOrder"], [d_id, w_id, no_o_id]) + self.cursor.execute(q["updateOrders"], [o_carrier_id, no_o_id, d_id, w_id]) + self.cursor.execute(q["updateOrderLine"], [ol_delivery_d, no_o_id, d_id, w_id]) + + # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) + # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure + # them out + # If there are no order lines, SUM returns null. There should always be order lines. + assert ol_total != None, "ol_total is NULL: there are no order lines. This should not happen" + assert ol_total > 0.0 + + self.cursor.execute(q["updateCustomer"], [ol_total, c_id, d_id, w_id]) + + result.append((d_id, no_o_id)) + ## FOR + + self.conn.commit() + return result + + ## ---------------------------------------------- + ## doNewOrder + ## ---------------------------------------------- + def doNewOrder(self, params): + q = TXN_QUERIES["NEW_ORDER"] + + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + o_entry_d = params["o_entry_d"] + i_ids = params["i_ids"] + i_w_ids = params["i_w_ids"] + i_qtys = params["i_qtys"] + + assert len(i_ids) > 0 + assert len(i_ids) == len(i_w_ids) + assert len(i_ids) == len(i_qtys) + + all_local = True + items = [ ] + for i in range(len(i_ids)): + ## Determine if this is an all local order or not + all_local = all_local and i_w_ids[i] == w_id + self.cursor.execute(q["getItemInfo"], [i_ids[i]]) + items.append(self.cursor.fetchone()) + assert len(items) == len(i_ids) + + ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. + ## Note that this will happen with 1% of transactions on purpose. + for item in items: + if len(item) == 0: + ## TODO Abort here! + return + ## FOR + + ## ---------------- + ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER + ## ---------------- + self.cursor.execute(q["getWarehouseTaxRate"], [w_id]) + w_tax = self.cursor.fetchone()[0] + + self.cursor.execute(q["getDistrict"], [d_id, w_id]) + district_info = self.cursor.fetchone() + d_tax = district_info[0] + d_next_o_id = district_info[1] + + self.cursor.execute(q["getCustomer"], [w_id, d_id, c_id]) + customer_info = self.cursor.fetchone() + c_discount = customer_info[0] + + ## ---------------- + ## Insert Order Information + ## ---------------- + ol_cnt = len(i_ids) + o_carrier_id = constants.NULL_CARRIER_ID + + self.cursor.execute(q["incrementNextOrderId"], [d_next_o_id + 1, d_id, w_id]) + self.cursor.execute(q["createOrder"], [d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, ol_cnt, all_local]) + self.cursor.execute(q["createNewOrder"], [d_next_o_id, d_id, w_id]) + + ## ---------------- + ## Insert Order Item Information + ## ---------------- + item_data = [ ] + total = 0 + for i in range(len(i_ids)): + ol_number = i + 1 + ol_supply_w_id = i_w_ids[i] + ol_i_id = i_ids[i] + ol_quantity = i_qtys[i] + + itemInfo = items[i] + i_name = itemInfo[1] + i_data = itemInfo[2] + i_price = itemInfo[0] + + self.cursor.execute(q["getStockInfo"] % (d_id), [ol_i_id, ol_supply_w_id]) + stockInfo = self.cursor.fetchone() + if len(stockInfo) == 0: + logging.warn("No STOCK record for (ol_i_id=%d, ol_supply_w_id=%d)" % (ol_i_id, ol_supply_w_id)) + continue + s_quantity = stockInfo[0] + s_ytd = stockInfo[2] + s_order_cnt = stockInfo[3] + s_remote_cnt = stockInfo[4] + s_data = stockInfo[1] + s_dist_xx = stockInfo[5] # Fetches data from the s_dist_[d_id] column + + ## Update stock + s_ytd += ol_quantity + if s_quantity >= ol_quantity + 10: + s_quantity = s_quantity - ol_quantity + else: + s_quantity = s_quantity + 91 - ol_quantity + s_order_cnt += 1 + + if ol_supply_w_id != w_id: s_remote_cnt += 1 + + self.cursor.execute(q["updateStock"], [s_quantity, s_ytd, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id]) + + if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: + brand_generic = 'B' + else: + brand_generic = 'G' + + ## Transaction profile states to use "ol_quantity * i_price" + ol_amount = ol_quantity * i_price + total += ol_amount + + self.cursor.execute(q["createOrderLine"], [d_next_o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, o_entry_d, ol_quantity, ol_amount, s_dist_xx]) + + ## Add the info to be returned + item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) + ## FOR + + ## Commit! + self.conn.commit() + + ## Adjust the total for the discount + #print "c_discount:", c_discount, type(c_discount) + #print "w_tax:", w_tax, type(w_tax) + #print "d_tax:", d_tax, type(d_tax) + total *= (1 - c_discount) * (1 + w_tax + d_tax) + + ## Pack up values the client is missing (see TPC-C 2.4.3.5) + misc = [ (w_tax, d_tax, d_next_o_id, total) ] + + return [ customer_info, misc, item_data ] + + ## ---------------------------------------------- + ## doOrderStatus + ## ---------------------------------------------- + def doOrderStatus(self, params): + q = TXN_QUERIES["ORDER_STATUS"] + + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + + assert w_id, pformat(params) + assert d_id, pformat(params) + + if c_id != None: + self.cursor.execute(q["getCustomerByCustomerId"], [w_id, d_id, c_id]) + customer = self.cursor.fetchone() + else: + # Get the midpoint customer's id + self.cursor.execute(q["getCustomersByLastName"], [w_id, d_id, c_last]) + all_customers = self.cursor.fetchall() + assert len(all_customers) > 0 + namecnt = len(all_customers) + index = (namecnt-1)/2 + customer = all_customers[index] + c_id = customer[0] + assert len(customer) > 0 + assert c_id != None + + self.cursor.execute(q["getLastOrder"], [w_id, d_id, c_id]) + order = self.cursor.fetchone() + if order: + self.cursor.execute(q["getOrderLines"], [w_id, d_id, order[0]]) + orderLines = self.cursor.fetchall() + else: + orderLines = [ ] + + self.conn.commit() + return [ customer, order, orderLines ] + + ## ---------------------------------------------- + ## doPayment + ## ---------------------------------------------- + def doPayment(self, params): + q = TXN_QUERIES["PAYMENT"] + + w_id = params["w_id"] + d_id = params["d_id"] + h_amount = params["h_amount"] + c_w_id = params["c_w_id"] + c_d_id = params["c_d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + h_date = params["h_date"] + + if c_id != None: + self.cursor.execute(q["getCustomerByCustomerId"], [w_id, d_id, c_id]) + customer = self.cursor.fetchone() + else: + # Get the midpoint customer's id + self.cursor.execute(q["getCustomersByLastName"], [w_id, d_id, c_last]) + all_customers = self.cursor.fetchall() + assert len(all_customers) > 0 + namecnt = len(all_customers) + index = (namecnt-1)/2 + customer = all_customers[index] + c_id = customer[0] + assert len(customer) > 0 + c_balance = customer[14] - h_amount + c_ytd_payment = customer[15] + h_amount + c_payment_cnt = customer[16] + 1 + c_data = customer[17] + + self.cursor.execute(q["getWarehouse"], [w_id]) + warehouse = self.cursor.fetchone() + + self.cursor.execute(q["getDistrict"], [w_id, d_id]) + district = self.cursor.fetchone() + + self.cursor.execute(q["updateWarehouseBalance"], [h_amount, w_id]) + self.cursor.execute(q["updateDistrictBalance"], [h_amount, w_id, d_id]) + + # Customer Credit Information + if customer[11] == constants.BAD_CREDIT: + newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) + c_data = (newData + "|" + c_data) + if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] + self.cursor.execute(q["updateBCCustomer"], [c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id]) + else: + c_data = "" + self.cursor.execute(q["updateGCCustomer"], [c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id]) + + # Concatenate w_name, four spaces, d_name + h_data = "%s %s" % (warehouse[0], district[0]) + # Create the history record + self.cursor.execute(q["insertHistory"], [c_id, c_d_id, c_w_id, d_id, w_id, h_date, h_amount, h_data]) + + self.conn.commit() + + # TPC-C 2.5.3.3: Must display the following fields: + # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, + # D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, + # C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, + # C_DISCOUNT, C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = "BC"), + # H_AMOUNT, and H_DATE. + + # Hand back all the warehouse, district, and customer data + return [ warehouse, district, customer ] + + ## ---------------------------------------------- + ## doStockLevel + ## ---------------------------------------------- + def doStockLevel(self, params): + q = TXN_QUERIES["STOCK_LEVEL"] + + w_id = params["w_id"] + d_id = params["d_id"] + threshold = params["threshold"] + + self.cursor.execute(q["getOId"], [w_id, d_id]) + result = self.cursor.fetchone() + assert result + o_id = result[0] + + self.cursor.execute(q["getStockCount"], [w_id, d_id, o_id, (o_id - 20), w_id, threshold]) + result = self.cursor.fetchone() + + self.conn.commit() + + return int(result[0]) + +## CLASS \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/tokyocabinetdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/tokyocabinetdriver.py new file mode 100644 index 00000000..a561dbf7 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/tokyocabinetdriver.py @@ -0,0 +1,1061 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Marcelo Martins +# http://www.cs.brown.edu/~martins/ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +from __future__ import with_statement +from abstractdriver import * +from pprint import pprint, pformat +from pyrant import protocol + +import constants +import logging +import os +import pyrant +import sys + +TABLE_COLUMNS = { + constants.TABLENAME_ITEM: [ + "I_ID", # INTEGER + "I_IM_ID", # INTEGER + "I_NAME", # VARCHAR + "I_PRICE", # FLOAT + "I_DATA", # VARCHAR + ], + constants.TABLENAME_WAREHOUSE: [ + "W_ID", # SMALLINT + "W_NAME", # VARCHAR + "W_STREET_1", # VARCHAR + "W_STREEET_2", # VARCHAR + "W_CITY", # VARCHAR + "W_STATE", # VARCHAR + "W_ZIP", # VARCHAR + "W_TAX", # FLOAT + "W_YTD", # FLOAT + ], + constants.TABLENAME_DISTRICT: [ + "D_ID", # TINYINT + "D_W_ID", # SMALLINT + "D_NAME", # VARCHAR + "D_STREET_1", # VARCHAR + "D_STREET_2", # VARCHAR + "D_CITY", # VARCHAR + "D_STATE", # VARCHAR + "D_ZIP", # VARCHAR + "D_TAX", # FLOAT + "D_YTD", # FLOAT + "D_NEXT_O_ID", # INT + ], + constants.TABLENAME_CUSTOMER: [ + "C_ID", # INTEGER + "C_D_ID", # TINYINT + "C_W_ID", # SMALLINT + "C_FIRST", # VARCHAR + "C_MIDDLE", # VARCHAR + "C_LAST", # VARCHAR + "C_STREET_1", # VARCHAR + "C_STREET_2", # VARCHAR + "C_CITY", # VARCHAR + "C_STATE", # VARCHAR + "C_ZIP", # VARCHAR + "C_PHONE", # VARCHAR + "C_SINCE", # TIMESTAMP + "C_CREDIT", # VARCHAR + "C_CREDIT_LIM", # FLOAT + "C_DISCOUNT", # FLOAT + "C_BALANCE", # FLOAT + "C_YTD_PAYMENT", # FLOAT + "C_PAYMENT_CNT", # INTEGER + "C_DELIVERY_CNT", # INTEGER + "C_DATA", # VARCHAR + ], + constants.TABLENAME_STOCK: [ + "S_I_ID", # INTEGER + "S_W_ID", # SMALLINT + "S_QUANTITY", # INTEGER + "S_DIST_01", # VARCHAR + "S_DIST_02", # VARCHAR + "S_DIST_03", # VARCHAR + "S_DIST_04", # VARCHAR + "S_DIST_05", # VARCHAR + "S_DIST_06", # VARCHAR + "S_DIST_07", # VARCHAR + "S_DIST_08", # VARCHAR + "S_DIST_09", # VARCHAR + "S_DIST_10", # VARCHAR + "S_YTD", # INTEGER + "S_ORDER_CNT", # INTEGER + "S_REMOTE_CNT", # INTEGER + "S_DATA", # VARCHAR + ], + constants.TABLENAME_ORDERS: [ + "O_ID", # INTEGER + "O_C_ID", # INTEGER + "O_D_ID", # TINYINT + "O_W_ID", # SMALLINT + "O_ENTRY_ID", # TIMESTAMP + "O_CARRIER_ID", # INTEGER + "O_OL_CNT", # INTEGER + "O_ALL_LOCAL", # INTEGER + ], + constants.TABLENAME_NEW_ORDER: [ + "NO_O_ID", # INTEGER + "NO_D_ID", # TINYINT + "NO_W_ID", # SMALLINT + ], + constants.TABLENAME_ORDER_LINE: [ + "OL_O_ID", # INTEGER + "OL_D_ID", # TINYINT + "OL_W_ID", # SMALLINT + "OL_NUMBER", # INTEGER + "OL_I_ID", # INTEGER + "OL_SUPPLY_W_ID", # SMALLINT + "OL_DELIVERY_D", # TIMESTAMP + "OL_QUANTITY", # INTEGER + "OL_AMOUNT", # FLOAT + "OL_DIST_INFO", # VARCHAR + ], + constants.TABLENAME_HISTORY: [ + "H_C_ID", # INTEGER + "H_C_D_ID", # TINYINT + "H_C_W_ID", # SMALLINT + "H_D_ID", # TINYINT + "H_W_ID", # SMALLINT + "H_DATA", # TIMESTAMP + "H_AMOUNT", # FLOAT + "H_DATA", # VARCHAR + ], +} +TABLE_INDEXES = { + constants.TABLENAME_ITEM: [ + "I_ID", + ], + constants.TABLENAME_WAREHOUSE: [ + "W_ID", + ], + constants.TABLENAME_DISTRICT: [ + "D_ID", + "D_W_ID", + ], + constants.TABLENAME_CUSTOMER: [ + "C_ID", + "C_D_ID", + "C_W_ID", + ], + constants.TABLENAME_STOCK: [ + "S_I_ID", + "S_W_ID", + ], + constants.TABLENAME_ORDERS: [ + "O_ID", + "O_D_ID", + "O_W_ID", + "O_C_ID", + ], + constants.TABLENAME_NEW_ORDER: [ + "NO_O_ID", + "NO_D_ID", + "NO_W_ID", + ], + constants.TABLENAME_ORDER_LINE: [ + "OL_O_ID", + "OL_D_ID", + "OL_W_ID", + ], +} + +## ============================================== +## TokyocabinetDriver +## ============================================== +class TokyocabinetDriver(AbstractDriver): + + ## Tokyo Tyrant provides one connection per *table*, not per database. + + ## Config files have no hierarchy. Let's set up our own hierarchy as a + ## stringfied dictionary and evaluate it later. + + ## Each table connection defines the host and port location of the Tokyo + ## Tyrant server + + DEFAULT_CONFIG = { "servers": ("Tokyo Tyrant server configuration", '{ 0: { "' + + constants.TABLENAME_ITEM + '":{ "host": "localhost", "port": 1978, },"' + + constants.TABLENAME_WAREHOUSE + '" : { "host": "localhost", "port": 1979, },"' + + constants.TABLENAME_DISTRICT + '" : { "host": "localhost", "port": 1980, },"' + + constants.TABLENAME_CUSTOMER + '" : { "host": "localhost", "port": 1981, },"' + + constants.TABLENAME_STOCK + '" : { "host": "localhost", "port": 1982, },"' + + constants.TABLENAME_ORDERS + '" : { "host": "localhost", "port": 1983, },"' + + constants.TABLENAME_NEW_ORDER + '" : { "host": "localhost", "port": 1984, },"' + + constants.TABLENAME_ORDER_LINE+ '" : { "host": "localhost", "port": 1985, },"' + + constants.TABLENAME_HISTORY + '" : { "host": "localhost", "port": 1986, }, }, }' ), } + + def __init__(self, ddl): + super(TokyocabinetDriver, self).__init__("tokyocabinet", ddl) + self.databases = dict() + self.conn = dict() + self.numServers = 0 + + ##----------------------------------------------- + ## self.tupleToString + ##----------------------------------------------- + def tupleToString(self, tuple, sep=":"): + """Tokyo-Cabinet table-type databases only accept strings as keys. + This function transforms a compound key (tuple) into a string. + Tuples elements are separated by the sep char""" + return sep.join(str(t) for t in tuple) + + ##----------------------------------------------- + ## self.getServer + ##----------------------------------------------- + def getServer(self, warehouseID): + """Tokyo Cabinet does not support data partitioning. For the TPC-C + benchmark, we manually partition data based on the warehouse ID""" + return (warehouseID % self.numServers) + + ## ---------------------------------------------- + ## makeDefaultConfig + ## ---------------------------------------------- + def makeDefaultConfig(self): + return TokyocabinetDriver.DEFAULT_CONFIG + + ## ---------------------------------------------- + ## loadConfig + ## ---------------------------------------------- + def loadConfig(self, config): + for key in TokyocabinetDriver.DEFAULT_CONFIG.keys(): + assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) + + if config["servers"]: + # Whn reading from INI file, we need to convert the server + # description-object from string to a real dictionary + config["servers"] = eval(config["servers"]) + for serverId, tables in config["servers"].iteritems(): + self.databases[serverId] = tables + + # First connect to databases + for serverId, tables in self.databases.iteritems(): + self.conn[serverId] = dict() + for tab, values in tables.iteritems(): + self.conn[serverId][tab] = pyrant.Tyrant(values["host"], values["port"]) + ## FOR + + # Remove previous data + if config["reset"]: + for serverId, tables in self.conn.iteritems(): + for tab in tables.keys(): + logging.debug("Deleting database '%s' at server '%s'" % (tab, serverId)) + self.conn[serverId][tab].clear() + ## FOR + ## FOR + ## IF + + self.numServers = len(self.databases.keys()) + logging.info("Number of servers: %s" % self.numServers) + + ## ------------------------------------------- + ## loadTuples + ## ------------------------------------------- + def loadTuples(self, tableName, tuples): + """Load tuples into tables of database + Each table is a connection to a Tyrant server. Each record is a key-value pair, + where key = primary key, values = concatenation of columns (dictionary). If + key is compound we transform it into a string, since TC does not support + compound keys. Data partitioning occurs based on Warehouse ID.""" + + if len(tuples) == 0: return + + logging.debug("Loading %d tuples of tableName %s" % (len(tuples), tableName)) + + assert tableName in TABLE_COLUMNS, "Unexpected table %s" % tableName + columns = TABLE_COLUMNS[tableName] + num_columns = xrange(len(columns)) + records = list() + + if tableName == constants.TABLENAME_WAREHOUSE: + for t in tuples: + w_key = t[0] # W_ID + sID = self.getServer(w_key) + cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) + records.append((str(w_key), cols)) + ## FOR + + try: + self.conn[sID][tableName].multi_set(records) + except KeyError, err: + sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) + sys.exit(1) + + elif tableName == constants.TABLENAME_DISTRICT: + for t in tuples: + w_key = t[1] # W_ID + sID = self.getServer(w_key) + d_key = self.tupleToString(t[:2]) # D_ID, D_W_ID + cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) + records.append((d_key, cols)) + ## FOR + + try: + self.conn[sID][tableName].multi_set(records) + except KeyError, err: + sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) + sys.exit(1) + + ## Item table doesn't have a w_id for partition. Replicate it to all + ## servers + elif tableName == constants.TABLENAME_ITEM: + for t in tuples: + i_key = str(t[0]) + cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) + records.append((i_key, cols)) + ## FOR + + for i in xrange(self.numServers): + try: + self.conn[i][tableName].multi_set(records) + except KeyError, err: + sys.stderr.write("%s(%s): server ID doesn't exist or is offline\n" %(KeyError, err)) + sys.exit(1) + ## FOR + ## FOR + + elif tableName == constants.TABLENAME_CUSTOMER: + for t in tuples: + w_key = t[2] # W_ID + sID = self.getServer(w_key) + c_key = self.tupleToString(t[:3]) # C_ID, C_D_ID, C_W_ID + cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) + records.append((c_key, cols)) + ## FOR + + try: + self.conn[sID][tableName].multi_set(records) + except KeyError, err: + sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) + sys.exit(1) + + elif tableName == constants.TABLENAME_HISTORY: + for t in tuples: + w_key = t[4] # W_ID + # Not really a primary key, but we need to generate + # something tobe our key + h_key = self.tupleToString(t[:3]) # H_C_ID, H_C_D, H_C_W + sID = self.getServer(w_key) + cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) + records.append((h_key, cols)) + ## FOR + + try: + self.conn[sID][tableName].multi_set(records) + except KeyError, err: + sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) + sys.exit(1) + + elif tableName == constants.TABLENAME_STOCK: + for t in tuples: + w_key = t[1] # W_ID + sID = self.getServer(w_key) + s_key = self.tupleToString(t[:2]) # S_ID, S_W_ID + cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) + records.append((s_key, cols)) + ## FOR + + try: + self.conn[sID][tableName].multi_set(records) + except KeyError, err: + sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) + sys.exit(1) + + elif tableName == constants.TABLENAME_ORDERS: + for t in tuples: + w_key = t[3] # W_ID + sID = self.getServer(w_key) + o_key = self.tupleToString(t[1:4]) # O_ID, O_D_ID, O_W_ID + cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) + records.append((o_key, cols)) + ## FOR + + try: + self.conn[sID][tableName].multi_set(records) + except KeyError, err: + sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) + sys.exit(1) + + elif tableName == constants.TABLENAME_NEW_ORDER: + for t in tuples: + w_key = t[2] # W_ID + sID = self.getServer(w_key) + no_key = self.tupleToString(t[:3]) # NO_O_ID, NO_D_ID, NO_W_ID + cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) + records.append((no_key, cols)) + ## FOR + + try: + self.conn[sID][tableName].multi_set(records) + except KeyError, err: + sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) + sys.exit(1) + + elif tableName == constants.TABLENAME_ORDER_LINE: + for t in tuples: + w_key = t[2] # W_ID + sID = self.getServer(w_key) + ol_key = self.tupleToString(t[:4]) # OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER + cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) + records.append((ol_key, cols)) + ## FOR + + try: + self.conn[sID][tableName].multi_set(records) + except KeyError, err: + sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) + sys.exit(1) + + logging.debug("Loaded %s tuples for tableName %s" % (len(tuples), tableName)) + return + + ## ------------------------------------------- + ## loadFinish + ## ------------------------------------------- + def loadFinish(self): + + conn = dict() + + logging.info("Creating indexes...") + # Add indexes to database after loading all data + for serverId, tables in self.databases.iteritems(): + conn[serverId] = dict() + for tab, connValues in tables.iteritems(): + conn[serverId][tab] = protocol.TyrantProtocol(connValues["host"], connValues["port"]) + for index_name in TABLE_COLUMNS[tab]: + conn[serverId][tab].add_index(index_name) + ## FOR + ## FOR + + logging.info("Optimizing indexes...") + # Optimize indexes for faster access + for serverId, tables in self.databases.iteritems(): + for tab, connValues in tables.iteritems(): + for index_name in TABLE_COLUMNS[tab]: + conn[serverId][tab].optimize_index(index_name) + ## FOR + ## FOR + + logging.info("Syncing to disk...") + # Finally, flush everything to disk + for tab in TABLE_COLUMNS.keys(): + for sID in self.conn.keys(): + self.conn[sID][tab].sync() + ## FOR + + logging.info("Finished loading tables") + + ## -------------------------------------------- + ## doDelivery + ## -------------------------------------------- + def doDelivery(self, params): + """Execute DELIVERY Transaction + Parameters Dict: + w_id + o_carrier_id + ol_delivery_id + """ + + w_id = params["w_id"] + o_carrier_id = params["o_carrier_id"] + ol_delivery_d = params["ol_delivery_d"] + + sID = self.getServer(w_id) + + newOrderQuery = self.conn[sID][constants.TABLENAME_NEW_ORDER].query + ordersQuery = self.conn[sID][constants.TABLENAME_ORDERS].query + orderLineQuery = self.conn[sID][constants.TABLENAME_ORDER_LINE].query + customerQuery = self.conn[sID][constants.TABLENAME_CUSTOMER].query + + results = [ ] + for d_id in xrange(1, constants.DISTRICTS_PER_WAREHOUSE+1): + + # getNewOrder + # SELECT NO_O_ID FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID > -1 LIMIT 1 + + newOrders = newOrderQuery.filter(NO_D_ID = d_id, NO_W_ID = w_id, NO_O_ID__gt = -1).columns("NO_O_ID") + if len(newOrders) == 0: + ## No orders for this district: skip it. Note: This must + ## reported if > 1% + continue + assert(newOrders) > 0 + no_o_id = int(newOrders[0]["NO_O_ID"]) + + # sumOLAmount + # SELECT SUM(OL_AMOUNT) FROM ORDER_LINE WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ? + + olines = orderLineQuery.filter(OL_O_ID = no_o_id, OL_D_ID = d_id, OL_W_ID = w_id).columns("OL_AMOUNT") + + # These must be logged in the "result file" according to TPC-C + # 2.7.22 (page 39) + # We remove the queued time, completed time, w_id, and + # o_carrier_id: the client can figure them out + # If there are no order lines, SUM returns null. There should + # always be order lines. + assert len(olines) > 0, "ol_total is NULL: there are no order lines. This should not happen" + + ol_total = sum(float(i["OL_AMOUNT"]) for i in olines) + + assert ol_total > 0.0 + + # deleteNewOrder + # DELETE FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID = ? + + orders = newOrderQuery.filter(NO_D_ID = d_id, NO_W_ID = w_id, NO_O_ID = no_o_id) + orders.delete(quick=True) + + # HACK: not transactionally safe + # getCId + # SELECT O_C_ID FROM ORDERS WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ? + + orders = ordersQuery.filter(O_ID = no_o_id, O_D_ID = d_id, O_W_ID = w_id) + assert len(orders) > 0 + c_id = int(orders.columns("O_C_ID")[0]["O_C_ID"]) + + # updateOrders + # UPDATE ORDERS SET O_CARRIER_ID = ? WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ? + + records = list() + for record in orders: + key, cols = record + cols["O_CARRIER_ID"] = o_carrier_id + records.append((key, cols)) + ## FOR + + self.conn[sID][constants.TABLENAME_ORDERS].multi_set(records) + + # updateOrderLine + # UPDATE ORDER_LINE SET OL_DELIVERY_D = ? WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ? + + orders = orderLineQuery.filter(OL_O_ID = no_o_id, OL_D_ID = d_id, OL_W_ID = w_id) + records = list() + for record in orders: + key, cols = record + cols["OL_DELIVERY_D"] = ol_delivery_d + records.append((key, cols)) + ## FOR + + self.conn[sID][constants.TABLENAME_ORDER_LINE].multi_set(records) + + # updateCustomer + # UPDATE CUSTOMER SET C_BALANCE = C_BALANCE + ? WHERE C_ID = ? AND C_D_ID = ? AND C_W_ID = ? + + customers = customerQuery.filter(C_ID = c_id, C_D_ID = d_id, C_W_ID = w_id) + records = list() + for record in customers: + key, cols = record + cols["C_BALANCE"] = float(cols["C_BALANCE"]) + ol_total + records.append((key, cols)) + ## FOR + + self.conn[sID][constants.TABLENAME_CUSTOMER].multi_add(records) + + results.append((d_id, no_o_id)) + ## FOR + + return results + + def doNewOrder(self, params): + """Execute NEW_ORDER Transaction + Parameters Dict: + w_id + d_id + c_id + o_entry_id + i_ids + i_w_ids + i_qtys + """ + + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + o_entry_d = params["o_entry_d"] + i_ids = params["i_ids"] + i_w_ids = params["i_w_ids"] + i_qtys = params["i_qtys"] + + assert len(i_ids) > 0 + assert len(i_ids) == len(i_w_ids) + assert len(i_ids) == len(i_qtys) + + sID = self.getServer(w_id) + + warehouseQuery = self.conn[sID][constants.TABLENAME_WAREHOUSE].query + districtQuery = self.conn[sID][constants.TABLENAME_DISTRICT].query + customerQuery = self.conn[sID][constants.TABLENAME_CUSTOMER].query + orderQuery = self.conn[sID][constants.TABLENAME_ORDERS].query + newOrderQuery = self.conn[sID][constants.TABLENAME_NEW_ORDER].query + itemQuery = self.conn[sID][constants.TABLENAME_ITEM].query + + all_local = True + items = [ ] + for i in xrange(len(i_ids)): + ## Determine if this is an all local order or not + all_local = all_local and i_w_ids[i] == w_id + # getItemInfo + retItems = itemQuery.filter(I_PRICE = i_ids[i]) + if len(items) > 0: + items.append(retItems.columns()) + else: + items.append([]) + assert len(items) == len(i_ids) + + ## TPCC define 1% of neworder gives a wrong itemid, causing rollback. + ## Note that this will happen with 1% of transactions on purpose. + for item in items: + if len(item) == 0: + ## TODO Abort here! + return + ## FOR + + ## ----------------- + ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER + ## ----------------- + + # getWarehouseTaxRate + # SELECT W_TAX FROM WAREHOUSE WHERE W_ID = ? + + taxes = warehouseQuery.filter(W_ID = w_id) + w_tax = float(taxes.columns("W_TAX")[0]["W_TAX"]) + + # getDistrict + # SELECT D_TAX, D_NEXT_O_ID FROM DISTRICT WHERE D_ID = ? AND D_W_ID = ? + + districts = districtQuery.filter(D_ID = d_id, D_W_ID = w_id) + districtInfo = districts.columns("D_TAX", "D_NEXT_O_ID")[0] + d_tax = float(districtInfo["D_TAX"]) + d_next_o_id = int(districtInfo["D_NEXT_O_ID"]) + + # incrementNextOrderId + # HACK: This is not transactionally safe! + # UPDATE DISTRICT SET D_NEXT_O_ID = ? WHERE D_ID = ? AND D_W_ID = ? + records = list() + for record in districts: + key, cols = record + cols["D_NEXT_O_ID"] = d_next_o_id + 1 + records.append((key, cols)) + ## FOR + + self.conn[sID][constants.TABLENAME_DISTRICT].multi_set(records) + + # getCustomer + # SELECT C_DISCOUNT, C_LAST, C_CREDIT FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? + + customers = customerQuery.filter(C_W_ID = w_id, C_D_ID = d_id, C_ID = c_id) + customerInfo = customers.columns("C_DISCOUNT", "C_LAST", "C_CREDIT")[0] + c_discount = float(customerInfo["C_DISCOUNT"]) + + ## ----------------- + ## Insert Order Information + ## ----------------- + ol_cnt = len(i_ids) + o_carrier_id = constants.NULL_CARRIER_ID + + # createOrder + # INSERT INTO ORDERS (O_ID, O_D_ID, O_W_ID, O_C_ID, O_ENTRY_D, O_CARRIER_ID, O_OL_CNT, + # O_ALL_LOCAL) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + + key = self.tupleToString((d_next_o_id, d_id, w_id)) + cols = {"O_ID": d_next_o_id, "O_D_ID": d_id, "O_W_ID": w_id, "O_C_ID": + c_id, "O_ENTRY_D": o_entry_d, "O_CARRIER_ID": + o_carrier_id, "O_OL_CNT": o_ol_cnt, "O_ALL_LOCAL": + all_local} + self.conn[sID][constants.TABLENAME_ORDERS].multi_set([(key, cols)]) + + # createNewOrder + # INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, NO_W_ID) VALUES (?, ?, ?) + + key = self.tupleToString((d_next_o_id, d_id, w_id)) + cols = {"NO_O_ID": d_next_o_id, "NO_D_ID": d_id, "NO_W_ID": w_id} + self.conn[sID][constants.TABLENAME_NEW_ORDER].multi_set([(key, cols)]) + + ## ------------------------------- + ## Insert Order Item Information + ## ------------------------------- + + item_data = [ ] + total = 0 + for i in xrange(len(i_id)): + ol_number = i+1 + ol_supply_w_id = i_w_ids[i] + ol_i_id = i_ids[i] + ol_quantity = i_qtys[i] + + # getItemInfo + # SELECT I_PRICE, I_NAME, I_DATA FROM ITEM WHERE I_ID = ? + + key, itemInfo = items[i] + i_price = float(itemInfo["I_PRICE"]) + i_name = itemInfo["I_NAME"] + i_data = itemInfo["I_DATA"] + + # getStockInfo + # SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DIST_%02d FROM STOCK + # WHERE S_I_ID = ? AND S_W_ID = ? + + stocks = stockQuery.filter(S_I_ID = ol_i_id, S_W_ID = ol_supply_w_id) + if len(stocks) == 0: + logging.warn("No STOCK record for (ol_i_id=%d, ol_supply_w_id=%d)" + % (ol_i_id, ol_supply_w_id)) + continue + stockInfo = stock.columns("S_QUANTITY", "S_DATA", "S_YTD", "S_ORDER_CNT", + "S_REMOTE_CNT", "S_DIST_%02d"%d_id)[0] + + s_quantity = int(stockInfo["S_QUANTITY"]) + s_ytd = float(stockInfo["S_YTD"]) + s_order_cnt = int(stockInfo["S_ORDER_CNT"]) + s_remote_cnt = int(stockInfo["S_REMOTE_CNT"]) + s_data = stockInfo["S_DATA"] + s_dist_xx = stockInfo["S_DIST_%02d"%d_id] # Fetches data from the + # s_dist_[d_id] column + + # updateStock + # UPDATE STOCK SET S_QUANTITY = ?, S_YTD = ?, S_ORDER_CNT = ?, S_REMOTE_CNT = ? + # WHERE S_I_ID = ? AND S_W_ID = ? + + s_ytd += ol_quantity + if s_quantity >= ol_quantity + 10: + s_quantity = s_quantity - ol_quantity + else: + s_quantity = s_quantity + 91 - ol_quantity + s_order_cnt += 1 + + if ol_supply_w_id != w_id: s_remote_cnt += 1 + + sSID = self.getServer(ol_supply_w_id) + stockQuery = self.conn[sID][constants.TABLENAME_STOCK].query + stocks = stockQuery.filter(S_I_ID = ol_i_id, S_W_ID = ol_supply_w_id) + records = list() + for record in stocks: + key, cols = record + cols["S_QUANTITY"] = s_quantity + cols["S_YTD"] = s_ytd + cols["S_ORDER_CNT"] = s_order_cnt + cols["S_REMOTE_CNT"] = s_remote_cnt + records.append((key, cols)) + ## FOR + + self.conn[sSID][constants.TABLENAME_STOCK].multi_set(records) + + if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: + brand_generic = 'B' + else: + brand_generic = 'G' + + ## Transaction profile states to use "ol_quantity * i_price" + ol_amount = ol_quantity * i_price + total += ol_amount + + # createOrderLine + # INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, + # OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + + key = tupletoString((d_next_o_id, d_id, w_id, ol_number)) + cols = {"OL_O_ID": d_next_o_id, "OL_D_ID": d_id, "OL_W_ID": w_id, + "OL_NUMBER": ol_number, "OL_I_ID": ol_i_id, + "OL_SUPPLY_W_ID": ol_supply_w_id, "OL_DELIVERY_D": + ol_entry_d, "OL_QUANTITY": ol_quantity, "OL_AMOUNT": + ol_amount, "OL_DIST_INFO": s_dist_xx} + self.conn[sID][constants.TABLENAME_ORDER_LINE].multi_set([(key, cols)]) + + ## Add the info to be returned + item_data.append((i_name, s_quantity, brand_generic, i_price, ol_amount)) + ## FOR + + ## Adjust the total for the discount + #print "c_discount:", c_discount, type(c_discount) + #print "w_tax:", w_tax, type(w_tax) + #print "d_tax:", d_tax, type(d_tax) + total *= (1 - c_discount) * (1 + w_tax + d_tax) + + ## Pack up values the client is missing (see TPC-C 2.4.3.5) + misc = [(w_tax, d_tax, d_next_o_id, total)] + + return [ customerInfo, misc, item_data ] + + def doOrderStatus(self, params): + """Execute ORDER_STATUS Transaction + Parameters Dict: + w_id + d_id + c_id + c_last + """ + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + + assert w_id, pformat(params) + assert d_id, pformat(params) + + sID = self.getServer(w_id) + + customerQuery = self.conn[sID][constants.TABLENAME_CUSTOMER].query + orderQuery = self.conn[sID][constants.TABLENAME_ORDERS].query + orderLineQuery= self.conn[sID][constants.TABLENAME_ORDER_LINE].query + + if c_id != None: + # getCustomerByCustomerId + # SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER + # WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? + + customers = customerQuery.filter(C_W_ID = w_id, C_D_ID = d_id, C_ID = c_id) + customerInfo = customers.columns("C_ID", "C_FIRST", "C_MIDDLE", "C_LAST", "C_BALANCE")[0] + else: + # Get the midpoint customer's id + # getCustomersByLastName + # SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER + # WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST + + customers = customerQuery.filter(C_W_ID = w_id, C_D_ID = d_id, C_LAST__contains = c_last).order_by("C_FIRST", numeric=False) + all_customers = customers.columns("C_ID", "C_FIRST", "C_MIDDLE", "C_LAST", "C_BALANCE") + namecnt = len(all_customers) + assert namecnt > 0 + index = (namecnt-1)/2 + customerInfo = all_customers[index] + c_id = int(customerInfo["C_ID"]) + assert len(customerInfo) > 0 + + # getLastOrder TODO: LIMIT 1 + # SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS + # WHERE O_W_ID = ? AND O_D_ID = ? AND O_C_ID = ? ORDER BY O_ID DESC LIMIT 1 + + orders = orderQuery.filter(O_W_ID = w_id, O_D_ID = d_id, O_C_ID = c_id).order_by("-O_ID", numeric=True) + orderInfo = orders.columns("O_ID", "O_CARRIER_ID", "O_ENTRY_D")[0] + + # getOrderLines + # SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE + # WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID = ? + if len(orders) > 0: + o_id = int(orderInfo["O_ID"]) + orders = orderLineQuery.filter(OL_W_ID = w_id, OL_D_ID = d_id, OL_O_ID = o_id) + orderLines = orders.columns("OL_SUPPLY_W_ID", "OL_I_ID", "OL_QUANTITY", "OL_AMOUNT", "OL_DELIVERY_D") + else: + orderLines = [ ] + + return [customerInfo, orderInfo, orderLines] + + def doPayment(self, params): + """Execute PAYMENT Transaction + Parameters Dict: + w_id + d_id + h_amount + c_w_id + c_d_id + c_id + c_last + h_date + """ + + w_id = params["w_id"] + d_id = params["d_id"] + h_amount = params["h_amount"] + c_w_id = params["c_w_id"] + c_d_id = params["c_d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + h_date = params["h_date"] + + sID = self.getServer(w_id) + cSID = self.getServer(c_w_id) + + customerQuery = self.conn[cSID][constants.TABLENAME_CUSTOMER].query + warehouseQuery = self.conn[sID][constants.TABLENAME_WAREHOUSE].query + districtQuery = self.conn[sID][constants.TABLENAME_DISTRICT].query + + if c_id != None: + # getCustomerByCustomerId + # SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, + # C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, + # C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER + # WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? + + customers = customerQuery.filter(C_W_ID = c_w_id, C_D_ID = d_id, C_ID = c_id) + customerInfo = customers.columns("C_ID", "C_BALANCE", "C_YTD_PAYMENT", "C_PAYMENT_CNT", "C_DATA")[0] + else: + # Get the midpoint customer's id + # getCustomersByLastName + # SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, + # C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, + # C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER + # WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST + + customers = customerQuery.filter(C_W_ID = c_w_id, C_D_ID = d_id, C_LAST__contains = c_last).order_by("C_FIRST", numeric=False) + all_customers = customers.columns("C_ID", "C_BALANCE", "C_YTD_PAYMENT", "C_PAYMENT_CNT", "C_DATA") + namecnt = len(all_customers) + assert namecnt > 0 + index = (namecnt-1)/2 + customerInfo = all_customers[index] + c_id = int(customerInfo["C_ID"]) + assert len(customerInfo) > 0 + + c_balance = float(customerInfo["C_BALANCE"]) - h_amount + c_ytd_payment = float(customerInfo["C_YTD_PAYMENT"]) + h_amount + c_payment_cnt = int(customerInfo["C_PAYMENT_CNT"]) + 1 + c_data = customerInfo["C_DATA"] + + # getWarehouse + # SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP FROM WAREHOUSE WHERE W_ID = ? + + warehouses = warehouseQuery.filter(W_ID = w_id) + warehouseInfo = warehouses.columns("W_NAME","W_STREET_1", "W_STREET_2", "W_CITY", "W_STATE", "W_ZIP")[0] + + # updateWarehouseBalance + # UPDATE WAREHOUSE SET W_YTD = W_YTD + ? WHERE W_ID = ? + + warehouses = warehouseQuery.filter(W_ID = w_id) + records = list() + for record in warehouses: + key, cols = record + cols["W_YTD"] = float(cols["W_YTD"]) + h_amount + records.append((key, cols)) + ## FOR + self.conn[sID][constants.TABLENAME_WAREHOUSE].multi_set(records) + + # getDistrict + # SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP FROM DISTRICT + # WHERE D_W_ID = ? AND D_ID = ? + + districts = districtQuery.filter(D_W_ID = w_id, D_ID = d_id) + districtInfo = districts.columns("D_NAME", "D_STREET_1", "D_STREET_2", "D_CITY", "D_STATE", "D_ZIP")[0] + + # updateDistrictBalance + # UPDATE DISTRICT SET D_YTD = D_YTD + ? WHERE D_W_ID = ? AND D_ID = ? + + districts = districtQuery.filter(W_ID = w_id, D_ID = d_id) + records = list() + for record in districts: + key, cols = record + cols["D_YTD"] = float(cols["D_YTD"]) + h_amount + records.append((key, cols)) + ## FOR + self.conn[sID][constants.TABLENAME_DISTRICT].multi_set(records) + + # Customer Credit Information + customers = customerQuery.filter(C_W_ID = c_w_id, C_D_ID = c_d_id, C_ID = c_id) + cInfo = customers.columns("C_CREDIT")[0] + + if cInfo["C_CREDIT"] == constants.BAD_CREDIT: + newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) + c_data = (newData + "|" + c_data) + if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] + + # updateBCCustomer + # UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ?, C_DATA = ? + # WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? + + records = list() + for record in customers: + key, cols = record + cols["C_BALANCE"] = c_balance + cols["C_YTD_PAYMENT"] = c_ytd_payment + cols["C_PAYMENT_CNT"] = c_payment_cnt + cols["C_DATA"] = c_data + records.append((key, cols)) + ## FOR + self.conn[cSID][constants.TABLENAME_CUSTOMER].multi_set(records) + else: + c_data = "" + + records = list() + # updateGCCustomer + # UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ? + # WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? + + for record in customers: + key, cols = record + cols["C_BALANCE"] = c_balance + cols["C_YTD_PAYMENT"] = c_ytd_payment + cols["C_PAYMENT_CNT"] = c_payment_cnt + records.append((key, cols)) + ## FOR + self.conn[cSID][constants.TABLENAME_CUSTOMER].multi_set(records) + + # Concatenate w_name, four space, d_name + h_data = "%s %s" % (warehouseInfo["W_NAME"], districtInfo["D_NAME"]) + + # Create the history record + # INSERT INTO HISTORY VALUES (?, ?, ?, ?, ?, ?, ?, ?) + + h_key = self.tupleToString((c_id, c_d_id, c_w_id)) + + cols = {"H_C_ID": c_id, "H_C_D_ID": c_d_id, "H_C_W_ID": c_w_id, "H_D_ID": + d_id, "H_W_ID": w_id, "H_DATE": h_date, "H_AMOUNT": + h_amount, "H_DATA": h_data} + self.conn[sID][constants.TABLENAME_HISTORY].multi_set([(key, cols)]) + + # TPC-C 2.5.3.3: Must display the following fields: + # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, + # W_STATE, W_ZIP, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, + # C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, + # C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, + # C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = + # "BC"), H_AMMOUNT, and H_DATE. + + # Hand back all the warehouse, district, and customer data + return [ warehouseInfo, districtInfo, customerInfo ] + + def doStockLevel(self, params): + """Execute STOCK_LEVEL Transaction + Parameters Dict: + w_id + d_id + threshold + """ + w_id = params["w_id"] + d_id = params["d_id"] + threshold = params["threshold"] + + sID = self.getServer(w_id) + + districtQuery = self.conn[sID][constants.TABLENAME_DISTRICT].query + orderLineQuery = self.conn[sID][constants.TABLENAME_ORDER_LINE].query + stockQuery = self.conn[sID][constants.TABLENAME_STOCK].query + + # getOId + # "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?" + + districts = districtQuery.filter(D_W_ID = w_id, D_ID = d_id) + o_id = int(districts.columns("D_NEXT_O_ID")[0]["D_NEXT_O_ID"]) + + # getStockCount + # SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK WHERE OL_W_ID = ? AND OL_D_ID = ? + # AND OL_O_ID < ? AND OL_O_ID >= ? AND S_W_ID = ? AND S_I_ID = OL_I_ID AND S_QUANTITY < ? + + orders = orderLineQuery.filter(OL_W_ID = w_id, OL_D_ID = d_id, OL_O_ID__between = [(o_id-20), (o_id-1)]) + ol_i_ids = set([i["OL_I_ID"] for i in orders.columns("OL_I_ID")]) + + stocks = stockQuery.filter(S_W_ID = w_id, S_I_ID__in = list(ol_i_ids), S_QUANTITY__lt = threshold).columns("S_W_ID") + + cnt = len(stocks) + + return cnt + +## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/message.py b/workloads/chbenchmark/py-tpcc/pytpcc/message.py new file mode 100644 index 00000000..78cca54b --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/message.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo & Yang Lu +# http:##www.cs.brown.edu/~pavlo/ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import sys +import os +import string +import datetime +import logging +import re +import argparse +import glob +import time +from pprint import pprint,pformat + +from util import * +from runtime import * +import drivers + +EMPTY = 0 +CMD_LOAD = 1 +CMD_EXECUTE = 2 +CMD_STOP = 3 +LOAD_COMPLETED = 4 +EXECUTE_COMPLETED = 5 + +class Message: + def __init__(self,header=EMPTY,data=None): + self.header=header + self.data=data diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/runtime/__init__.py b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/__init__.py new file mode 100644 index 00000000..0e480b16 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +__all__ = ["executor", "loader"] diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py new file mode 100644 index 00000000..4e048c21 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py @@ -0,0 +1,252 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Original Java Version: +# Copyright (C) 2008 +# Evan Jones +# Massachusetts Institute of Technology +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import sys +import multiprocessing +import time +import random +import traceback +import logging +from datetime import datetime +from pprint import pprint,pformat + +import constants +from util import * + + +class Executor: + + def __init__(self, driver, scaleParameters, stop_on_error = False): + self.driver = driver + self.scaleParameters = scaleParameters + self.stop_on_error = stop_on_error + ## DEF + + def execute(self, duration): + r = results.Results() + assert r + logging.info("Executing benchmark for %d seconds" % duration) + start = r.startBenchmark() + debug = logging.getLogger().isEnabledFor(logging.DEBUG) + + while (time.time() - start) <= duration: + txn, params = self.doOne() + txn_id = r.startTransaction(txn) + + if debug: logging.debug("Executing '%s' transaction" % txn) + try: + val = self.driver.executeTransaction(txn, params) + except KeyboardInterrupt: + return -1 + except (Exception, AssertionError), ex: + logging.warn("Failed to execute Transaction '%s': %s" % (txn, ex)) + if debug: traceback.print_exc(file=sys.stdout) + if self.stop_on_error: raise + r.abortTransaction(txn_id) + continue + + #if debug: logging.debug("%s\nParameters:\n%s\nResult:\n%s" % (txn, pformat(params), pformat(val))) + + r.stopTransaction(txn_id) + ## WHILE + + r.stopBenchmark() + return (r) + ## DEF + + def doOne(self): + """Selects and executes a transaction at random. The number of new order transactions executed per minute is the official "tpmC" metric. See TPC-C 5.4.2 (page 71).""" + + ## This is not strictly accurate: The requirement is for certain + ## *minimum* percentages to be maintained. This is close to the right + ## thing, but not precisely correct. See TPC-C 5.2.4 (page 68). + x = rand.number(1, 100) + params = None + txn = None + if x <= 4: ## 4% + txn, params = (constants.TransactionTypes.STOCK_LEVEL, self.generateStockLevelParams()) + elif x <= 4 + 4: ## 4% + txn, params = (constants.TransactionTypes.DELIVERY, self.generateDeliveryParams()) + elif x <= 4 + 4 + 4: ## 4% + txn, params = (constants.TransactionTypes.ORDER_STATUS, self.generateOrderStatusParams()) + elif x <= 43 + 4 + 4 + 4: ## 43% + txn, params = (constants.TransactionTypes.PAYMENT, self.generatePaymentParams()) + else: ## 45% + assert x > 100 - 45 + txn, params = (constants.TransactionTypes.NEW_ORDER, self.generateNewOrderParams()) + + return (txn, params) + ## DEF + + ## ---------------------------------------------- + ## generateDeliveryParams + ## ---------------------------------------------- + def generateDeliveryParams(self): + """Return parameters for DELIVERY""" + w_id = self.makeWarehouseId() + o_carrier_id = rand.number(constants.MIN_CARRIER_ID, constants.MAX_CARRIER_ID) + ol_delivery_d = datetime.now() + return makeParameterDict(locals(), "w_id", "o_carrier_id", "ol_delivery_d") + ## DEF + + ## ---------------------------------------------- + ## generateNewOrderParams + ## ---------------------------------------------- + def generateNewOrderParams(self): + """Return parameters for NEW_ORDER""" + w_id = self.makeWarehouseId() + d_id = self.makeDistrictId() + c_id = self.makeCustomerId() + ol_cnt = rand.number(constants.MIN_OL_CNT, constants.MAX_OL_CNT) + o_entry_d = datetime.now() + + ## 1% of transactions roll back + rollback = False # FIXME rand.number(1, 100) == 1 + + i_ids = [ ] + i_w_ids = [ ] + i_qtys = [ ] + for i in range(0, ol_cnt): + if rollback and i + 1 == ol_cnt: + i_ids.append(self.scaleParameters.items + 1) + else: + i_id = self.makeItemId() + while i_id in i_ids: + i_id = self.makeItemId() + i_ids.append(i_id) + + ## 1% of items are from a remote warehouse + remote = (rand.number(1, 100) == 1) + if self.scaleParameters.warehouses > 1 and remote: + i_w_ids.append(rand.numberExcluding(self.scaleParameters.starting_warehouse, self.scaleParameters.ending_warehouse, w_id)) + else: + i_w_ids.append(w_id) + + i_qtys.append(rand.number(1, constants.MAX_OL_QUANTITY)) + ## FOR + + return makeParameterDict(locals(), "w_id", "d_id", "c_id", "o_entry_d", "i_ids", "i_w_ids", "i_qtys") + ## DEF + + ## ---------------------------------------------- + ## generateOrderStatusParams + ## ---------------------------------------------- + def generateOrderStatusParams(self): + """Return parameters for ORDER_STATUS""" + w_id = self.makeWarehouseId() + d_id = self.makeDistrictId() + c_last = None + c_id = None + + ## 60%: order status by last name + if rand.number(1, 100) <= 60: + c_last = rand.makeRandomLastName(self.scaleParameters.customersPerDistrict) + + ## 40%: order status by id + else: + c_id = self.makeCustomerId() + + return makeParameterDict(locals(), "w_id", "d_id", "c_id", "c_last") + ## DEF + + ## ---------------------------------------------- + ## generatePaymentParams + ## ---------------------------------------------- + def generatePaymentParams(self): + """Return parameters for PAYMENT""" + x = rand.number(1, 100) + y = rand.number(1, 100) + + w_id = self.makeWarehouseId() + d_id = self.makeDistrictId() + c_w_id = None + c_d_id = None + c_id = None + c_last = None + h_amount = rand.fixedPoint(2, constants.MIN_PAYMENT, constants.MAX_PAYMENT) + h_date = datetime.now() + + ## 85%: paying through own warehouse (or there is only 1 warehouse) + if self.scaleParameters.warehouses == 1 or x <= 85: + c_w_id = w_id + c_d_id = d_id + ## 15%: paying through another warehouse: + else: + ## select in range [1, num_warehouses] excluding w_id + c_w_id = rand.numberExcluding(self.scaleParameters.starting_warehouse, self.scaleParameters.ending_warehouse, w_id) + assert c_w_id != w_id + c_d_id = self.makeDistrictId() + + ## 60%: payment by last name + if y <= 60: + c_last = rand.makeRandomLastName(self.scaleParameters.customersPerDistrict) + ## 40%: payment by id + else: + assert y > 60 + c_id = self.makeCustomerId() + + return makeParameterDict(locals(), "w_id", "d_id", "h_amount", "c_w_id", "c_d_id", "c_id", "c_last", "h_date") + ## DEF + + ## ---------------------------------------------- + ## generateStockLevelParams + ## ---------------------------------------------- + def generateStockLevelParams(self): + """Returns parameters for STOCK_LEVEL""" + w_id = self.makeWarehouseId() + d_id = self.makeDistrictId() + threshold = rand.number(constants.MIN_STOCK_LEVEL_THRESHOLD, constants.MAX_STOCK_LEVEL_THRESHOLD) + return makeParameterDict(locals(), "w_id", "d_id", "threshold") + ## DEF + + def makeWarehouseId(self): + w_id = rand.number(self.scaleParameters.starting_warehouse, self.scaleParameters.ending_warehouse) + assert(w_id >= self.scaleParameters.starting_warehouse), "Invalid W_ID: %d" % w_id + assert(w_id <= self.scaleParameters.ending_warehouse), "Invalid W_ID: %d" % w_id + return w_id + ## DEF + + def makeDistrictId(self): + return rand.number(1, self.scaleParameters.districtsPerWarehouse) + ## DEF + + def makeCustomerId(self): + return rand.NURand(1023, 1, self.scaleParameters.customersPerDistrict) + ## DEF + + def makeItemId(self): + return rand.NURand(8191, 1, self.scaleParameters.items) + ## DEF +## CLASS + +def makeParameterDict(values, *args): + return dict(map(lambda x: (x, values[x]), args)) +## DEF diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/runtime/loader.py b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/loader.py new file mode 100644 index 00000000..1c9225e1 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/loader.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http:##www.cs.brown.edu/~pavlo/ +# +# Original Java Version: +# Copyright (C) 2008 +# Evan Jones +# Massachusetts Institute of Technology +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import os +import sys + +import logging +from datetime import datetime +from random import shuffle +from pprint import pprint,pformat + +import constants +from util import * + +class Loader: + + def __init__(self, handle, scaleParameters, w_ids, needLoadItems): + self.handle = handle + self.scaleParameters = scaleParameters + self.w_ids = w_ids + self.needLoadItems = needLoadItems + self.batch_size = 2500 + + ## ============================================== + ## execute + ## ============================================== + def execute(self): + + ## Item Table + if self.needLoadItems: + logging.debug("Loading ITEM table") + self.loadItems() + self.handle.loadFinishItem() + + ## Then create the warehouse-specific tuples + for w_id in self.w_ids: + self.loadWarehouse(w_id) + self.handle.loadFinishWarehouse(w_id) + ## FOR + + return (None) + + ## ============================================== + ## loadItems + ## ============================================== + def loadItems(self): + ## Select 10% of the rows to be marked "original" + originalRows = rand.selectUniqueIds(self.scaleParameters.items / 10, 1, self.scaleParameters.items) + + ## Load all of the items + tuples = [ ] + total_tuples = 0 + for i in range(1, self.scaleParameters.items+1): + original = (i in originalRows) + tuples.append(self.generateItem(i, original)) + total_tuples += 1 + if len(tuples) == self.batch_size: + logging.debug("LOAD - %s: %5d / %d" % (constants.TABLENAME_ITEM, total_tuples, self.scaleParameters.items)) + self.handle.loadTuples(constants.TABLENAME_ITEM, tuples) + tuples = [ ] + ## FOR + if len(tuples) > 0: + logging.debug("LOAD - %s: %5d / %d" % (constants.TABLENAME_ITEM, total_tuples, self.scaleParameters.items)) + self.handle.loadTuples(constants.TABLENAME_ITEM, tuples) + ## DEF + + ## ============================================== + ## loadWarehouse + ## ============================================== + def loadWarehouse(self, w_id): + logging.debug("LOAD - %s: %d / %d" % (constants.TABLENAME_WAREHOUSE, w_id, len(self.w_ids))) + + ## WAREHOUSE + w_tuples = [ self.generateWarehouse(w_id) ] + self.handle.loadTuples(constants.TABLENAME_WAREHOUSE, w_tuples) + + ## DISTRICT + d_tuples = [ ] + for d_id in range(1, self.scaleParameters.districtsPerWarehouse+1): + d_next_o_id = self.scaleParameters.customersPerDistrict + 1 + d_tuples = [ self.generateDistrict(w_id, d_id, d_next_o_id) ] + + c_tuples = [ ] + h_tuples = [ ] + + ## Select 10% of the customers to have bad credit + selectedRows = rand.selectUniqueIds(self.scaleParameters.customersPerDistrict / 10, 1, self.scaleParameters.customersPerDistrict) + + ## TPC-C 4.3.3.1. says that o_c_id should be a permutation of [1, 3000]. But since it + ## is a c_id field, it seems to make sense to have it be a permutation of the + ## customers. For the "real" thing this will be equivalent + cIdPermutation = [ ] + + for c_id in range(1, self.scaleParameters.customersPerDistrict+1): + badCredit = (c_id in selectedRows) + c_tuples.append(self.generateCustomer(w_id, d_id, c_id, badCredit, True)) + h_tuples.append(self.generateHistory(w_id, d_id, c_id)) + cIdPermutation.append(c_id) + ## FOR + assert cIdPermutation[0] == 1 + assert cIdPermutation[self.scaleParameters.customersPerDistrict - 1] == self.scaleParameters.customersPerDistrict + shuffle(cIdPermutation) + + o_tuples = [ ] + ol_tuples = [ ] + no_tuples = [ ] + + for o_id in range(1, self.scaleParameters.customersPerDistrict+1): + o_ol_cnt = rand.number(constants.MIN_OL_CNT, constants.MAX_OL_CNT) + + ## The last newOrdersPerDistrict are new orders + newOrder = ((self.scaleParameters.customersPerDistrict - self.scaleParameters.newOrdersPerDistrict) < o_id) + o_tuples.append(self.generateOrder(w_id, d_id, o_id, cIdPermutation[o_id - 1], o_ol_cnt, newOrder)) + + ## Generate each OrderLine for the order + for ol_number in range(0, o_ol_cnt): + ol_tuples.append(self.generateOrderLine(w_id, d_id, o_id, ol_number, self.scaleParameters.items, newOrder)) + ## FOR + + ## This is a new order: make one for it + if newOrder: no_tuples.append([o_id, d_id, w_id]) + ## FOR + + self.handle.loadTuples(constants.TABLENAME_DISTRICT, d_tuples) + self.handle.loadTuples(constants.TABLENAME_CUSTOMER, c_tuples) + self.handle.loadTuples(constants.TABLENAME_ORDERS, o_tuples) + self.handle.loadTuples(constants.TABLENAME_ORDER_LINE, ol_tuples) + self.handle.loadTuples(constants.TABLENAME_NEW_ORDER, no_tuples) + self.handle.loadTuples(constants.TABLENAME_HISTORY, h_tuples) + self.handle.loadFinishDistrict(w_id, d_id) + ## FOR + + ## Select 10% of the stock to be marked "original" + s_tuples = [ ] + selectedRows = rand.selectUniqueIds(self.scaleParameters.items / 10, 1, self.scaleParameters.items) + total_tuples = 0 + for i_id in range(1, self.scaleParameters.items+1): + original = (i_id in selectedRows) + s_tuples.append(self.generateStock(w_id, i_id, original)) + if len(s_tuples) >= self.batch_size: + logging.debug("LOAD - %s [W_ID=%d]: %5d / %d" % (constants.TABLENAME_STOCK, w_id, total_tuples, self.scaleParameters.items)) + self.handle.loadTuples(constants.TABLENAME_STOCK, s_tuples) + s_tuples = [ ] + total_tuples += 1 + ## FOR + if len(s_tuples) > 0: + logging.debug("LOAD - %s [W_ID=%d]: %5d / %d" % (constants.TABLENAME_STOCK, w_id, total_tuples, self.scaleParameters.items)) + self.handle.loadTuples(constants.TABLENAME_STOCK, s_tuples) + ## DEF + + ## ============================================== + ## generateItem + ## ============================================== + def generateItem(self, id, original): + i_id = id + i_im_id = rand.number(constants.MIN_IM, constants.MAX_IM) + i_name = rand.astring(constants.MIN_I_NAME, constants.MAX_I_NAME) + i_price = rand.fixedPoint(constants.MONEY_DECIMALS, constants.MIN_PRICE, constants.MAX_PRICE) + i_data = rand.astring(constants.MIN_I_DATA, constants.MAX_I_DATA) + if original: i_data = self.fillOriginal(i_data) + + return [i_id, i_im_id, i_name, i_price, i_data] + ## DEF + + ## ============================================== + ## generateWarehouse + ## ============================================== + def generateWarehouse(self, w_id): + w_tax = self.generateTax() + w_ytd = constants.INITIAL_W_YTD + w_address = self.generateAddress() + return [w_id] + w_address + [w_tax, w_ytd] + ## DEF + + ## ============================================== + ## generateDistrict + ## ============================================== + def generateDistrict(self, d_w_id, d_id, d_next_o_id): + d_tax = self.generateTax() + d_ytd = constants.INITIAL_D_YTD + d_address = self.generateAddress() + return [d_id, d_w_id] + d_address + [d_tax, d_ytd, d_next_o_id] + ## DEF + + ## ============================================== + ## generateCustomer + ## ============================================== + def generateCustomer(self, c_w_id, c_d_id, c_id, badCredit, doesReplicateName): + c_first = rand.astring(constants.MIN_FIRST, constants.MAX_FIRST) + c_middle = constants.MIDDLE + + assert 1 <= c_id and c_id <= constants.CUSTOMERS_PER_DISTRICT + if c_id <= 1000: + c_last = rand.makeLastName(c_id - 1) + else: + c_last = rand.makeRandomLastName(constants.CUSTOMERS_PER_DISTRICT) + + c_phone = rand.nstring(constants.PHONE, constants.PHONE) + c_since = datetime.now() + c_credit = constants.BAD_CREDIT if badCredit else constants.GOOD_CREDIT + c_credit_lim = constants.INITIAL_CREDIT_LIM + c_discount = rand.fixedPoint(constants.DISCOUNT_DECIMALS, constants.MIN_DISCOUNT, constants.MAX_DISCOUNT) + c_balance = constants.INITIAL_BALANCE + c_ytd_payment = constants.INITIAL_YTD_PAYMENT + c_payment_cnt = constants.INITIAL_PAYMENT_CNT + c_delivery_cnt = constants.INITIAL_DELIVERY_CNT + c_data = rand.astring(constants.MIN_C_DATA, constants.MAX_C_DATA) + + c_street1 = rand.astring(constants.MIN_STREET, constants.MAX_STREET) + c_street2 = rand.astring(constants.MIN_STREET, constants.MAX_STREET) + c_city = rand.astring(constants.MIN_CITY, constants.MAX_CITY) + c_state = rand.astring(constants.STATE, constants.STATE) + c_zip = self.generateZip() + + return [ c_id, c_d_id, c_w_id, c_first, c_middle, c_last, \ + c_street1, c_street2, c_city, c_state, c_zip, \ + c_phone, c_since, c_credit, c_credit_lim, c_discount, c_balance, \ + c_ytd_payment, c_payment_cnt, c_delivery_cnt, c_data ] + ## DEF + + ## ============================================== + ## generateOrder + ## ============================================== + def generateOrder(self, o_w_id, o_d_id, o_id, o_c_id, o_ol_cnt, newOrder): + """Returns the generated o_ol_cnt value.""" + o_entry_d = datetime.now() + o_carrier_id = constants.NULL_CARRIER_ID if newOrder else rand.number(constants.MIN_CARRIER_ID, constants.MAX_CARRIER_ID) + o_all_local = constants.INITIAL_ALL_LOCAL + return [ o_id, o_c_id, o_d_id, o_w_id, o_entry_d, o_carrier_id, o_ol_cnt, o_all_local ] + ## DEF + + ## ============================================== + ## generateOrderLine + ## ============================================== + def generateOrderLine(self, ol_w_id, ol_d_id, ol_o_id, ol_number, max_items, newOrder): + ol_i_id = rand.number(1, max_items) + ol_supply_w_id = ol_w_id + ol_delivery_d = datetime.now() + ol_quantity = constants.INITIAL_QUANTITY + + ## 1% of items are from a remote warehouse + remote = (rand.number(1, 100) == 1) + if self.scaleParameters.warehouses > 1 and remote: + ol_supply_w_id = rand.numberExcluding(self.scaleParameters.starting_warehouse, + self.scaleParameters.ending_warehouse, + ol_w_id) + + if newOrder == False: + ol_amount = 0.00 + else: + ol_amount = rand.fixedPoint(constants.MONEY_DECIMALS, constants.MIN_AMOUNT, constants.MAX_PRICE * constants.MAX_OL_QUANTITY) + ol_delivery_d = None + ol_dist_info = rand.astring(constants.DIST, constants.DIST) + + return [ ol_o_id, ol_d_id, ol_w_id, ol_number, ol_i_id, ol_supply_w_id, ol_delivery_d, ol_quantity, ol_amount, ol_dist_info ] + ## DEF + + ## ============================================== + ## generateStock + ## ============================================== + def generateStock(self, s_w_id, s_i_id, original): + s_quantity = rand.number(constants.MIN_QUANTITY, constants.MAX_QUANTITY); + s_ytd = 0; + s_order_cnt = 0; + s_remote_cnt = 0; + + s_data = rand.astring(constants.MIN_I_DATA, constants.MAX_I_DATA); + if original: self.fillOriginal(s_data) + + s_dists = [ ] + for i in range(0, constants.DISTRICTS_PER_WAREHOUSE): + s_dists.append(rand.astring(constants.DIST, constants.DIST)) + + return [ s_i_id, s_w_id, s_quantity ] + \ + s_dists + \ + [ s_ytd, s_order_cnt, s_remote_cnt, s_data ] + ## DEF + + ## ============================================== + ## generateHistory + ## ============================================== + def generateHistory(self, h_c_w_id, h_c_d_id, h_c_id): + h_w_id = h_c_w_id + h_d_id = h_c_d_id + h_date = datetime.now() + h_amount = constants.INITIAL_AMOUNT + h_data = rand.astring(constants.MIN_DATA, constants.MAX_DATA) + return [ h_c_id, h_c_d_id, h_c_w_id, h_d_id, h_w_id, h_date, h_amount, h_data ] + ## DEF + + ## ============================================== + ## generateAddress + ## ============================================== + def generateAddress(self): + """ + Returns a name and a street address + Used by both generateWarehouse and generateDistrict. + """ + name = rand.astring(constants.MIN_NAME, constants.MAX_NAME) + return [ name ] + self.generateStreetAddress() + ## DEF + + ## ============================================== + ## generateStreetAddress + ## ============================================== + def generateStreetAddress(self): + """ + Returns a list for a street address + Used for warehouses, districts and customers. + """ + street1 = rand.astring(constants.MIN_STREET, constants.MAX_STREET) + street2 = rand.astring(constants.MIN_STREET, constants.MAX_STREET) + city = rand.astring(constants.MIN_CITY, constants.MAX_CITY) + state = rand.astring(constants.STATE, constants.STATE) + zip = self.generateZip() + + return [ street1, street2, city, state, zip ] + ## DEF + + ## ============================================== + ## generateTax + ## ============================================== + def generateTax(self): + return rand.fixedPoint(constants.TAX_DECIMALS, constants.MIN_TAX, constants.MAX_TAX) + ## DEF + + ## ============================================== + ## generateZip + ## ============================================== + def generateZip(self): + length = constants.ZIP_LENGTH - len(constants.ZIP_SUFFIX) + return rand.nstring(length, length) + constants.ZIP_SUFFIX + ## DEF + + ## ============================================== + ## fillOriginal + ## ============================================== + def fillOriginal(self, data): + """ + a string with ORIGINAL_STRING at a random position + """ + originalLength = len(constants.ORIGINAL_STRING) + position = rand.number(0, len(data) - originalLength) + out = data[:position] + constants.ORIGINAL_STRING + data[position + originalLength:] + assert len(out) == len(data) + return out + ## DEF +## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py b/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py new file mode 100755 index 00000000..61041e82 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http:##www.cs.brown.edu/~pavlo/ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import sys +import os +import string +import datetime +import logging +import re +import argparse +import glob +import time +import multiprocessing +from ConfigParser import SafeConfigParser +from pprint import pprint,pformat + +from util import * +from runtime import * +import drivers + +logging.basicConfig(level = logging.INFO, + format="%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s", + datefmt="%m-%d-%Y %H:%M:%S", + stream = sys.stdout) + +## ============================================== +## createDriverClass +## ============================================== +def createDriverClass(name): + full_name = "%sDriver" % name.title() + mod = __import__('drivers.%s' % full_name.lower(), globals(), locals(), [full_name]) + klass = getattr(mod, full_name) + return klass +## DEF + +## ============================================== +## getDrivers +## ============================================== +def getDrivers(): + drivers = [ ] + for f in map(lambda x: os.path.basename(x).replace("driver.py", ""), glob.glob("./drivers/*driver.py")): + if f != "abstract": drivers.append(f) + return (drivers) +## DEF + +## ============================================== +## startLoading +## ============================================== +def startLoading(driverClass, scaleParameters, args, config): + logging.debug("Creating client pool with %d processes" % args['clients']) + pool = multiprocessing.Pool(args['clients']) + debug = logging.getLogger().isEnabledFor(logging.DEBUG) + + # Split the warehouses into chunks + w_ids = map(lambda x: [ ], range(args['clients'])) + for w_id in range(scaleParameters.starting_warehouse, scaleParameters.ending_warehouse+1): + idx = w_id % args['clients'] + w_ids[idx].append(w_id) + ## FOR + + loader_results = [ ] + for i in range(args['clients']): + r = pool.apply_async(loaderFunc, (driverClass, scaleParameters, args, config, w_ids[i], True)) + loader_results.append(r) + ## FOR + + pool.close() + logging.debug("Waiting for %d loaders to finish" % args['clients']) + pool.join() +## DEF + +## ============================================== +## loaderFunc +## ============================================== +def loaderFunc(driverClass, scaleParameters, args, config, w_ids, debug): + driver = driverClass(args['ddl']) + assert driver != None + logging.debug("Starting client execution: %s [warehouses=%d]" % (driver, len(w_ids))) + + config['load'] = True + config['execute'] = False + config['reset'] = False + driver.loadConfig(config) + + try: + loadItems = (1 in w_ids) + l = loader.Loader(driver, scaleParameters, w_ids, loadItems) + driver.loadStart() + l.execute() + driver.loadFinish() + except KeyboardInterrupt: + return -1 + except (Exception, AssertionError), ex: + logging.warn("Failed to load data: %s" % (ex)) + #if debug: + traceback.print_exc(file=sys.stdout) + raise + +## DEF + +## ============================================== +## startExecution +## ============================================== +def startExecution(driverClass, scaleParameters, args, config): + logging.debug("Creating client pool with %d processes" % args['clients']) + pool = multiprocessing.Pool(args['clients']) + debug = logging.getLogger().isEnabledFor(logging.DEBUG) + + worker_results = [ ] + for i in range(args['clients']): + r = pool.apply_async(executorFunc, (driverClass, scaleParameters, args, config, debug,)) + worker_results.append(r) + ## FOR + pool.close() + pool.join() + + total_results = results.Results() + for asyncr in worker_results: + asyncr.wait() + r = asyncr.get() + assert r != None, "No results object returned!" + if type(r) == int and r == -1: sys.exit(1) + total_results.append(r) + ## FOR + + return (total_results) +## DEF + +## ============================================== +## executorFunc +## ============================================== +def executorFunc(driverClass, scaleParameters, args, config, debug): + driver = driverClass(args['ddl']) + assert driver != None + logging.debug("Starting client execution: %s" % driver) + + config['execute'] = True + config['reset'] = False + driver.loadConfig(config) + + e = executor.Executor(driver, scaleParameters, stop_on_error=args['stop_on_error']) + driver.executeStart() + results = e.execute(args['duration']) + driver.executeFinish() + + return results +## DEF + +## ============================================== +## main +## ============================================== +if __name__ == '__main__': + aparser = argparse.ArgumentParser(description='Python implementation of the TPC-C Benchmark') + aparser.add_argument('system', choices=getDrivers(), + help='Target system driver') + aparser.add_argument('--config', type=file, + help='Path to driver configuration file') + aparser.add_argument('--reset', action='store_true', + help='Instruct the driver to reset the contents of the database') + aparser.add_argument('--scalefactor', default=1, type=float, metavar='SF', + help='Benchmark scale factor') + aparser.add_argument('--warehouses', default=4, type=int, metavar='W', + help='Number of Warehouses') + aparser.add_argument('--duration', default=60, type=int, metavar='D', + help='How long to run the benchmark in seconds') + aparser.add_argument('--ddl', default=os.path.realpath(os.path.join(os.path.dirname(__file__), "tpcc.sql")), + help='Path to the TPC-C DDL SQL file') + aparser.add_argument('--clients', default=1, type=int, metavar='N', + help='The number of blocking clients to fork') + aparser.add_argument('--stop-on-error', action='store_true', + help='Stop the transaction execution when the driver throws an exception.') + aparser.add_argument('--no-load', action='store_true', + help='Disable loading the data') + aparser.add_argument('--no-execute', action='store_true', + help='Disable executing the workload') + aparser.add_argument('--print-config', action='store_true', + help='Print out the default configuration file for the system and exit') + aparser.add_argument('--debug', action='store_true', + help='Enable debug log messages') + args = vars(aparser.parse_args()) + + if args['debug']: logging.getLogger().setLevel(logging.DEBUG) + + ## Arguments validation + assert args['reset'] == False or args['no_load'] == False, \ + "'--reset' and '--no-load' are incompatible with each other" + + ## Create a handle to the target client driver + driverClass = createDriverClass(args['system']) + assert driverClass != None, "Failed to find '%s' class" % args['system'] + driver = driverClass(args['ddl']) + assert driver != None, "Failed to create '%s' driver" % args['system'] + if args['print_config']: + config = driver.makeDefaultConfig() + print driver.formatConfig(config) + print + sys.exit(0) + + ## Load Configuration file + if args['config']: + logging.debug("Loading configuration file '%s'" % args['config']) + cparser = SafeConfigParser() + cparser.read(os.path.realpath(args['config'].name)) + config = dict(cparser.items(args['system'])) + else: + logging.debug("Using default configuration for %s" % args['system']) + defaultConfig = driver.makeDefaultConfig() + config = dict(map(lambda x: (x, defaultConfig[x][1]), defaultConfig.keys())) + config['reset'] = args['reset'] + config['load'] = False + config['execute'] = False + if config['reset']: logging.info("Reseting database") + driver.loadConfig(config) + logging.info("Initializing TPC-C benchmark using %s" % driver) + + ## Create ScaleParameters + scaleParameters = scaleparameters.makeWithScaleFactor(args['warehouses'], args['scalefactor']) + nurand = rand.setNURand(nurand.makeForLoad()) + if args['debug']: logging.debug("Scale Parameters:\n%s" % scaleParameters) + + ## DATA LOADER!!! + load_time = None + if not args['no_load']: + logging.info("Loading TPC-C benchmark data using %s" % (driver)) + load_start = time.time() + if args['clients'] == 1: + l = loader.Loader(driver, scaleParameters, range(scaleParameters.starting_warehouse, scaleParameters.ending_warehouse+1), True) + driver.loadStart() + l.execute() + driver.loadFinish() + else: + startLoading(driverClass, scaleParameters, args, config) + load_time = time.time() - load_start + ## IF + + ## WORKLOAD DRIVER!!! + if not args['no_execute']: + if args['clients'] == 1: + e = executor.Executor(driver, scaleParameters, stop_on_error=args['stop_on_error']) + driver.executeStart() + results = e.execute(args['duration']) + driver.executeFinish() + else: + results = startExecution(driverClass, scaleParameters, args, config) + assert results + print results.show(load_time) + ## IF + +## MAIN \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.sql b/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.sql new file mode 100644 index 00000000..42f6dc28 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.sql @@ -0,0 +1,140 @@ +CREATE TABLE WAREHOUSE ( + W_ID SMALLINT DEFAULT '0' NOT NULL, + W_NAME VARCHAR(16) DEFAULT NULL, + W_STREET_1 VARCHAR(32) DEFAULT NULL, + W_STREET_2 VARCHAR(32) DEFAULT NULL, + W_CITY VARCHAR(32) DEFAULT NULL, + W_STATE VARCHAR(2) DEFAULT NULL, + W_ZIP VARCHAR(9) DEFAULT NULL, + W_TAX FLOAT DEFAULT NULL, + W_YTD FLOAT DEFAULT NULL, + CONSTRAINT W_PK_ARRAY PRIMARY KEY (W_ID) +); + +CREATE TABLE DISTRICT ( + D_ID TINYINT DEFAULT '0' NOT NULL, + D_W_ID SMALLINT DEFAULT '0' NOT NULL REFERENCES WAREHOUSE (W_ID), + D_NAME VARCHAR(16) DEFAULT NULL, + D_STREET_1 VARCHAR(32) DEFAULT NULL, + D_STREET_2 VARCHAR(32) DEFAULT NULL, + D_CITY VARCHAR(32) DEFAULT NULL, + D_STATE VARCHAR(2) DEFAULT NULL, + D_ZIP VARCHAR(9) DEFAULT NULL, + D_TAX FLOAT DEFAULT NULL, + D_YTD FLOAT DEFAULT NULL, + D_NEXT_O_ID INT DEFAULT NULL, + PRIMARY KEY (D_W_ID,D_ID) +); + +CREATE TABLE ITEM ( + I_ID INTEGER DEFAULT '0' NOT NULL, + I_IM_ID INTEGER DEFAULT NULL, + I_NAME VARCHAR(32) DEFAULT NULL, + I_PRICE FLOAT DEFAULT NULL, + I_DATA VARCHAR(64) DEFAULT NULL, + CONSTRAINT I_PK_ARRAY PRIMARY KEY (I_ID) +); + +CREATE TABLE CUSTOMER ( + C_ID INTEGER DEFAULT '0' NOT NULL, + C_D_ID TINYINT DEFAULT '0' NOT NULL, + C_W_ID SMALLINT DEFAULT '0' NOT NULL, + C_FIRST VARCHAR(32) DEFAULT NULL, + C_MIDDLE VARCHAR(2) DEFAULT NULL, + C_LAST VARCHAR(32) DEFAULT NULL, + C_STREET_1 VARCHAR(32) DEFAULT NULL, + C_STREET_2 VARCHAR(32) DEFAULT NULL, + C_CITY VARCHAR(32) DEFAULT NULL, + C_STATE VARCHAR(2) DEFAULT NULL, + C_ZIP VARCHAR(9) DEFAULT NULL, + C_PHONE VARCHAR(32) DEFAULT NULL, + C_SINCE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, + C_CREDIT VARCHAR(2) DEFAULT NULL, + C_CREDIT_LIM FLOAT DEFAULT NULL, + C_DISCOUNT FLOAT DEFAULT NULL, + C_BALANCE FLOAT DEFAULT NULL, + C_YTD_PAYMENT FLOAT DEFAULT NULL, + C_PAYMENT_CNT INTEGER DEFAULT NULL, + C_DELIVERY_CNT INTEGER DEFAULT NULL, + C_DATA VARCHAR(500), + PRIMARY KEY (C_W_ID,C_D_ID,C_ID), + UNIQUE (C_W_ID,C_D_ID,C_LAST,C_FIRST), + CONSTRAINT C_FKEY_D FOREIGN KEY (C_D_ID, C_W_ID) REFERENCES DISTRICT (D_ID, D_W_ID) +); +CREATE INDEX IDX_CUSTOMER ON CUSTOMER (C_W_ID,C_D_ID,C_LAST); + +CREATE TABLE HISTORY ( + H_C_ID INTEGER DEFAULT NULL, + H_C_D_ID TINYINT DEFAULT NULL, + H_C_W_ID SMALLINT DEFAULT NULL, + H_D_ID TINYINT DEFAULT NULL, + H_W_ID SMALLINT DEFAULT '0' NOT NULL, + H_DATE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, + H_AMOUNT FLOAT DEFAULT NULL, + H_DATA VARCHAR(32) DEFAULT NULL, + CONSTRAINT H_FKEY_C FOREIGN KEY (H_C_ID, H_C_D_ID, H_C_W_ID) REFERENCES CUSTOMER (C_ID, C_D_ID, C_W_ID), + CONSTRAINT H_FKEY_D FOREIGN KEY (H_D_ID, H_W_ID) REFERENCES DISTRICT (D_ID, D_W_ID) +); + +CREATE TABLE STOCK ( + S_I_ID INTEGER DEFAULT '0' NOT NULL REFERENCES ITEM (I_ID), + S_W_ID SMALLINT DEFAULT '0 ' NOT NULL REFERENCES WAREHOUSE (W_ID), + S_QUANTITY INTEGER DEFAULT '0' NOT NULL, + S_DIST_01 VARCHAR(32) DEFAULT NULL, + S_DIST_02 VARCHAR(32) DEFAULT NULL, + S_DIST_03 VARCHAR(32) DEFAULT NULL, + S_DIST_04 VARCHAR(32) DEFAULT NULL, + S_DIST_05 VARCHAR(32) DEFAULT NULL, + S_DIST_06 VARCHAR(32) DEFAULT NULL, + S_DIST_07 VARCHAR(32) DEFAULT NULL, + S_DIST_08 VARCHAR(32) DEFAULT NULL, + S_DIST_09 VARCHAR(32) DEFAULT NULL, + S_DIST_10 VARCHAR(32) DEFAULT NULL, + S_YTD INTEGER DEFAULT NULL, + S_ORDER_CNT INTEGER DEFAULT NULL, + S_REMOTE_CNT INTEGER DEFAULT NULL, + S_DATA VARCHAR(64) DEFAULT NULL, + PRIMARY KEY (S_W_ID,S_I_ID) +); + +CREATE TABLE ORDERS ( + O_ID INTEGER DEFAULT '0' NOT NULL, + O_C_ID INTEGER DEFAULT NULL, + O_D_ID TINYINT DEFAULT '0' NOT NULL, + O_W_ID SMALLINT DEFAULT '0' NOT NULL, + O_ENTRY_D TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, + O_CARRIER_ID INTEGER DEFAULT NULL, + O_OL_CNT INTEGER DEFAULT NULL, + O_ALL_LOCAL INTEGER DEFAULT NULL, + PRIMARY KEY (O_W_ID,O_D_ID,O_ID), + UNIQUE (O_W_ID,O_D_ID,O_C_ID,O_ID), + CONSTRAINT O_FKEY_C FOREIGN KEY (O_C_ID, O_D_ID, O_W_ID) REFERENCES CUSTOMER (C_ID, C_D_ID, C_W_ID) +); +CREATE INDEX IDX_ORDERS ON ORDERS (O_W_ID,O_D_ID,O_C_ID); + +CREATE TABLE NEW_ORDER ( + NO_O_ID INTEGER DEFAULT '0' NOT NULL, + NO_D_ID TINYINT DEFAULT '0' NOT NULL, + NO_W_ID SMALLINT DEFAULT '0' NOT NULL, + CONSTRAINT NO_PK_TREE PRIMARY KEY (NO_D_ID,NO_W_ID,NO_O_ID), + CONSTRAINT NO_FKEY_O FOREIGN KEY (NO_O_ID, NO_D_ID, NO_W_ID) REFERENCES ORDERS (O_ID, O_D_ID, O_W_ID) +); + +CREATE TABLE ORDER_LINE ( + OL_O_ID INTEGER DEFAULT '0' NOT NULL, + OL_D_ID TINYINT DEFAULT '0' NOT NULL, + OL_W_ID SMALLINT DEFAULT '0' NOT NULL, + OL_NUMBER INTEGER DEFAULT '0' NOT NULL, + OL_I_ID INTEGER DEFAULT NULL, + OL_SUPPLY_W_ID SMALLINT DEFAULT NULL, + OL_DELIVERY_D TIMESTAMP DEFAULT NULL, + OL_QUANTITY INTEGER DEFAULT NULL, + OL_AMOUNT FLOAT DEFAULT NULL, + OL_DIST_INFO VARCHAR(32) DEFAULT NULL, + PRIMARY KEY (OL_W_ID,OL_D_ID,OL_O_ID,OL_NUMBER), + CONSTRAINT OL_FKEY_O FOREIGN KEY (OL_O_ID, OL_D_ID, OL_W_ID) REFERENCES ORDERS (O_ID, O_D_ID, O_W_ID), + CONSTRAINT OL_FKEY_S FOREIGN KEY (OL_I_ID, OL_SUPPLY_W_ID) REFERENCES STOCK (S_I_ID, S_W_ID) +); +--CREATE INDEX IDX_ORDER_LINE_3COL ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); +--CREATE INDEX IDX_ORDER_LINE_2COL ON ORDER_LINE (OL_W_ID,OL_D_ID); +CREATE INDEX IDX_ORDER_LINE_TREE ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/util/__init__.py b/workloads/chbenchmark/py-tpcc/pytpcc/util/__init__.py new file mode 100644 index 00000000..2d2d802f --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/util/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +__all__ = ["scaleparameters", "rand", "nurand", "results"] \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/util/nurand.py b/workloads/chbenchmark/py-tpcc/pytpcc/util/nurand.py new file mode 100644 index 00000000..c4854228 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/util/nurand.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Original Java Version: +# Copyright (C) 2008 +# Evan Jones +# Massachusetts Institute of Technology +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import rand + +def makeForLoad(): + """Create random NURand constants, appropriate for loading the database.""" + cLast = rand.number(0, 255) + cId = rand.number(0, 1023) + orderLineItemId = rand.number(0, 8191) + return NURandC(cLast, cId, orderLineItemId) + +def validCRun(cRun, cLoad): + """Returns true if the cRun value is valid for running. See TPC-C 2.1.6.1 (page 20)""" + cDelta = abs(cRun - cLoad) + return 65 <= cDelta and cDelta <= 119 and cDelta != 96 and cDelta != 112 + +def makeForRun(loadC): + """Create random NURand constants for running TPC-C. TPC-C 2.1.6.1. (page 20) specifies the valid range for these constants.""" + cRun = rand.number(0, 255) + while validCRun(cRun, loadC.cLast) == False: + cRun = rand.number(0, 255) + assert validCRun(cRun, loadC.cLast) + + cId = rand.number(0, 1023) + orderLineItemId = rand.number(0, 8191) + return NURandC(cRun, cId, orderLineItemId) + +class NURandC: + def __init__(self, cLast, cId, orderLineItemId): + self.cLast = cLast + self.cId = cId + self.orderLineItemId = orderLineItemId diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/util/rand.py b/workloads/chbenchmark/py-tpcc/pytpcc/util/rand.py new file mode 100644 index 00000000..4e876659 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/util/rand.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Original Java Version: +# Copyright (C) 2008 +# Evan Jones +# Massachusetts Institute of Technology +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import random +import nurand + +SYLLABLES = [ "BAR", "OUGHT", "ABLE", "PRI", "PRES", "ESE", "ANTI", "CALLY", "ATION", "EING" ] + +nurandVar = None # NURand +def setNURand(nu): + global nurandVar + nurandVar = nu +## DEF + +def NURand(a, x, y): + """A non-uniform random number, as defined by TPC-C 2.1.6. (page 20).""" + global nurandVar + assert x <= y + if nurandVar is None: + setNURand(nurand.makeForLoad()) + + if a == 255: + c = nurandVar.cLast + elif a == 1023: + c = nurandVar.cId + elif a == 8191: + c = nurandVar.orderLineItemId + else: + raise Exception("a = " + a + " is not a supported value") + + return (((number(0, a) | number(x, y)) + c) % (y - x + 1)) + x +## DEF + +def number(minimum, maximum): + value = random.randint(minimum, maximum) + assert minimum <= value and value <= maximum + return value +## DEF + +def numberExcluding(minimum, maximum, excluding): + """An in the range [minimum, maximum], excluding excluding.""" + assert minimum < maximum + assert minimum <= excluding and excluding <= maximum + + ## Generate 1 less number than the range + num = number(minimum, maximum-1) + + ## Adjust the numbers to remove excluding + if num >= excluding: num += 1 + assert minimum <= num and num <= maximum and num != excluding + return num +## DEF + +def fixedPoint(decimal_places, minimum, maximum): + assert decimal_places > 0 + assert minimum < maximum + + multiplier = 1 + for i in range(0, decimal_places): + multiplier *= 10 + + int_min = int(minimum * multiplier + 0.5) + int_max = int(maximum * multiplier + 0.5) + + return float(number(int_min, int_max) / float(multiplier)) +## DEF + +def selectUniqueIds(numUnique, minimum, maximum): + rows = set() + for i in range(0, numUnique): + index = None + while index == None or index in rows: + index = number(minimum, maximum) + ## WHILE + rows.add(index) + ## FOR + assert len(rows) == numUnique + return rows +## DEF + +def astring(minimum_length, maximum_length): + """A random alphabetic string with length in range [minimum_length, maximum_length].""" + return randomString(minimum_length, maximum_length, 'a', 26) +## DEF + +def nstring(minimum_length, maximum_length): + """A random numeric string with length in range [minimum_length, maximum_length].""" + return randomString(minimum_length, maximum_length, '0', 10) +## DEF + +def randomString(minimum_length, maximum_length, base, numCharacters): + length = number(minimum_length, maximum_length) + baseByte = ord(base) + string = "" + for i in range(length): + string += chr(baseByte + number(0, numCharacters-1)) + return string +## DEF + +def makeLastName(number): + """A last name as defined by TPC-C 4.3.2.3. Not actually random.""" + global SYLLABLES + assert 0 <= number and number <= 999 + indicies = [ number/100, (number/10)%10, number%10 ] + return "".join(map(lambda x: SYLLABLES[x], indicies)) +## DEF + +def makeRandomLastName(maxCID): + """A non-uniform random last name, as defined by TPC-C 4.3.2.3. The name will be limited to maxCID.""" + min_cid = 999 + if (maxCID - 1) < min_cid: min_cid = maxCID - 1 + return makeLastName(NURand(255, 0, min_cid)) +## DEF diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/util/results.py b/workloads/chbenchmark/py-tpcc/pytpcc/util/results.py new file mode 100644 index 00000000..33bf9dd9 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/util/results.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import logging +import time + +class Results: + + def __init__(self): + self.start = None + self.stop = None + self.txn_id = 0 + + self.txn_counters = { } + self.txn_times = { } + self.running = { } + + def startBenchmark(self): + """Mark the benchmark as having been started""" + assert self.start == None + logging.debug("Starting benchmark statistics collection") + self.start = time.time() + return self.start + + def stopBenchmark(self): + """Mark the benchmark as having been stopped""" + assert self.start != None + assert self.stop == None + logging.debug("Stopping benchmark statistics collection") + self.stop = time.time() + + def startTransaction(self, txn): + self.txn_id += 1 + id = self.txn_id + self.running[id] = (txn, time.time()) + return id + + def abortTransaction(self, id): + """Abort a transaction and discard its times""" + assert id in self.running + txn_name, txn_start = self.running[id] + del self.running[id] + + def stopTransaction(self, id): + """Record that the benchmark completed an invocation of the given transaction""" + assert id in self.running + txn_name, txn_start = self.running[id] + del self.running[id] + + duration = time.time() - txn_start + total_time = self.txn_times.get(txn_name, 0) + self.txn_times[txn_name] = total_time + duration + + total_cnt = self.txn_counters.get(txn_name, 0) + self.txn_counters[txn_name] = total_cnt + 1 + + def append(self, r): + for txn_name in r.txn_counters.keys(): + orig_cnt = self.txn_counters.get(txn_name, 0) + orig_time = self.txn_times.get(txn_name, 0) + + self.txn_counters[txn_name] = orig_cnt + r.txn_counters[txn_name] + self.txn_times[txn_name] = orig_time + r.txn_times[txn_name] + #logging.debug("%s [cnt=%d, time=%d]" % (txn_name, self.txn_counters[txn_name], self.txn_times[txn_name])) + ## HACK + self.start = r.start + self.stop = r.stop + + def __str__(self): + return self.show() + + def show(self, load_time = None): + if self.start == None: + return "Benchmark not started" + if self.stop == None: + duration = time.time() - self.start + else: + duration = self.stop - self.start + + col_width = 16 + total_width = (col_width*4)+2 + f = "\n " + (("%-" + str(col_width) + "s")*4) + line = "-"*total_width + + ret = u"" + "="*total_width + "\n" + if load_time != None: + ret += "Data Loading Time: %d seconds\n\n" % (load_time) + + ret += "Execution Results after %d seconds\n%s" % (duration, line) + ret += f % ("", "Executed", u"Time (µs)", "Rate") + + total_time = 0 + total_cnt = 0 + for txn in sorted(self.txn_counters.keys()): + txn_time = self.txn_times[txn] + txn_cnt = self.txn_counters[txn] + rate = u"%.02f txn/s" % ((txn_cnt / txn_time)) + ret += f % (txn, str(txn_cnt), str(txn_time * 1000000), rate) + + total_time += txn_time + total_cnt += txn_cnt + ret += "\n" + ("-"*total_width) + total_rate = "%.02f txn/s" % ((total_cnt / total_time)) + ret += f % ("TOTAL", str(total_cnt), str(total_time * 1000000), total_rate) + + return (ret.encode('utf-8')) +## CLASS \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/util/scaleparameters.py b/workloads/chbenchmark/py-tpcc/pytpcc/util/scaleparameters.py new file mode 100644 index 00000000..00417968 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/util/scaleparameters.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo +# http://www.cs.brown.edu/~pavlo/ +# +# Original Java Version: +# Copyright (C) 2008 +# Evan Jones +# Massachusetts Institute of Technology +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import constants + +def makeDefault(warehouses): + return ScaleParameters(constants.NUM_ITEMS, \ + warehouses, \ + constants.DISTRICTS_PER_WAREHOUSE, \ + constants.CUSTOMERS_PER_DISTRICT, \ + constants.INITIAL_NEW_ORDERS_PER_DISTRICT) +## DEF + +def makeWithScaleFactor(warehouses, scaleFactor): + assert scaleFactor >= 1.0 + + items = int(constants.NUM_ITEMS/scaleFactor) + if items <= 0: items = 1 + districts = int(max(constants.DISTRICTS_PER_WAREHOUSE, 1)) + customers = int(max(constants.CUSTOMERS_PER_DISTRICT/scaleFactor, 1)) + newOrders = int(max(constants.INITIAL_NEW_ORDERS_PER_DISTRICT/scaleFactor, 0)) + + return ScaleParameters(items, warehouses, districts, customers, newOrders) +## DEF + +class ScaleParameters: + + def __init__(self, items, warehouses, districtsPerWarehouse, customersPerDistrict, newOrdersPerDistrict): + assert 1 <= items and items <= constants.NUM_ITEMS + self.items = items + assert warehouses > 0 + self.warehouses = warehouses + self.starting_warehouse = 1 + assert 1 <= districtsPerWarehouse and districtsPerWarehouse <= constants.DISTRICTS_PER_WAREHOUSE + self.districtsPerWarehouse = districtsPerWarehouse + assert 1 <= customersPerDistrict and customersPerDistrict <= constants.CUSTOMERS_PER_DISTRICT + self.customersPerDistrict = customersPerDistrict + assert 0 <= newOrdersPerDistrict and newOrdersPerDistrict <= constants.CUSTOMERS_PER_DISTRICT + assert newOrdersPerDistrict <= constants.INITIAL_NEW_ORDERS_PER_DISTRICT + self.newOrdersPerDistrict = newOrdersPerDistrict + self.ending_warehouse = (self.warehouses + self.starting_warehouse - 1) + ## DEF + + def __str__(self): + out = "%d items\n" % self.items + out += "%d warehouses\n" % self.warehouses + out += "%d districts/warehouse\n" % self.districtsPerWarehouse + out += "%d customers/district\n" % self.customersPerDistrict + out += "%d initial new orders/district" % self.newOrdersPerDistrict + return out + ## DEF + +## CLASS \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/worker.py b/workloads/chbenchmark/py-tpcc/pytpcc/worker.py new file mode 100755 index 00000000..5e2f9a62 --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/worker.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# Copyright (C) 2011 +# Andy Pavlo & Yang Lu +# http:##www.cs.brown.edu/~pavlo/ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# ----------------------------------------------------------------------- + +import sys +import os +import string +import datetime +import logging +import re +import argparse +import glob +import time +import message +import pickle +import traceback +from pprint import pprint,pformat + +from util import * +from runtime import * +import drivers + +## ============================================== +## createDriverClass +## ============================================== +def createDriverClass(name): + full_name = "%sDriver" % name.title() + mod = __import__('drivers.%s' % full_name.lower(), globals(), locals(), [full_name]) + klass = getattr(mod, full_name) + return klass +## DEF + + +## ============================================== +## loaderFunc +## ============================================== +def loaderFunc(driverClass, scaleParameters, args, config, w_ids, debug): + driver = driverClass(args['ddl']) + assert driver != None + logging.debug("Starting client execution: %s [warehouses=%d]" % (driver, len(w_ids))) + + config['load'] = True + config['execute'] = False + config['reset'] = False + driver.loadConfig(config) + + try: + loadItems = (1 in w_ids) + l = loader.Loader(driver, scaleParameters, w_ids, loadItems) + driver.loadStart() + l.execute() + driver.loadFinish() + except KeyboardInterrupt: + return -1 + except (Exception, AssertionError), ex: + logging.warn("Failed to load data: %s" % (ex)) + #if debug: + traceback.print_exc(file=sys.stdout) + raise + +## DEF + + +## ============================================== +## executorFunc +## ============================================== +def executorFunc(driverClass, scaleParameters, args, config, debug): + driver = driverClass(args['ddl']) + assert driver != None + logging.debug("Starting client execution: %s" % driver) + + config['execute'] = True + config['reset'] = False + driver.loadConfig(config) + + e = executor.Executor(driver, scaleParameters, stop_on_error=args['stop_on_error']) + driver.executeStart() + results = e.execute(args['duration']) + driver.executeFinish() + + return results +## DEF + +## MAIN +if __name__=='__channelexec__': + driverClass=None + for item in channel: + command=pickle.loads(item) + if command.header==message.CMD_LOAD: + scaleParameters=command.data[0] + args=command.data[1] + config=command.data[2] + w_ids=command.data[3] + + ## Create a handle to the target client driver at the client side + driverClass = createDriverClass(args['system']) + assert driverClass != None, "Failed to find '%s' class" % args['system'] + driver = driverClass(args['ddl']) + assert driver != None, "Failed to create '%s' driver" % args['system'] + + loaderFunc(driverClass,scaleParameters,args,config,w_ids,True) + m=message.Message(header=message.LOAD_COMPLETED) + channel.send(pickle.dumps(m,-1)) + elif command.header==message.CMD_EXECUTE: + scaleParameters=command.data[0] + args=command.data[1] + config=command.data[2] + + ## Create a handle to the target client driver at the client side + if driverClass==None: + driverClass = createDriverClass(args['system']) + assert driverClass != None, "Failed to find '%s' class" % args['system'] + driver = driverClass(args['ddl']) + assert driver != None, "Failed to create '%s' driver" % args['system'] + + results=executorFunc(driverClass,scaleParameters,args,config,True) + m=message.Message(header=message.EXECUTE_COMPLETED,data=results) + channel.send(pickle.dumps(m,-1)) + + elif command.header==message.CMD_STOP: + pass + else: + pass + \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/setup.py b/workloads/chbenchmark/py-tpcc/setup.py new file mode 100644 index 00000000..ba005d4a --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/setup.py @@ -0,0 +1,26 @@ +from setuptools import setup, find_packages +import sys, os + +version = '0.0' + +setup(name='py-tpcc', + version=version, + description="Python implementation of the TPC-C benchmark", + long_description="""\ +""", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='', + author='Andy Pavlo', + author_email='pavlo@cs.brown.edu', + url='http://www.cs.brown.edu/~pavlo/', + license='BSD', + packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), + include_package_data=True, + zip_safe=False, + install_requires=[ + # -*- Extra requirements: -*- + ], + entry_points=""" + # -*- Entry points: -*- + """, + ) From 0ca4eac4e5b351a048ff4b0edcac9c17b0804254 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Fri, 12 Apr 2024 17:01:10 -0400 Subject: [PATCH 2/5] Python 2 to 3 fixes --- .../chbenchmark/py-tpcc/pytpcc/constants.py | 2 +- .../chbenchmark/py-tpcc/pytpcc/coordinator.py | 8 +- .../py-tpcc/pytpcc/drivers/cassandradriver.py | 771 -------- .../py-tpcc/pytpcc/drivers/hbasedriver.py | 1408 -------------- .../py-tpcc/pytpcc/drivers/membasedriver.py | 932 --------- .../py-tpcc/pytpcc/drivers/redisdriver.py | 1663 ----------------- .../py-tpcc/pytpcc/drivers/scalarisdriver.py | 1039 ---------- .../pytpcc/drivers/tokyocabinetdriver.py | 1061 ----------- .../chbenchmark/py-tpcc/pytpcc/message.py | 8 +- .../py-tpcc/pytpcc/runtime/executor.py | 2 +- workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py | 8 +- .../chbenchmark/py-tpcc/pytpcc/util/rand.py | 73 +- .../chbenchmark/py-tpcc/pytpcc/worker.py | 85 +- 13 files changed, 116 insertions(+), 6944 deletions(-) delete mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/cassandradriver.py delete mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/hbasedriver.py delete mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/membasedriver.py delete mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/redisdriver.py delete mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/scalarisdriver.py delete mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/tokyocabinetdriver.py diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/constants.py b/workloads/chbenchmark/py-tpcc/pytpcc/constants.py index b5a9bbef..3918b3cd 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/constants.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/constants.py @@ -91,7 +91,7 @@ MIN_CARRIER_ID = 1 MAX_CARRIER_ID = 10 # HACK: This is not strictly correct, but it works -NULL_CARRIER_ID = 0L +NULL_CARRIER_ID = 0 # o_id < than this value, carrier != null, >= -> carrier == null NULL_CARRIER_LOWER_BOUND = 2101 MIN_OL_CNT = 5 diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py b/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py index cece61f3..9ddfb0dc 100755 --- a/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py @@ -80,7 +80,7 @@ def startLoading(scalParameters,args,config,channels): for w_id in range(scaleParameters.starting_warehouse, scaleParameters.ending_warehouse+1): idx = w_id % procs w_ids[idx].append(w_id) - print w_ids + print(w_ids) load_start=time.time() for i in range(len(channels)): @@ -157,7 +157,7 @@ def startExecution(scaleParameters, args, config,channels): assert driver != None, "Failed to create '%s' driver" % args['system'] if args['print_config']: config = driver.makeDefaultConfig() - print driver.formatConfig(config) + print(driver.formatConfig(config)) print sys.exit(0) @@ -212,7 +212,7 @@ def startExecution(scaleParameters, args, config,channels): if not args['no_execute']: results = startExecution(scaleParameters, args, config,channels) assert results - print results.show(load_time) + print(results.show(load_time)) ## IF -## MAIN \ No newline at end of file +## MAIN diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/cassandradriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/cassandradriver.py deleted file mode 100644 index 3f597e17..00000000 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/cassandradriver.py +++ /dev/null @@ -1,771 +0,0 @@ -# -*- coding: utf-8 -*- -# copyright (C) 2011 -# Jingxin Feng, Xiaowei Wang -# jxfeng@cs.brown.edu -# xiaowei@cs.brown.edu -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# ----------------------------------------------------------------------- - - -import pycassa -from pycassa.index import * -from pycassa.system_manager import * - -import os -import logging -import commands -import uuid -from pprint import pprint,pformat -import constants - -from abstractdriver import * -## ============================================== -## AbstractDriver -## ============================================== -class CassandraDriver(AbstractDriver): - - DEFAULT_CONFIG = { - "hostname": ("The host address to the Cassandra database","localhost"), - "port": ("Port number",9160), - "name": ("Name","tpcc"), - "keyspace":("Keyspace", "Keyspace1"), - "replicationfactor": ("ReplicationFactor", 1) - } - - - - def __init__(self, ddl): - super(CassandraDriver,self).__init__("cassandra",ddl) - self.conn = None - self.name = "cassandra" - self.database = None - - self.new_ordercf= None - self.orderscf= None - self.order_linecf= None - self.customercf = None - self.warehousecf = None - self.districtcf = None - self.historycf = None - self.stockcf = None - self.itemcf = None - def makeDefaultConfig(self): - return CassandraDriver.DEFAULT_CONFIG - - def loadConfig(self,config): - for key in CassandraDriver.DEFAULT_CONFIG.keys(): - assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) - - - connection = str(config["hostname"]+':'+str(config["port"])) - - - keyspace = str(config["keyspace"]) - self.sys = SystemManager(connection) - keyspaces = self.sys.list_keyspaces() - fl = 0 - for i in range(len(keyspaces)): - if str(keyspaces[i]) == keyspace: - fl = 1 - break - if fl == 0: - self.sys.create_keyspace(keyspace, SIMPLE_STRATEGY,{'replication_factor' : str(config["replicationfactor"])}) - self.sys.create_column_family(keyspace, 'NEW_ORDER', comparator_type = UTF8_TYPE) - self.sys.create_column_family(keyspace, 'ORDERS', comparator_type = UTF8_TYPE) - self.sys.create_column_family(keyspace, 'ORDER_LINE', comparator_type = UTF8_TYPE) - self.sys.create_column_family(keyspace, 'CUSTOMER', comparator_type = UTF8_TYPE) - self.sys.create_column_family(keyspace, 'WAREHOUSE', comparator_type = UTF8_TYPE) - self.sys.create_column_family(keyspace, 'DISTRICT', comparator_type = UTF8_TYPE) - self.sys.create_column_family(keyspace, 'HISTORY', comparator_type = UTF8_TYPE) - self.sys.create_column_family(keyspace, 'STOCK', comparator_type = UTF8_TYPE) - self.sys.create_column_family(keyspace, 'ITEM', comparator_type = UTF8_TYPE) - - - self.sys.alter_column(keyspace,'WAREHOUSE','W_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'WAREHOUSE','W_NAME',UTF8_TYPE) - self.sys.alter_column(keyspace,'WAREHOUSE','W_STREET_1',UTF8_TYPE) - self.sys.alter_column(keyspace,'WAREHOUSE','W_STREET_2',UTF8_TYPE) - self.sys.alter_column(keyspace,'WAREHOUSE','W_CITY',UTF8_TYPE) - self.sys.alter_column(keyspace,'WAREHOUSE','W_STATE',UTF8_TYPE) - self.sys.alter_column(keyspace,'WAREHOUSE','W_ZIP',UTF8_TYPE) - self.sys.alter_column(keyspace,'WAREHOUSE','W_TAX',UTF8_TYPE) - self.sys.alter_column(keyspace,'WAREHOUSE','W_YTD',UTF8_TYPE) - - - self.sys.alter_column(keyspace,'DISTRICT','D_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'DISTRICT','D_W_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'DISTRICT','D_NAME',UTF8_TYPE) - self.sys.alter_column(keyspace,'DISTRICT','D_STREET_1',UTF8_TYPE) - self.sys.alter_column(keyspace,'DISTRICT','D_STREET_2',UTF8_TYPE) - self.sys.alter_column(keyspace,'DISTRICT','D_CITY',UTF8_TYPE) - self.sys.alter_column(keyspace,'DISTRICT','D_STATE',UTF8_TYPE) - self.sys.alter_column(keyspace,'DISTRICT','D_ZIP',UTF8_TYPE) - self.sys.alter_column(keyspace,'DISTRICT','D_TAX',UTF8_TYPE) - self.sys.alter_column(keyspace,'DISTRICT','D_YTD',UTF8_TYPE) - self.sys.alter_column(keyspace,'DISTRICT','D_NEXT_O_ID',UTF8_TYPE) - - - self.sys.alter_column(keyspace,'CUSTOMER','C_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_D_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_W_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_FIRST',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_MIDDLE',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_LAST',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_STREET_1',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_STREET_2',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_CITY',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_STATE',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_ZIP',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_PHONE',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_SINCE',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_CREDIT',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_CREDIT_LIM',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_DISCOUNT',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_BALANCE',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_YTD_PAYMENT',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_PAYMENT_CNT',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_DELIVERY_CNT',UTF8_TYPE) - self.sys.alter_column(keyspace,'CUSTOMER','C_DATA',UTF8_TYPE) - - - - - - - - - self.sys.alter_column(keyspace,'HISTORY','H_C_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'HISTORY','H_C_D_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'HISTORY','H_C_W_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'HISTORY','H_D_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'HISTORY','H_W_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'HISTORY','H_DATE',UTF8_TYPE) - self.sys.alter_column(keyspace,'HISTORY','H_AMOUNT',UTF8_TYPE) - self.sys.alter_column(keyspace,'HISTORY','H_DATA',UTF8_TYPE) - - - - self.sys.alter_column(keyspace,'NEW_ORDER','NO_O_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'NEW_ORDER','NO_D_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'NEW_ORDER','NO_W_ID',UTF8_TYPE) - - self.sys.alter_column(keyspace,'ORDERS','O_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDERS','O_D_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDERS','O_W_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDERS','O_C_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDERS','O_ENTRY_D',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDERS','O_CARRIER_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDERS','O_OL_CNT',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDERS','O_ALL_LOCAL',UTF8_TYPE) - - - - self.sys.alter_column(keyspace,'ORDER_LINE','OL_O_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDER_LINE','OL_D_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDER_LINE','OL_W_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDER_LINE','OL_NUMBER',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDER_LINE','OL_I_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDER_LINE','OL_SUPPLY_W_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDER_LINE','OL_DELIVERY_D',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDER_LINE','OL_QUANTITY',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDER_LINE','OL_AMOUNT',UTF8_TYPE) - self.sys.alter_column(keyspace,'ORDER_LINE','OL_DIST_INFO',UTF8_TYPE) - - - self.sys.alter_column(keyspace,'ITEM','I_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ITEM','I_IM_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'ITEM','I_NAME',UTF8_TYPE) - self.sys.alter_column(keyspace,'ITEM','I_PRICE',UTF8_TYPE) - self.sys.alter_column(keyspace,'ITEM','I_DATA',UTF8_TYPE) - - - self.sys.alter_column(keyspace,'STOCK','S_I_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_W_ID',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_QUANTITY',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DIST_01',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DIST_02',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DIST_03',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DIST_04',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DIST_05',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DIST_06',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DIST_07',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DIST_08',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DIST_09',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DIST_10',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_YTD',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_ORDER_CNT',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_REMOTE_CNT',UTF8_TYPE) - self.sys.alter_column(keyspace,'STOCK','S_DATA',UTF8_TYPE) - - - self.sys.create_index(keyspace,'CUSTOMER','C_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'CUSTOMER','C_D_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'CUSTOMER','C_W_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'CUSTOMER','C_LAST', UTF8_TYPE) - self.sys.create_index(keyspace,'NEW_ORDER','NO_O_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'NEW_ORDER','NO_D_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'NEW_ORDER','NO_W_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'ORDERS','O_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'ORDERS','O_D_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'ORDERS','O_W_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'ORDERS','O_C_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'ORDER_LINE','OL_O_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'ORDER_LINE','OL_D_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'ORDER_LINE','OL_W_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'STOCK','S_W_ID', UTF8_TYPE) - self.sys.create_index(keyspace,'STOCK','S_QUANTITY', UTF8_TYPE) - - - self.conn = pycassa.connect(str(config["keyspace"]),[connection]) - self.new_ordercf=pycassa.ColumnFamily(self.conn,'NEW_ORDER') - self.orderscf=pycassa.ColumnFamily(self.conn, 'ORDERS') - self.order_linecf=pycassa.ColumnFamily(self.conn, 'ORDER_LINE') - self.customercf=pycassa.ColumnFamily(self.conn, 'CUSTOMER') - self.warehousecf = pycassa.ColumnFamily(self.conn,'WAREHOUSE') - self.districtcf = pycassa.ColumnFamily(self.conn, 'DISTRICT') - self.historycf = pycassa.ColumnFamily(self.conn,'HISTORY') - self.stockcf = pycassa.ColumnFamily(self.conn,'STOCK') - self.itemcf = pycassa.ColumnFamily(self.conn, 'ITEM') - - def loadTuples(self, tableName, tuples): - if len(tuples) == 0: - return - logging.debug("loading") - col_fam = pycassa.ColumnFamily(self.conn, tableName) - if tableName == 'ITEM': - for row in tuples: - row_key = str(row[0]).zfill(5) - i_id = str(row[0]) - i_im_id = str(row[1]) - i_name = str(row[2]) - i_price = str(row[3]) - i_data = str(row[4]) - col_fam.insert(row_key, {'I_ID':i_id, 'I_IM_ID':i_im_id, 'I_NAME':i_name, 'I_PRICE':i_price, 'I_DATA':i_data}) - if tableName == 'WAREHOUSE': - if len(tuples[0])!=9: return - for row in tuples: - row_key = str(row[0]).zfill(5) #w_ID - w_id = str(row[0]) - w_name =str(row[1]) - w_street_1 = str(row[2]) - w_street_2 = str(row[3]) - w_city = str(row[4]) - w_state = str(row[5]) - w_zip = str(row[6]) - w_tax = str(row[7]) - w_ytd = str(row[8]) - col_fam.insert(row_key,{'W_ID':w_id, 'W_NAME':w_name, 'W_STREET_1': w_street_1, 'W_STREET_2': w_street_2, 'W_CITY':w_city, 'W_STATE':w_state, 'W_ZIP':w_zip, 'W_TAX':w_tax, 'W_YTD':w_ytd}) - if tableName == 'CUSTOMER': - for row in tuples: - row_key = str(row[0]).zfill(5)+ str(row[1]).zfill(5)+ str(row[2]).zfill(5) - c_id = str(row[0]) - c_d_id =str(row[1]) - c_w_id =str(row[2]) - c_first =str(row[3]) - c_middle = str(row[4]) - c_last = str(row[5]) - c_street_1 = str(row[6]) - c_street_2 = str(row[7]) - c_city = str(row[8]) - c_state = str(row[9]) - c_zip = str(row[10]) - c_phone = str(row[11]) - c_since = str(row[12]) - c_credit = str(row[13]) - c_credit_lim = str(row[14]) - c_discount = str(row[15]) - c_balance = str(row[16]) - c_ytd_payment = str(row[17]) - c_payment_cnt = str(row[18]) - c_delivery_cnt = str(row[19]) - c_data = str(row[20]) - col_fam.insert(row_key, {'C_ID':c_id, 'C_D_ID':c_d_id, 'C_W_ID':c_w_id, 'C_FIRST':c_first, 'C_MIDDLE':c_middle, 'C_LAST':c_last, 'C_STREET_1':c_street_1,'C_STREET_2':c_street_2, 'C_CITY':c_city, 'C_STATE':c_state, 'C_ZIP':c_zip, 'C_PHONE':c_phone, 'C_SINCE':c_since, 'C_CREDIT':c_credit, 'C_CREDIT_LIM':c_credit_lim, 'C_DISCOUNT':c_discount, 'C_BALANCE':c_balance, 'C_YTD_PAYMENT':c_ytd_payment, 'C_PAYMENT_CNT':c_payment_cnt, 'C_DELIVERY_CNT':c_delivery_cnt, 'C_DATA':c_data}) - - if tableName == 'ORDERS': - for row in tuples: - row_key = str(row[0]).zfill(5)+str(row[1]).zfill(5)+ str(row[2]).zfill(5) - o_id = str(row[0]) - o_d_id = str(row[1]) - o_w_id = str(row[2]) - o_c_id = str(row[3]) - o_entry_d = str(row[4]) - o_carrier_id = str(row[5]) - o_ol_cnt = str(row[6]) - o_all_local = str(row[7]) - col_fam.insert(row_key,{'O_ID':o_id, 'O_D_ID':o_d_id, 'O_W_ID':o_w_id, 'O_C_ID':o_c_id, 'O_ENTRY_D':o_entry_d, 'O_CARRIER_ID':o_carrier_id, 'O_OL_CNT':o_ol_cnt, 'O_ALL_LOCAL':o_all_local}) - - - if tableName == 'STOCK': - for row in tuples: - row_key = str(row[0]).zfill(5)+str(row[1]).zfill(5) - s_i_id = str(row[0]) - s_w_id = str(row[1]) - s_quantity = str(row[2]) - s_dist_01 = str(row[3]) - s_dist_02 = str(row[4]) - s_dist_03 = str(row[5]) - s_dist_04 = str(row[6]) - s_dist_05 = str(row[7]) - s_dist_06 = str(row[8]) - s_dist_07 = str(row[9]) - s_dist_08 = str(row[10]) - s_dist_09 = str(row[11]) - s_dist_10 = str(row[12]) - s_ytd = str(row[13]) - s_order_cnt = str(row[14]) - s_remote_cnt = str(row[15]) - s_data = str(row[16]) - col_fam.insert(row_key,{'S_I_ID':s_i_id, 'S_W_ID':s_w_id, 'S_QUANTITY':s_quantity, 'S_DIST_01':s_dist_01,'S_DIST_02':s_dist_02,'S_DIST_03':s_dist_03,'S_DIST_04':s_dist_04,'S_DIST_05':s_dist_05,'S_DIST_06':s_dist_06,'S_DIST_07':s_dist_07,'S_DIST_08':s_dist_08,'S_DIST_09':s_dist_09,'S_DIST_10':s_dist_10, 'S_YTD': s_ytd, 'S_ORDER_CNT':s_order_cnt, 'S_REMOTE_CNT':s_remote_cnt, 'S_DATA':s_data}) - - if tableName == 'DISTRICT': - for row in tuples: - row_key = str(row[0]).zfill(5)+str(row[1]).zfill(5) - d_id = str(row[0]) - d_w_id = str(row[1]) - d_name = str(row[2]) - d_street_1 = str(row[3]) - d_street_2 = str(row[4]) - d_city = str(row[5]) - d_state = str(row[6]) - d_zip = str(row[7]) - d_tax =str(row[8]) - d_ytd = str(row[9]) - d_next_o_id = str(row[10]) - col_fam.insert(row_key,{'D_ID':d_id, 'D_W_ID':d_w_id, 'D_NAME':d_name, 'D_STREET_1':d_street_1, 'D_STREET_2':d_street_2,'D_CITY':d_city, 'D_STATE':d_state, 'D_ZIP':d_zip, 'D_TAX':d_tax, 'D_YTD':d_ytd, 'D_NEXT_O_ID':d_next_o_id}) - - if tableName == 'NEW_ORDER': - for row in tuples: - row_key = str(row[0]).zfill(5)+str(row[1]).zfill(5)+str(row[2]).zfill(5) - no_o_id = str(row[0]) - no_d_id = str(row[1]) - no_w_id = str(row[2]) - col_fam.insert(row_key,{'NO_O_ID':no_o_id, 'NO_D_ID':no_d_id, 'NO_W_ID':no_w_id}) - if tableName == 'ORDER_LINE': - for row in tuples: - row_key = str(row[0]).zfill(5)+str(row[1]).zfill(5)+str(row[2]).zfill(5)+str(row[3]).zfill(5) - ol_o_id = str(row[0]) - ol_d_id = str(row[1]) - ol_w_id = str(row[2]) - ol_number = str(row[3]) - ol_i_id = str(row[4]) - ol_supply_w_id = str(row[5]) - ol_delivery_d = str(row[6]) - ol_quantity = str(row[7]) - ol_amount = str(row[8]) - ol_dist_info = str(row[9]) - col_fam.insert(row_key,{'OL_O_ID':ol_o_id, 'OL_D_ID':ol_d_id, 'OL_W_ID':ol_w_id, 'OL_NUMBER':ol_number, 'OL_I_ID':ol_i_id, 'OL_SUPPLY_W_ID':ol_supply_w_id, 'OL_DELIVERY_D': ol_delivery_d, 'OL_QUANTITY':ol_quantity,'OL_AMOUNT':ol_amount, 'OL_DIST_INFO':ol_dist_info}) - - if tableName == 'HISTORY': - for i in range(len(tuples)): - #row_key = str(i) - row_key = str(uuid.uuid1()) - h_c_id = str(tuples[i][0]) - h_c_d_id = str(tuples[i][1]) - h_c_w_id = str(tuples[i][2]) - h_d_id = str(tuples[i][3]) - h_w_id = str(tuples[i][4]) - h_date = str(tuples[i][5]) - h_amount = str(tuples[i][6]) - h_data = str(tuples[i][7]) - col_fam.insert(row_key, {'H_C_ID':h_c_id, 'H_C_D_ID':h_c_d_id, 'H_C_W_ID':h_c_w_id, 'H_D_ID':h_d_id, 'H_W_ID':h_w_id, 'H_DATE':h_date,'H_AMOUNT':h_amount, 'H_DATA':h_data}) -# print tableName+'--' + str(len(tuples)) - - def loadFinish(self): - logging.info("Commiting changes to database") - - - - - ##----------------------------------- - ## doDelivery - ##---------------------------------- - def doDelivery(self, params): - logging.debug("do delivery") - - w_id = params["w_id"] - o_carrier_id = params["o_carrier_id"] - ol_delivery_d = params["ol_delivery_d"] - - - - result = [ ] - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): - did_expr = create_index_expression('NO_D_ID',str(d_id)) - wid_expr = create_index_expression('NO_W_ID',str(w_id)) - clause = create_index_clause([did_expr,wid_expr],count=1) - newOrder=self.new_ordercf.get_indexed_slices(clause) - flag=0 - for key, column in newOrder: - #print column - no_o_id=column['NO_O_ID'] - flag=1 - if flag==0: - continue - if int(no_o_id)<=-1: - continue - if no_o_id==None: - continue - # print no_o_id - # print d_id - # print w_id - orders_rowkey=no_o_id.zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) - #print orders_rowkey - o=self.orderscf.get(orders_rowkey) - - - c_id=str(o['O_C_ID']) - - oid_expr = create_index_expression('OL_O_ID',str(no_o_id)) - did_expr = create_index_expression('OL_D_ID',str(d_id)) - wid_expr = create_index_expression('OL_W_ID',str(w_id)) - - clause = create_index_clause([oid_expr,did_expr,wid_expr],count=100000) - orderLine=self.order_linecf.get_indexed_slices(clause) - - ol_total=0 - for key, column in orderLine: - ol_total+=float(column['OL_AMOUNT']) - - deleteKey=no_o_id.zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) - self.new_ordercf.remove(deleteKey) - self.orderscf.insert(deleteKey, {'O_CARRIER_ID': str(o_carrier_id)}) - self.order_linecf.insert(deleteKey,{'OL_DELIVERY_D':str(ol_delivery_d)}) - - c=self.customercf.get(str(c_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5)) - old_balance=float(c['C_BALANCE']) - new_balance=str(old_balance+ol_total) - self.customercf.insert(str(c_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5),{'C_BALANCE': str(new_balance)}) - - result.append((str(d_id),str(no_o_id))) - ##for - - return result - ##----------------------------------- - ## doNewOrder - ##----------------------------------- - - def doNewOrder(self, params): - logging.debug("do new order") - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - o_entry_d = params["o_entry_d"] - i_ids = params["i_ids"] - i_w_ids = params["i_w_ids"] - i_qtys = params["i_qtys"] - - - assert len(i_ids) > 0 - assert len(i_ids) ==len(i_w_ids) - assert len(i_ids) ==len(i_qtys) - - all_local = True - items = [ ] - for i in range(len(i_ids)): - all_local = all_local and i_w_ids[i] == w_id - ol_i_id = i_ids[i] - itm=self.itemcf.get(str(ol_i_id).zfill(5), columns=['I_PRICE','I_NAME','I_DATA']) - items.append(itm) - assert len(items)==len(i_ids) - - - for itm in items: - if len(itm)==0: - return - - - - - - #getWarehouseTaxRate - w_tax_c = self.warehousecf.get(str(w_id).zfill(5),columns=['W_TAX']) - w_tax =float(w_tax_c['W_TAX']) - #getDistrict - row_key = str(d_id).zfill(5) +str(w_id).zfill(5) - o=self.districtcf.get(row_key, columns=['D_TAX','D_NEXT_O_ID']) - d_tax = float(o['D_TAX']) - #incrementNextOrderId - d_next_o_id = int(o['D_NEXT_O_ID']) - - #getCustomer - row_key = str(c_id).zfill(5) +str(d_id).zfill(5)+str(w_id).zfill(5) - customer_info = self.customercf.get(row_key,columns=['C_DISCOUNT','C_LAST','C_CREDIT']) - c_discount=float(customer_info['C_DISCOUNT']) - - o_carrier_id = constants.NULL_CARRIER_ID - ol_cnt = len(i_ids) - - #incrementNextOrderId - row_key = str(d_id).zfill(5)+str(w_id).zfill(5) - self.districtcf.insert(row_key,{'D_NEXT_O_ID':str(d_next_o_id+1)}) - - #createOrder - - order_rowkey=str(d_next_o_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) - # print "d_next_o_id " +str(d_next_o_id) - # print "d_id "+str(d_id) - # print "order_rowkey " + order_rowkey - self.orderscf.insert(order_rowkey,{'O_ID':str(d_next_o_id), 'O_D_ID':str(d_id), 'O_W_ID':str(w_id), 'O_C_ID':str(c_id), 'O_ENTRY_D':str(o_entry_d), 'O_CARRIER_ID':str(o_carrier_id), 'O_OL_CNT':str(ol_cnt), 'O_ALL_LOCAL':str(all_local)}) - - #createNewOrder - neworder_rowkey=str(d_next_o_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) - # print 'neworder_rowkey ' + neworder_rowkey - self.new_ordercf.insert(neworder_rowkey, {'NO_O_ID':str(d_next_o_id), 'NO_D_ID':str(d_id), 'NO_W_ID':str(w_id)}) - #getItemInfo - total = 0 - item_data = [ ] - for i in range(len(i_ids)): - itemInfo = items[i] - i_name = itemInfo['I_NAME'] - i_data = itemInfo['I_DATA'] - i_price =float(itemInfo['I_PRICE']) - - #"getStockInfo": "SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DIST_%02d FROM STOCK WHERE S_I_ID = ? AND S_W_ID = ?", # d_id, ol_i_id, ol_supply_w_id - ol_i_id = i_ids[i] - ol_number = i+1 - ol_supply_w_id = i_w_ids[i] - ol_quantity = i_qtys[i] - - stockInfo = self.stockcf.get(str(ol_i_id).zfill(5)+str(ol_supply_w_id).zfill(5)) - #"updateStock": "UPDATE STOCK SET S_QUANTITY = ?, S_YTD = ?, S_ORDER_CNT = ?, S_REMOTE_CNT = ? WHERE S_I_ID = ? AND S_W_ID = ?", # s_quantity, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id - if len(stockInfo)==0: - logging.warn("No STOCK record for (ol_i_id=%d, ol_supply_w_id=%d)" % (ol_i_id, ol_supply_w_id)) - continue - s_quantity = int(stockInfo['S_QUANTITY']) - s_ytd = int(stockInfo['S_YTD']) - s_order_cnt = int(stockInfo['S_ORDER_CNT']) - s_remote_cnt = int(stockInfo['S_REMOTE_CNT']) - s_data = stockInfo['S_DATA'] - if d_id < 10: - s_dist_col='S_DIST_'+'0'+str(d_id) - else: - s_dist_col='S_DIST_'+str(d_id) - s_dist_xx = stockInfo[s_dist_col] - - - ## Update stock - s_ytd += ol_quantity - if s_quantity >= ol_quantity + 10: - s_quantity = s_quantity - ol_quantity - else: - s_quantity = s_quantity + 91 - ol_quantity - s_order_cnt += 1 - if ol_supply_w_id != w_id: s_remote_cnt += 1 - self.stockcf.insert(str(ol_i_id).zfill(5)+str(ol_supply_w_id).zfill(5),{'S_QUANTITY':str(s_quantity), 'S_YTD':str(s_ytd), 'S_ORDER_CNT':str(s_order_cnt) , 'S_REMOTE_CNT':str(s_remote_cnt)}) - - ##"createOrderLine": "INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", # o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info - if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING)!= -1: - brand_generic = 'B' - else: - brand_generic = 'G' - ol_amount = ol_quantity * i_price - total += ol_amount - - orderline_rowkey=str(d_next_o_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) - self.order_linecf.insert(orderline_rowkey,{'OL_O_ID': str(d_next_o_id), 'OL_D_ID':str(d_id), 'OL_W_ID':str(w_id), 'OL_NUMBER':str(ol_number), 'OL_I_ID':str(ol_i_id), 'OL_SUPPLY_W_ID':str(ol_supply_w_id), 'OL_DELIVERY_D': str(o_entry_d), 'OL_QUANTITY':str(ol_quantity),'OL_AMOUNT':str(ol_amount), 'OL_DIST_INFO':str(s_dist_xx)}) - item_data.append( (i_name, s_quantity, brand_generic,i_price, ol_amount) ) - total *= (1 - c_discount) * (1 + w_tax + d_tax) - misc = [ (w_tax, d_tax, d_next_o_id, total) ] - return [ customer_info, misc, item_data ] - ##---------------------------- - ## doPayment - ##---------------------------- - - def doPayment(self, params): - logging.debug("do payment") - w_id = params["w_id"] - d_id = params["d_id"] - h_amount = params["h_amount"] - c_w_id = params["c_w_id"] - c_d_id = params["c_d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - h_date = params["h_date"] - - - if c_id != None: - #getCustomerByCustomerId - row_key = str(c_id).zfill(5) +str(d_id).zfill(5)+str(w_id).zfill(5) - customer = self.customercf.get(row_key) - assert len(customer)>0 - c_balance = float(str(customer['C_BALANCE']))- h_amount - c_ytd_payment = float(str(customer['C_YTD_PAYMENT'])) + h_amount - c_payment_cnt = int(str(customer['C_PAYMENT_CNT']))+1 - c_data = str(customer['C_DATA']) - c_credit = str(customer['C_CREDIT']) - else: - #getCustomerByLastName - c_1_expr = create_index_expression('C_W_ID',str(w_id)) - c_2_expr = create_index_expression('C_D_ID',str(d_id)) - c_3_expr = create_index_expression('C_LAST',str(c_last)) - clause = create_index_clause([c_1_expr,c_2_expr,c_3_expr],count=1000) - - newcustomer = self.customercf.get_indexed_slices(clause) - firstnames=[] - - namecnt=0 - for key, column in newcustomer: - firstnames.append(column['C_FIRST']) - namecnt+=1 - # print namecnt - index = (namecnt-1)/2 - firstname=firstnames[index] - c_4_expr = create_index_expression('C_LAST',str(c_last)) - clause = create_index_clause([c_1_expr,c_2_expr,c_3_expr,c_4_expr],count=1) - newcustomer = self.customercf.get_indexed_slices(clause) - for key, column in newcustomer: - c_id = column['C_ID'] - c_balance = float(column['C_BALANCE'])- h_amount - c_ytd_payment = float(column['C_YTD_PAYMENT']) + h_amount - c_payment_cnt = int(column['C_PAYMENT_CNT'])+1 - c_data = column['C_DATA'] - c_credit =column['C_CREDIT'] - row_key = str(c_id).zfill(5) +str(d_id).zfill(5)+str(w_id).zfill(5) - customer = self.customercf.get(row_key) - warehouse = self.warehousecf.get(str(w_id).zfill(5)) - district = self.districtcf.get(str(d_id).zfill(5)+str(w_id).zfill(5)) - - self.warehousecf.insert(str(w_id).zfill(5),{'W_YTD':str(float(warehouse['W_YTD'])+h_amount)}) - - self.districtcf.insert(str(d_id).zfill(5)+str(w_id).zfill(5),{'D_YTD': str(float(district['D_YTD'])+h_amount)}) - - if c_credit == constants.BAD_CREDIT: - newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) - c_data = (newData + "|" + c_data) - if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] - self.customercf.insert(str(c_id).zfill(5)+str(c_d_id).zfill(5)+str(c_w_id).zfill(5),{ 'C_BALANCE' : str(c_balance), 'C_YTD_PAYMENT':str(c_ytd_payment) , 'C_PAYMENT_CNT':str(c_payment_cnt), 'C_DATA' : str(c_data)}) - else: - c_data = "" - self.customercf.insert(str(c_id).zfill(5)+str(c_d_id).zfill(5)+str(c_w_id).zfill(5),{ 'C_BALANCE' : str(c_balance), 'C_YTD_PAYMENT':str(c_ytd_payment) , 'C_PAYMENT_CNT':str(c_payment_cnt)}) - h_data= "%s %s" % (warehouse['W_NAME'], district['D_NAME']) - self.historycf.insert(str(uuid.uuid1()), {'H_C_ID':str(c_id), 'H_C_D_ID':str(c_d_id), 'H_C_W_ID':str(c_w_id), 'H_D_ID':str(d_id), 'H_W_ID':str(w_id), 'H_DATE':str(h_date),'H_AMOUNT':str(h_amount), 'H_DATA':str(h_data)}) - return [warehouse, district, customer] - ##----------------------------------- - ## doOrderStatus - ##----------------------------------- - def doOrderStatus(self, params): - logging.info("do orderStatus") - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - - assert w_id, pformat(params) - assert d_id, pformat(params) - - - if c_id == None: - last_expr = create_index_expression('C_LAST',str(c_last)) - did_expr = create_index_expression('C_D_ID',str(d_id)) - wid_expr = create_index_expression('C_W_ID',str(w_id)) - clause = create_index_clause([last_expr,did_expr,wid_expr],count=10000) - all_customers=self.customercf.get_indexed_slices(clause) - first_names=[ ] - c_ids=[] - namecnt=0 - for key, column in all_customers: - first_names.append(column['C_FIRST']) - c_ids.append(column['C_ID']) - namecnt = namecnt+1 - namecnt=len(first_names) - assert namecnt>0 - index=(namecnt-1)/2 - first_name=first_names[index] - assert first_name!=None - c_id=c_ids[index] - assert c_id!=None - - key1=str(c_id).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5) - res1=self.customercf.get(key1) - customer=[res1['C_ID'],res1['C_FIRST'],res1['C_MIDDLE'],res1['C_LAST'],res1['C_BALANCE']] - - cid_expr = create_index_expression('O_C_ID',str(c_id)) - did_expr = create_index_expression('O_D_ID',str(d_id)) - wid_expr = create_index_expression('O_W_ID',str(w_id)) - clause = create_index_clause([cid_expr,did_expr,wid_expr],count=100000) - all_orders=self.orderscf.get_indexed_slices(clause) - - last_order_oid=0 - order=[] - for key, column in all_orders: - if int(column['O_ID'])>last_order_oid: - last_order_oid=int(column['O_ID']) - if last_order_oid>0: - o=self.orderscf.get(str(last_order_oid).zfill(5)+str(d_id).zfill(5)+str(w_id).zfill(5)) - order=[o['O_ID'],o['O_CARRIER_ID'],o['O_ENTRY_D']] - - - orderLines = [] - if last_order_oid>0: - oid_expr = create_index_expression('OL_O_ID',str(last_order_oid)) - did_expr = create_index_expression('OL_D_ID',str(d_id)) - wid_expr = create_index_expression('OL_W_ID',str(w_id)) - clause = create_index_clause([oid_expr,did_expr,wid_expr]) - orderLine=self.order_linecf.get_indexed_slices(clause) - for key, column in orderLine: - orderLines.append([column['OL_SUPPLY_W_ID'],column['OL_I_ID'],column['OL_QUANTITY'],column['OL_AMOUNT'],column['OL_DELIVERY_D']]) - - return [ customer, order, orderLines ] - - ##---------------------------- - ## doStockLevel - ##---------------------------- - - - def doStockLevel(self, params): - logging.info("do stocklevel") - w_id = params["w_id"] - d_id = params["d_id"] - threshold = params["threshold"] - - - #"getOId": "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", - d = self.districtcf.get(str(d_id).zfill(5)+str(w_id).zfill(5),columns=['D_NEXT_O_ID']) - assert d - #getStockCount - o_id = d['D_NEXT_O_ID'] - - - s_q_expr = create_index_expression('S_QUANTITY',str(threshold), LT) - s_q_expr2 = create_index_expression('S_W_ID',str(w_id)) - clause = create_index_clause([s_q_expr,s_q_expr2]) - newstock = self.stockcf.get_indexed_slices(clause) - - - ol_expr = create_index_expression('OL_W_ID',str(w_id)) - ol_expr2 = create_index_expression('OL_D_ID',str(d_id)) - ol_expr3 = create_index_expression('OL_O_ID',str(o_id),LT) - ol_expr4 = create_index_expression('OL_O_ID', str(int(o_id)-20),GTE) - clause2 = create_index_clause([ol_expr,ol_expr2]) - neworderline = self.order_linecf.get_indexed_slices(clause2) - - count = 0 - for key, column in newstock: - for key2, column2 in neworderline: - tmp1 = column['S_I_ID'] - s_i_id = int(tmp1) - tmp2 = column2['OL_I_ID'] - ol_i_id = int(tmp2) - if s_i_id == ol_i_id: - count= count+1 - - return count - diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/hbasedriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/hbasedriver.py deleted file mode 100644 index 3eb5bf9e..00000000 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/hbasedriver.py +++ /dev/null @@ -1,1408 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------- -# HBase version: -# Copyright (C) 2011 -# Zikai Wang -# galaxyngc1309@gmail.com -# -# Original SQLite version: -# Copyright (C) 2011 -# Andy Pavlo -# http://www.cs.brown.edu/~pavlo/ -# -# Original Java Version: -# Copyright (C) 2008 -# Evan Jones -# Massachusetts Institute of Technology -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# ----------------------------------------------------------------------- -import uuid -import constants -from java.lang import Integer, Float, String -from org.apache.hadoop.hbase import HBaseConfiguration, HTableDescriptor, HColumnDescriptor -from org.apache.hadoop.hbase.client import HBaseAdmin, HTable, Put, Get, Scan, Delete, Result, ResultScanner -from org.apache.hadoop.hbase.util import Bytes -from org.apache.hadoop.hbase.filter import PrefixFilter -from abstractdriver import AbstractDriver -from pprint import pprint,pformat - - -## ============================================== -## HBase Tables Layout -## ============================================== -## TABLE WAREHOUSE -## W_ID W_NAME W_STREET_1 W_STREET_2 W_CITY W_STATE W_ZIP W_TAX W_YTD -## Row key: W_ID -## Column Family 1: W_NAME W_STREET_1 W_STREET_2 W_CITY W_STATE W_ZIP -## Column Family 2: W_TAX -## Column Family 3: W_YTD - -## TABLE DISTRICT -## D_ID D_W_ID D_NAME D_STREET_1 D_STREET_2 D_CITY D_STATE_ D_ZIP D_TAX D_YTD D_NEXT_O_ID -## Row key: D_W_ID D_ID -## Column Family 1: D_TAX D_NEXT_O_ID -## Column Family 2: D_NAME D_STREET_1 D_STREET_2 D_CITY D_STATE D_ZIP -## Column Family 3: D_YTD - -## TABLE CUSTOMER -## C_ID C_D_ID C_W_ID C_FIRST C_MIDDLE C_LAST C_STREET_1 C_STREET_2 C_CITY #C_STATE C_ZIP C_PHONE C_SINCE C_CREDIT C_CREDIT_LIM C_DISCOUNT C_BALANCE C_YTD_PAYMENT C_PAYMENT_CNT C_DELIVERY_CNT C_DATA -## Row key: C_W_ID C_D_ID C_ID -## Column Family 1: C_FIRST C_MIDDLE C_LAST C_STREET_1 C_STREET_2 C_CITY #C_STATE C_ZIP C_PHONE C_SINCE C_CREDIT C_CREDIT_LIM C_DISCOUNT C_BALANCE C_YTD_PAYMENT C_PAYMENT_CNT C_DATA -## Colmn Family 2: C_DELIVERY_CNT - -## TABLE HISTORY -## H_C_ID H_C_D_ID H_C_W_ID H_D_ID H_W_ID H_DATE H_AMOUNT H_DATA -## Row key: UUID -## Column Family 1: H_C_ID H_C_D_ID H_C_W_ID H_D_ID H_W_ID H_DATE H_AMOUNT H_DATA - -## TABLE NEW_ORDER -## NO_O_ID NO_D_ID NO_W_ID -## Row key: (NO_W_ID NO_D_ID NO_O_ID) -## Column Family 1: (NO_O_ID NO_D_ID NO_W_ID) - -## TABLE ORDERS -## O_ID O_D_ID O_W_ID O_C_ID O_ENTRY_D O_CARRIER_ID O_OL_CNT O_ALL_LOCAL -## Row key: O_W_ID O_D_ID O_ID -## Column Family 1: (O_C_ID O_ENTRY_D O_CARRIER_ID) -## Column Family 2: (O_OL_CNT O_ALL_LOCAL) - -## TABLE ORDER_LINE -## OL_O_ID OL_D_ID OL_W_ID OL_NUMBER OL_I_ID OL_SUPPLY_W_ID OL_DELIVERY_D OL_QUANTITY OL_AMOUNT OL_DIST_INFO -## Row key: OL_W_ID OL_D_ID OL_O_ID OL_NUMBER -## Column Family 1: OL_I_ID OL_SUPPLY_W_ID OL_DELIVERY_D OL_QUANTITY OL_AMOUNT -## Column Family 2: OL_DIST_INFO - -## TABLE ITEM -## I_ID I_IM_ID I_NAME I_PRICE I_DATA -## Row key: I_ID -## Column Family1: I_NAME I_PRICE I_DATA -## Column Family2: I_IM_ID - -## TABLE STOCK -## S_I_ID S_W_ID S_QUANTITY S_DIST_01 S_DIST_02 S_DIST_03 S_DIST_04 S_DIST_05 S_DIST_06 S_DIST_07 S_DIST_08 S_DIST_09 S_DIST_10 S_YTD S_ORDER_CNT S_REMOTE_CNT S_DATA -## Row key: S_W_ID S_I_ID -## Column Family1: S_QUANTITY S_DIST_01 S_DIST_02 S_DIST_03 S_DIST_04 S_DIST_05 S_DIST_06 S_DIST_07 S_DIST_08 S_DIST_09 S_DIST_10 S_YTD S_ORDER_CNT S_REMOTE_CNT S_DATA - -## Note that HBase cell values are uninterpreted bytes. Therefore, TPC-C values are serialized from their original types with org.apache.hadoop.hbase.util.Bytes package - -## TPC-C tables -TABLES = ["WAREHOUSE", "DISTRICT", "CUSTOMER", "HISTORY", "NEW_ORDER", "ORDERS", "ORDER_LINE", "ITEM", "STOCK"] - -## column qualifiers for all columns in TPC-C tables -COLUMN_NAME = { - "W_ID" : Bytes.toBytes("a"), - "W_NAME" : Bytes.toBytes("b"), - "W_STREET_1" : Bytes.toBytes("c"), - "W_STREET_2" : Bytes.toBytes("d"), - "W_CITY" : Bytes.toBytes("e"), - "W_STATE" : Bytes.toBytes("f"), - "W_ZIP" : Bytes.toBytes("g"), - "W_TAX" : Bytes.toBytes("h"), - "W_YTD" : Bytes.toBytes("i"), - "D_ID" : Bytes.toBytes("a"), - "D_W_ID" : Bytes.toBytes("b"), - "D_NAME" : Bytes.toBytes("c"), - "D_STREET_1" : Bytes.toBytes("d"), - "D_STREET_2" : Bytes.toBytes("e"), - "D_CITY" : Bytes.toBytes("f"), - "D_STATE" : Bytes.toBytes("g"), - "D_ZIP" : Bytes.toBytes("h"), - "D_TAX" : Bytes.toBytes("i"), - "D_YTD" : Bytes.toBytes("j"), - "D_NEXT_O_ID" : Bytes.toBytes("k"), - "C_ID" : Bytes.toBytes("a"), - "C_D_ID" : Bytes.toBytes("b"), - "C_W_ID" : Bytes.toBytes("c"), - "C_FIRST" : Bytes.toBytes("d"), - "C_MIDDLE" : Bytes.toBytes("e"), - "C_LAST" : Bytes.toBytes("f"), - "C_STREET_1" : Bytes.toBytes("g"), - "C_STREET_2" : Bytes.toBytes("h"), - "C_CITY" : Bytes.toBytes("i"), - "C_STATE" : Bytes.toBytes("j"), - "C_ZIP" : Bytes.toBytes("k"), - "C_PHONE" : Bytes.toBytes("l"), - "C_SINCE" : Bytes.toBytes("m"), - "C_CREDIT" : Bytes.toBytes("n"), - "C_CREDIT_LIM" : Bytes.toBytes("o"), - "C_DISCOUNT" : Bytes.toBytes("p"), - "C_BALANCE" : Bytes.toBytes("q"), - "C_YTD_PAYMENT" : Bytes.toBytes("r"), - "C_PAYMENT_CNT" : Bytes.toBytes("s"), - "C_DELIVERY_CNT" : Bytes.toBytes("t"), - "C_DATA" : Bytes.toBytes("u"), - "H_C_ID" : Bytes.toBytes("a"), - "H_C_D_ID" : Bytes.toBytes("b"), - "H_C_W_ID" : Bytes.toBytes("c"), - "H_D_ID" : Bytes.toBytes("d"), - "H_W_ID" : Bytes.toBytes("e"), - "H_DATE" : Bytes.toBytes("f"), - "H_AMOUNT" : Bytes.toBytes("g"), - "H_DATA" : Bytes.toBytes("h"), - "NO_O_ID" : Bytes.toBytes("a"), - "NO_D_ID" : Bytes.toBytes("b"), - "NO_W_ID" : Bytes.toBytes("c"), - "O_ID" : Bytes.toBytes("a"), - "O_D_ID" : Bytes.toBytes("b"), - "O_W_ID" : Bytes.toBytes("c"), - "O_C_ID" : Bytes.toBytes("d"), - "O_ENTRY_D" : Bytes.toBytes("e"), - "O_CARRIER_ID" : Bytes.toBytes("f"), - "O_OL_CNT" : Bytes.toBytes("g"), - "O_ALL_LOCAL" : Bytes.toBytes("h"), - "OL_O_ID" : Bytes.toBytes("a"), - "OL_D_ID" : Bytes.toBytes("b"), - "OL_W_ID" : Bytes.toBytes("c"), - "OL_NUMBER" : Bytes.toBytes("d"), - "OL_I_ID" : Bytes.toBytes("e"), - "OL_SUPPLY_W_ID" : Bytes.toBytes("f"), - "OL_DELIVERY_D" : Bytes.toBytes("g"), - "OL_QUANTITY" : Bytes.toBytes("h"), - "OL_AMOUNT" : Bytes.toBytes("i"), - "OL_DIST_INFO" : Bytes.toBytes("j"), - "I_ID" : Bytes.toBytes("a"), - "I_IM_ID" : Bytes.toBytes("b"), - "I_NAME" : Bytes.toBytes("c"), - "I_PRICE" : Bytes.toBytes("d"), - "I_DATA" : Bytes.toBytes("e"), - "S_I_ID" : Bytes.toBytes("a"), - "S_W_ID" : Bytes.toBytes("b"), - "S_QUANTITY" : Bytes.toBytes("c"), - "S_DIST_01" : Bytes.toBytes("d"), - "S_DIST_02" : Bytes.toBytes("e"), - "S_DIST_03" : Bytes.toBytes("f"), - "S_DIST_04" : Bytes.toBytes("g"), - "S_DIST_05" : Bytes.toBytes("h"), - "S_DIST_06" : Bytes.toBytes("i"), - "S_DIST_07" : Bytes.toBytes("j"), - "S_DIST_08" : Bytes.toBytes("k"), - "S_DIST_09" : Bytes.toBytes("l"), - "S_DIST_10" : Bytes.toBytes("m"), - "S_YTD" : Bytes.toBytes("n"), - "S_ORDER_CNT" : Bytes.toBytes("o"), - "S_REMOTE_CNT" : Bytes.toBytes("p"), - "S_DATA" : Bytes.toBytes("q"), -} - - -## ============================================== -## Optimization Options -## ============================================== - -## do not write to WAL for PUT -WRITE_TO_WAL = False -## a scanner will pre-fetch 100 tuples at once -SCANNER_CACHING = 100 -## turn off auto flushing for PUT -AUTOFLUSH = False -## in-mem column family -DATA_IN_MEM = True - -## ============================================== -## HBaseDriver -## ============================================== -class HbaseDriver(AbstractDriver): - DEFAULT_CONFIG = {} - - def __init__(self, ddl): - super(HbaseDriver, self).__init__("hbase", ddl) - self.config = None - self.admin = None - - self.warehouse_tbl = None - self.district_tbl = None - self.customer_tbl = None - self.history_tbl = None - self.new_order_tbl = None - self.order_tbl = None - self.order_line_tbl = None - self.item_tbl = None - self.stock_tbl = None - - ## column families for all columns in TPC-C tables - self.col_fam1 = Bytes.toBytes("1") - self.col_fam2 = Bytes.toBytes("2") - self.col_fam3 = Bytes.toBytes("3") - - def makeDefaultConfig(self): - return HbaseDriver.DEFAULT_CONFIG - - def loadConfig(self, config): - pass - - ## ============================================== - ## loadStart - ## ============================================== - def loadStart(self): - ## get HBase client - self.config = HBaseConfiguration.create() - self.admin = HBaseAdmin(self.config) - - ## Drop tables if they already exist - for table in TABLES: - if self.admin.tableExists(table) and self.admin.isTableAvailable(table): - if self.admin.isTableEnabled(table): - self.admin.disableTable(table) - self.admin.deleteTable(table) - ## FOR - - ## CREATE TABLE WAREHOUSE - htd = HTableDescriptor("WAREHOUSE") - hcd = HColumnDescriptor("1") - hcd.setInMemory(DATA_IN_MEM) - htd.addFamily(hcd) - hcd = HColumnDescriptor("2") - hcd.setInMemory(DATA_IN_MEM) - htd.addFamily(hcd) - hcd = HColumnDescriptor("3") - hcd.setInMemory(DATA_IN_MEM) - htd.addFamily(hcd) - self.admin.createTable(htd) - - ## CREATE TABLE DISTRICT - htd = HTableDescriptor("DISTRICT") - hcd = HColumnDescriptor("1") - hcd.setInMemory(DATA_IN_MEM) - htd.addFamily(hcd) - hcd = HColumnDescriptor("2") - hcd.setInMemory(DATA_IN_MEM) - htd.addFamily(hcd) - hcd = HColumnDescriptor("3") - hcd.setInMemory(DATA_IN_MEM) - htd.addFamily(hcd) - self.admin.createTable(htd) - - ## CREATE TABLE CUSTOMER - htd = HTableDescriptor("CUSTOMER") - hcd = HColumnDescriptor("1") - htd.addFamily(hcd) - htd.addFamily(HColumnDescriptor("2")) - self.admin.createTable(htd) - - ## CREATE TABLE HISTORY - htd = HTableDescriptor("HISTORY") - htd.addFamily(HColumnDescriptor("1")) - self.admin.createTable(htd) - - ## CREATE TABLE NEW_ORDER - htd = HTableDescriptor("NEW_ORDER") - hcd = HColumnDescriptor("1") - htd.addFamily(hcd) - self.admin.createTable(htd) - - ## CREATE TABLE ORDERS - htd = HTableDescriptor("ORDERS") - hcd = HColumnDescriptor("1") - htd.addFamily(hcd) - htd.addFamily(HColumnDescriptor("2")) - self.admin.createTable(htd) - - ## CREATE TABLE ORDER_LINE - htd = HTableDescriptor("ORDER_LINE") - hcd = HColumnDescriptor("1") - htd.addFamily(hcd) - htd.addFamily(HColumnDescriptor("2")) - self.admin.createTable(htd) - - ## CREATE TABLE ITEM - htd = HTableDescriptor("ITEM") - htd.addFamily(HColumnDescriptor("1")) - htd.addFamily(HColumnDescriptor("2")) - self.admin.createTable(htd) - - ## CREATE TABLE STOCK - htd = HTableDescriptor("STOCK") - htd.addFamily(HColumnDescriptor("1")) - self.admin.createTable(htd) - - ## get handlers to all tables - self.warehouse_tbl = HTable(self.config, "WAREHOUSE") - self.warehouse_tbl.setAutoFlush(AUTOFLUSH) - self.district_tbl = HTable(self.config, "DISTRICT") - self.district_tbl.setAutoFlush(AUTOFLUSH) - self.customer_tbl = HTable(self.config, "CUSTOMER") - self.customer_tbl.setAutoFlush(AUTOFLUSH) - self.history_tbl = HTable(self.config, "HISTORY") - self.history_tbl.setAutoFlush(AUTOFLUSH) - self.new_order_tbl = HTable(self.config, "NEW_ORDER") - self.new_order_tbl.setAutoFlush(AUTOFLUSH) - self.order_tbl = HTable(self.config, "ORDERS") - self.order_tbl.setAutoFlush(AUTOFLUSH) - self.order_line_tbl = HTable(self.config, "ORDER_LINE") - self.order_line_tbl.setAutoFlush(AUTOFLUSH) - self.item_tbl = HTable(self.config, "ITEM") - self.item_tbl.setAutoFlush(AUTOFLUSH) - self.stock_tbl = HTable(self.config, "STOCK") - self.stock_tbl.setAutoFlush(AUTOFLUSH) - - ## ============================================== - ## loadFinish - ## ============================================== - def loadFinish(self): - ## close handlers to all tables - self.warehouse_tbl.close() - self.district_tbl.close() - self.customer_tbl.close() - self.history_tbl.close() - self.new_order_tbl.close() - self.order_tbl.close() - self.order_line_tbl.close() - self.item_tbl.close() - self.stock_tbl.close() - - ## ============================================== - ## loadTuples - ## ============================================== - def loadTuples(self, tableName, tuples): - ## load TPC-C data - if tableName == "WAREHOUSE": - for tuplei in tuples: - w_id = Bytes.toBytes(Integer(tuplei[0])) - w_name = Bytes.toBytes(String(tuplei[1])) - w_street_1 = Bytes.toBytes(String(tuplei[2])) - w_street_2 = Bytes.toBytes(String(tuplei[3])) - w_city = Bytes.toBytes(String(tuplei[4])) - w_state = Bytes.toBytes(String(tuplei[5])) - w_zip = Bytes.toBytes(String(tuplei[6])) - w_tax = Bytes.toBytes(Float(tuplei[7])) - w_ytd = Bytes.toBytes(Float(tuplei[8])) - - row_key = w_id - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["W_NAME"], w_name) - put.add(self.col_fam1, COLUMN_NAME["W_STREET_1"], w_street_1) - put.add(self.col_fam1, COLUMN_NAME["W_STREET_2"], w_street_2) - put.add(self.col_fam1, COLUMN_NAME["W_CITY"], w_city) - put.add(self.col_fam1, COLUMN_NAME["W_STATE"], w_state) - put.add(self.col_fam1, COLUMN_NAME["W_ZIP"], w_zip) - put.add(self.col_fam2, COLUMN_NAME["W_TAX"], w_tax) - put.add(self.col_fam3, COLUMN_NAME["W_YTD"], w_ytd) - self.warehouse_tbl.put(put) - ## FOR - - elif tableName == "DISTRICT": - for tuplei in tuples: - d_id = Bytes.toBytes(Integer(tuplei[0])) - d_w_id = Bytes.toBytes(Integer(tuplei[1])) - d_name = Bytes.toBytes(String(tuplei[2])) - d_street_1 = Bytes.toBytes(String(tuplei[3])) - d_street_2 = Bytes.toBytes(String(tuplei[4])) - d_city = Bytes.toBytes(String(tuplei[5])) - d_state = Bytes.toBytes(String(tuplei[6])) - d_zip = Bytes.toBytes(String(tuplei[7])) - d_tax = Bytes.toBytes(Float(tuplei[8])) - d_ytd = Bytes.toBytes(Float(tuplei[9])) - d_next_o_id = Bytes.toBytes(Integer(tuplei[10])) - - row_key = d_w_id - row_key.append(0) - row_key.extend(d_id) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["D_TAX"], d_tax) - put.add(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"], d_next_o_id) - put.add(self.col_fam2, COLUMN_NAME["D_NAME"], d_name) - put.add(self.col_fam2, COLUMN_NAME["D_STREET_1"], d_street_1) - put.add(self.col_fam2, COLUMN_NAME["D_STREET_2"], d_street_2) - put.add(self.col_fam2, COLUMN_NAME["D_CITY"], d_city) - put.add(self.col_fam2, COLUMN_NAME["D_STATE"], d_state) - put.add(self.col_fam2, COLUMN_NAME["D_ZIP"], d_zip) - put.add(self.col_fam3, COLUMN_NAME["D_YTD"], d_ytd) - self.district_tbl.put(put) - ## FOR - - - elif tableName == "CUSTOMER": - for tuplei in tuples: - c_id = Bytes.toBytes(Integer(tuplei[0])) - c_d_id = Bytes.toBytes(Integer(tuplei[1])) - c_w_id = Bytes.toBytes(Integer(tuplei[2])) - c_first = Bytes.toBytes(String(tuplei[3])) - c_middle = Bytes.toBytes(String(tuplei[4])) - c_last = Bytes.toBytes(String(tuplei[5])) - c_street_1 = Bytes.toBytes(String(tuplei[6])) - c_street_2 = Bytes.toBytes(String(tuplei[7])) - c_city = Bytes.toBytes(String(tuplei[8])) - c_state = Bytes.toBytes(String(tuplei[9])) - c_zip = Bytes.toBytes(String(tuplei[10])) - c_phone = Bytes.toBytes(String(tuplei[11])) - c_since = Bytes.toBytes(String(str(tuplei[12]))) - c_credit = Bytes.toBytes(String(tuplei[13])) - c_credit_lim = Bytes.toBytes(Float(tuplei[14])) - c_discount = Bytes.toBytes(Float(tuplei[15])) - c_balance = Bytes.toBytes(Float(tuplei[16])) - c_ytd_payment = Bytes.toBytes(Float(tuplei[17])) - c_payment_cnt = Bytes.toBytes(Integer(tuplei[18])) - c_delivery_cnt = Bytes.toBytes(Integer(tuplei[19])) - c_data = Bytes.toBytes(String(tuplei[20])) - - row_key = c_w_id - row_key.append(0) - row_key.extend(c_d_id) - row_key.append(0) - row_key.extend(c_id) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["C_FIRST"], c_first) - put.add(self.col_fam1, COLUMN_NAME["C_MIDDLE"], c_middle) - put.add(self.col_fam1, COLUMN_NAME["C_LAST"], c_last) - put.add(self.col_fam1, COLUMN_NAME["C_STREET_1"], c_street_1) - put.add(self.col_fam1, COLUMN_NAME["C_STREET_2"], c_street_2) - put.add(self.col_fam1, COLUMN_NAME["C_CITY"], c_city) - put.add(self.col_fam1, COLUMN_NAME["C_STATE"], c_state) - put.add(self.col_fam1, COLUMN_NAME["C_ZIP"], c_zip) - put.add(self.col_fam1, COLUMN_NAME["C_PHONE"], c_phone) - put.add(self.col_fam1, COLUMN_NAME["C_SINCE"], c_since) - put.add(self.col_fam1, COLUMN_NAME["C_CREDIT"], c_credit) - put.add(self.col_fam1, COLUMN_NAME["C_CREDIT_LIM"], c_credit_lim) - put.add(self.col_fam1, COLUMN_NAME["C_DISCOUNT"], c_discount) - put.add(self.col_fam1, COLUMN_NAME["C_BALANCE"], c_balance) - put.add(self.col_fam1, COLUMN_NAME["C_YTD_PAYMENT"], c_ytd_payment) - put.add(self.col_fam1, COLUMN_NAME["C_PAYMENT_CNT"], c_payment_cnt) - put.add(self.col_fam1, COLUMN_NAME["C_DATA"], c_data) - put.add(self.col_fam2, COLUMN_NAME["C_DELIVERY_CNT"], c_delivery_cnt) - self.customer_tbl.put(put) - ## FOR - - elif tableName == "HISTORY": - for tuplei in tuples: - h_c_id = Bytes.toBytes(Integer(tuplei[0])) - h_c_d_id = Bytes.toBytes(Integer(tuplei[1])) - h_c_w_id = Bytes.toBytes(Integer(tuplei[2])) - h_d_id = Bytes.toBytes(Integer(tuplei[3])) - h_w_id = Bytes.toBytes(Integer(tuplei[4])) - h_date = Bytes.toBytes(String(str(tuplei[5]))) - h_amount = Bytes.toBytes(Float(tuplei[6])) - h_data = Bytes.toBytes(String(tuplei[7])) - - row_key = str(uuid.uuid1()) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["H_C_ID"], h_c_id) - put.add(self.col_fam1, COLUMN_NAME["H_C_D_ID"], h_c_d_id) - put.add(self.col_fam1, COLUMN_NAME["H_C_W_ID"], h_c_w_id) - put.add(self.col_fam1, COLUMN_NAME["H_D_ID"], h_d_id) - put.add(self.col_fam1, COLUMN_NAME["H_W_ID"], h_w_id) - put.add(self.col_fam1, COLUMN_NAME["H_DATE"], h_date) - put.add(self.col_fam1, COLUMN_NAME["H_AMOUNT"], h_amount) - put.add(self.col_fam1, COLUMN_NAME["H_DATA"], h_data) - self.history_tbl.put(put) - ## FOR - - elif tableName == "NEW_ORDER": - for tuplei in tuples: - no_o_id = Bytes.toBytes(Integer(tuplei[0])) - no_d_id = Bytes.toBytes(Integer(tuplei[1])) - no_w_id = Bytes.toBytes(Integer(tuplei[2])) - - row_key = no_w_id - row_key.append(0) - row_key.extend(no_d_id) - row_key.append(0) - row_key.extend(no_o_id) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["NO_O_ID"], no_o_id) - put.add(self.col_fam1, COLUMN_NAME["NO_D_ID"], no_d_id) - put.add(self.col_fam1, COLUMN_NAME["NO_W_ID"], no_w_id) - self.new_order_tbl.put(put) - ## FOR - - elif tableName == "ORDERS": - for tuplei in tuples: - o_id = Bytes.toBytes(Integer(tuplei[0])) - o_d_id = Bytes.toBytes(Integer(tuplei[2])) - o_w_id = Bytes.toBytes(Integer(tuplei[3])) - o_c_id = Bytes.toBytes(Integer(tuplei[1])) - o_entry_d = Bytes.toBytes(String(str(tuplei[4]))) - o_carrier_id = Bytes.toBytes(String(str(tuplei[5]))) - o_ol_cnt = Bytes.toBytes(Integer(tuplei[6])) - o_all_local = Bytes.toBytes(Integer(tuplei[7])) - - row_key = o_w_id - row_key.append(0) - row_key.extend(o_d_id) - row_key.append(0) - row_key.extend(o_id) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["O_C_ID"], o_c_id) - put.add(self.col_fam1, COLUMN_NAME["O_ENTRY_D"], o_entry_d) - put.add(self.col_fam1, COLUMN_NAME["O_CARRIER_ID"], o_carrier_id) - put.add(self.col_fam2, COLUMN_NAME["O_OL_CNT"], o_ol_cnt) - put.add(self.col_fam2, COLUMN_NAME["O_ALL_LOCAL"], o_all_local) - self.order_tbl.put(put) - ## FOR - - elif tableName == "ORDER_LINE": - for tuplei in tuples: - ol_o_id= Bytes.toBytes(Integer(tuplei[0])) - ol_d_id = Bytes.toBytes(Integer(tuplei[1])) - ol_w_id = Bytes.toBytes(Integer(tuplei[2])) - ol_number = Bytes.toBytes(Integer(tuplei[3])) - ol_i_id = Bytes.toBytes(Integer(tuplei[4])) - ol_supply_w_id = Bytes.toBytes(Integer(tuplei[5])) - ol_delivery_d = Bytes.toBytes(String(str(tuplei[6]))) - ol_quantity = Bytes.toBytes(Integer(tuplei[7])) - ol_amount = Bytes.toBytes(Float(tuplei[8])) - ol_dist_info = Bytes.toBytes(String(tuplei[9])) - - row_key = ol_w_id - row_key.append(0) - row_key.extend(ol_d_id) - row_key.append(0) - row_key.extend(ol_o_id) - row_key.append(0) - row_key.extend(ol_number) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["OL_I_ID"], ol_i_id) - put.add(self.col_fam1, COLUMN_NAME["OL_SUPPLY_W_ID"], ol_supply_w_id) - put.add(self.col_fam1, COLUMN_NAME["OL_DELIVERY_D"], ol_delivery_d) - put.add(self.col_fam1, COLUMN_NAME["OL_QUANTITY"], ol_quantity) - put.add(self.col_fam1, COLUMN_NAME["OL_AMOUNT"], ol_amount) - put.add(self.col_fam2, COLUMN_NAME["OL_DIST_INFO"], ol_dist_info) - self.order_line_tbl.put(put) - ## FOR - - elif tableName == "ITEM": - for tuplei in tuples: - i_id = Bytes.toBytes(Integer(tuplei[0])) - i_im_id = Bytes.toBytes(Integer(tuplei[1])) - i_name = Bytes.toBytes(String(tuplei[2])) - i_price = Bytes.toBytes(Float(tuplei[3])) - i_data = Bytes.toBytes(String(tuplei[4])) - - row_key = i_id - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["I_NAME"], i_name) - put.add(self.col_fam1, COLUMN_NAME["I_PRICE"], i_price) - put.add(self.col_fam1, COLUMN_NAME["I_DATA"], i_data) - put.add(self.col_fam2, COLUMN_NAME["I_IM_ID"], i_im_id) - self.item_tbl.put(put) - ## FOR - - elif tableName == "STOCK": - for tuplei in tuples: - s_i_id = Bytes.toBytes(Integer(tuplei[0])) - s_w_id = Bytes.toBytes(Integer(tuplei[1])) - s_quantity = Bytes.toBytes(Integer(tuplei[2])) - s_dist_01 = Bytes.toBytes(String(tuplei[3])) - s_dist_02 = Bytes.toBytes(String(tuplei[4])) - s_dist_03 = Bytes.toBytes(String(tuplei[5])) - s_dist_04 = Bytes.toBytes(String(tuplei[6])) - s_dist_05 = Bytes.toBytes(String(tuplei[7])) - s_dist_06 = Bytes.toBytes(String(tuplei[8])) - s_dist_07 = Bytes.toBytes(String(tuplei[9])) - s_dist_08 = Bytes.toBytes(String(tuplei[10])) - s_dist_09 = Bytes.toBytes(String(tuplei[11])) - s_dist_10 = Bytes.toBytes(String(tuplei[12])) - s_ytd = Bytes.toBytes(Integer(tuplei[13])) - s_order_cnt = Bytes.toBytes(Integer(tuplei[14])) - s_remote_cnt = Bytes.toBytes(Integer(tuplei[15])) - s_data = Bytes.toBytes(String(tuplei[16])) - - row_key = s_w_id - row_key.append(0) - row_key.extend(s_i_id) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["S_QUANTITY"], s_quantity) - put.add(self.col_fam1, COLUMN_NAME["S_DIST_01"], s_dist_01) - put.add(self.col_fam1, COLUMN_NAME["S_DIST_02"], s_dist_02) - put.add(self.col_fam1, COLUMN_NAME["S_DIST_03"], s_dist_03) - put.add(self.col_fam1, COLUMN_NAME["S_DIST_04"], s_dist_04) - put.add(self.col_fam1, COLUMN_NAME["S_DIST_05"], s_dist_05) - put.add(self.col_fam1, COLUMN_NAME["S_DIST_06"], s_dist_06) - put.add(self.col_fam1, COLUMN_NAME["S_DIST_07"], s_dist_07) - put.add(self.col_fam1, COLUMN_NAME["S_DIST_08"], s_dist_08) - put.add(self.col_fam1, COLUMN_NAME["S_DIST_09"], s_dist_09) - put.add(self.col_fam1, COLUMN_NAME["S_DIST_10"], s_dist_10) - put.add(self.col_fam1, COLUMN_NAME["S_YTD"], s_ytd) - put.add(self.col_fam1, COLUMN_NAME["S_ORDER_CNT"], s_order_cnt) - put.add(self.col_fam1, COLUMN_NAME["S_REMOTE_CNT"], s_remote_cnt) - put.add(self.col_fam1, COLUMN_NAME["S_DATA"], s_data) - self.stock_tbl.put(put) - ## FOR - - ## ============================================== - ## executeStart - ## ============================================== - def executeStart(self): - ## get HBase client - self.config = HBaseConfiguration.create() - self.admin = HBaseAdmin(self.config) - - ## get handlers to all tables - self.warehouse_tbl = HTable(self.config, "WAREHOUSE") - self.warehouse_tbl.setScannerCaching(SCANNER_CACHING) - self.district_tbl = HTable(self.config, "DISTRICT") - self.district_tbl.setScannerCaching(SCANNER_CACHING) - self.customer_tbl = HTable(self.config, "CUSTOMER") - self.customer_tbl.setScannerCaching(SCANNER_CACHING) - self.history_tbl = HTable(self.config, "HISTORY") - self.history_tbl.setScannerCaching(SCANNER_CACHING) - self.new_order_tbl = HTable(self.config, "NEW_ORDER") - self.new_order_tbl.setScannerCaching(SCANNER_CACHING) - self.order_tbl = HTable(self.config, "ORDERS") - self.order_tbl.setScannerCaching(SCANNER_CACHING) - self.order_line_tbl = HTable(self.config, "ORDER_LINE") - self.order_line_tbl.setScannerCaching(SCANNER_CACHING) - self.item_tbl = HTable(self.config, "ITEM") - self.item_tbl.setScannerCaching(SCANNER_CACHING) - self.stock_tbl = HTable(self.config, "STOCK") - self.stock_tbl.setScannerCaching(SCANNER_CACHING) - - ## ============================================== - ## executeFinish - ## ============================================== - def executeFinish(self): - ## close handlers to all tables - self.warehouse_tbl.close() - self.district_tbl.close() - self.customer_tbl.close() - self.history_tbl.close() - self.new_order_tbl.close() - self.order_tbl.close() - self.order_line_tbl.close() - self.item_tbl.close() - self.stock_tbl.close() - - ## ============================================== - ## doDelivery - ## ============================================== - def doDelivery(self, params): - w_id = params["w_id"] - o_carrier_id = params["o_carrier_id"] - ol_delivery_d = params["ol_delivery_d"] - - result = [ ] - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): - ## getNewOrder - start_row = self.getRowKey([w_id, d_id]) - s = Scan(start_row, self.getLexNextRowKey(start_row)) - s.addColumn(self.col_fam1, COLUMN_NAME["NO_W_ID"]) - scanner = self.new_order_tbl.getScanner(s) - newOrderRes = [ ] - for res in scanner: - current_row = res.getRow() - no_o_id = Bytes.toInt(current_row[10:14]) - if no_o_id > -1: - newOrderRes.append(res) - ## FOR - scanner.close() - - if len(newOrderRes) == 0: - ## No orders for this district: skip it. Note: This must be reported if > 1% - continue - assert len(newOrderRes) > 0 - newOrder = newOrderRes[0] - no_o_id = Bytes.toInt(newOrder.getRow()[10:14]) - - ## getCId - row_key = self.getRowKey([w_id, d_id, no_o_id]) - get = Get(row_key) - get.addColumn(self.col_fam1, COLUMN_NAME["O_C_ID"]) - res = self.order_tbl.get(get) - c_id = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["O_C_ID"])) - - ## sumOLAmount - start_row = self.getRowKey([w_id, d_id, no_o_id]) - s = Scan(start_row, self.getLexNextRowKey(start_row)) - s.addColumn(self.col_fam1, COLUMN_NAME["OL_AMOUNT"]) - scanner = self.order_line_tbl.getScanner(s) - ol_total = 0.0 - scanner_count = 0 - rows = [ ] - for res in scanner: - rows.append(res.getRow()) - scanner_count = scanner_count + 1 - ol_total = ol_total + Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["OL_AMOUNT"])) - ## FOR - scanner.close() - - ## deleteNewOrder - row_key = self.getRowKey([w_id, d_id, no_o_id]) - delete = Delete(row_key) - self.new_order_tbl.delete(delete) - - ## updateOrders - row_key = self.getRowKey([w_id, d_id, no_o_id]) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["O_CARRIER_ID"], Bytes.toBytes(String(str(o_carrier_id)))) - self.order_tbl.put(put) - - ## updateOrderLine - ## get the rows to update from results of sumOLAmount - for current_row in rows: - put = Put(current_row) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["OL_DELIVERY_D"], Bytes.toBytes(String(str(ol_delivery_d)))) - self.order_line_tbl.put(put) - ## FOR - - # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) - # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure - # them out - assert scanner_count>0 - assert ol_total > 0.0 - - ## "updateCustomer" - row_key = self.getRowKey([w_id, d_id, c_id]) - get = Get(row_key) - get.addColumn(self.col_fam1, COLUMN_NAME["C_BALANCE"]) - res = self.customer_tbl.get(get) - c_balance = Bytes.toFloat(res.getValue(self.col_fam1, COLUMN_NAME["C_BALANCE"])) + ol_total - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["C_BALANCE"], Bytes.toBytes(Float(c_balance))) - self.customer_tbl.put(put) - - result.append((d_id, no_o_id)) - ## FOR - return result - - ## ============================================== - ## doNewOrder - ## ============================================== - def doNewOrder(self, params): - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - o_entry_d = params["o_entry_d"] - i_ids = params["i_ids"] - i_w_ids = params["i_w_ids"] - i_qtys = params["i_qtys"] - - assert len(i_ids) > 0 - assert len(i_ids) == len(i_w_ids) - assert len(i_ids) == len(i_qtys) - - all_local = True - items = [ ] - for i in range(len(i_ids)): - ## Determine if this is an all local order or not - all_local = all_local and i_w_ids[i] == w_id - ## getItemInfo - row_key = self.getRowKey([i_ids[i]]) - get = Get(row_key) - get.addColumn(self.col_fam1, COLUMN_NAME["I_PRICE"]) - get.addColumn(self.col_fam1, COLUMN_NAME["I_NAME"]) - get.addColumn(self.col_fam1, COLUMN_NAME["I_DATA"]) - res = self.item_tbl.get(get) - current_item = [ ] - if res.getRow() is not None: - current_price = Bytes.toFloat(res.getValue(self.col_fam1, COLUMN_NAME["I_PRICE"])) - byte_arr = res.getValue(self.col_fam1, COLUMN_NAME["I_NAME"]) - current_name = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1, COLUMN_NAME["I_DATA"]) - current_data = Bytes.toString(byte_arr, 0, len(byte_arr)) - current_item = [current_price, current_name, current_data] - items.append(current_item) - ## FOR - - assert len(items) == len(i_ids) - - ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. - ## Note that this will happen with 1% of transactions on purpose. - for item in items: - if len(item) == 0: - return - ## FOR - - ## ---------------- - ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER - ## ---------------- - ## getWarehouseTaxRate - row_key = self.getRowKey([w_id]) - get = Get(row_key) - get.addColumn(self.col_fam2, COLUMN_NAME["W_TAX"]) - res = self.warehouse_tbl.get(get) - w_tax = Bytes.toFloat(res.getValue(self.col_fam2, COLUMN_NAME["W_TAX"])) - - ## getDistrict - row_key = self.getRowKey([w_id, d_id]) - get = Get(row_key) - get.addColumn(self.col_fam1, COLUMN_NAME["D_TAX"]) - get.addColumn(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"]) - res = self.district_tbl.get(get) - d_tax = Bytes.toFloat(res.getValue(self.col_fam1, COLUMN_NAME["D_TAX"])) - d_next_o_id = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"])) - - ## getCustomer - row_key = self.getRowKey([w_id, d_id, c_id]) - get = Get(row_key) - get.addColumn(self.col_fam1, COLUMN_NAME["C_DISCOUNT"]) - get.addColumn(self.col_fam1, COLUMN_NAME["C_LAST"]) - get.addColumn(self.col_fam1, COLUMN_NAME["C_CREDIT"]) - res = self.customer_tbl.get(get) - c_discount = Bytes.toFloat(res.getValue(self.col_fam1, COLUMN_NAME["C_DISCOUNT"])) - byte_arr_1 = res.getValue(self.col_fam1, COLUMN_NAME["C_LAST"]) - byte_arr_2 = res.getValue(self.col_fam1, COLUMN_NAME["C_CREDIT"]) - customer_info = [c_discount, Bytes.toString(byte_arr_1, 0, len(byte_arr_1)), Bytes.toString(byte_arr_2, 0 ,len(byte_arr_2))] - - ## ---------------- - ## Insert Order Information - ## ---------------- - ol_cnt = len(i_ids) - o_carrier_id = constants.NULL_CARRIER_ID - - ## incrementNextOrderId - row_key = self.getRowKey([w_id, d_id]) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"], Bytes.toBytes(Integer(d_next_o_id+1))) - self.district_tbl.put(put) - - ## createOrder - row_key = self.getRowKey([w_id, d_id, d_next_o_id]) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["O_C_ID"], Bytes.toBytes(Integer(c_id))) - put.add(self.col_fam1, COLUMN_NAME["O_ENTRY_D"], Bytes.toBytes(String(str(o_entry_d)))) - put.add(self.col_fam1, COLUMN_NAME["O_CARRIER_ID"], Bytes.toBytes(String(str(o_carrier_id)))) - put.add(self.col_fam2, COLUMN_NAME["O_OL_CNT"], Bytes.toBytes(Integer(ol_cnt))) - put.add(self.col_fam2, COLUMN_NAME["O_ALL_LOCAL"], Bytes.toBytes(Integer(all_local))) - self.order_tbl.put(put) - - ## createNewOrder - row_key = self.getRowKey([w_id, d_id, d_next_o_id]) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["NO_O_ID"], Bytes.toBytes(Integer(d_next_o_id))) - put.add(self.col_fam1, COLUMN_NAME["NO_D_ID"], Bytes.toBytes(Integer(d_id))) - put.add(self.col_fam1, COLUMN_NAME["NO_W_ID"], Bytes.toBytes(Integer(w_id))) - self.new_order_tbl.put(put) - - ## ---------------- - ## Insert Order Item Information - ## ---------------- - item_data = [ ] - total = 0 - for i in range(len(i_ids)): - ol_number = i + 1 - ol_supply_w_id = i_w_ids[i] - ol_i_id = i_ids[i] - ol_quantity = i_qtys[i] - - itemInfo = items[i] - i_name = itemInfo[1] - i_data = itemInfo[2] - i_price = itemInfo[0] - - ## getStockInfo - row_key = self.getRowKey([ol_supply_w_id, ol_i_id]) - get = Get(row_key) - get.addColumn(self.col_fam1, COLUMN_NAME["S_QUANTITY"]) - get.addColumn(self.col_fam1, COLUMN_NAME["S_DATA"]) - get.addColumn(self.col_fam1, COLUMN_NAME["S_YTD"]) - get.addColumn(self.col_fam1, COLUMN_NAME["S_ORDER_CNT"]) - get.addColumn(self.col_fam1, COLUMN_NAME["S_REMOTE_CNT"]) - get.addColumn(self.col_fam1, COLUMN_NAME["S_DIST_"+("%02d" % d_id)]) - res = self.stock_tbl.get(get) - s_quantity = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["S_QUANTITY"])) - s_ytd = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["S_YTD"])) - s_order_cnt = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["S_ORDER_CNT"])) - s_remote_cnt = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["S_REMOTE_CNT"])) - byte_arr = res.getValue(self.col_fam1, COLUMN_NAME["S_DATA"]) - s_data = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1, COLUMN_NAME["S_DIST_"+("%02d" % d_id)]) - s_dist_xx = Bytes.toString(byte_arr, 0, len(byte_arr)) # Fetches data from the s_dist_[d_id] column - - ## Update stock - s_ytd += ol_quantity - if s_quantity >= ol_quantity + 10: - s_quantity = s_quantity - ol_quantity - else: - s_quantity = s_quantity + 91 - ol_quantity - s_order_cnt += 1 - - if ol_supply_w_id != w_id: s_remote_cnt += 1 - - ## updateStock - row_key = self.getRowKey([ol_supply_w_id, ol_i_id]) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["S_QUANTITY"], Bytes.toBytes(Integer(s_quantity))) - put.add(self.col_fam1, COLUMN_NAME["S_YTD"], Bytes.toBytes(Integer(s_ytd))) - put.add(self.col_fam1, COLUMN_NAME["S_ORDER_CNT"], Bytes.toBytes(Integer(s_order_cnt))) - put.add(self.col_fam1, COLUMN_NAME["S_REMOTE_CNT"], Bytes.toBytes(Integer(s_remote_cnt))) - self.stock_tbl.put(put) - - if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: - brand_generic = 'B' - else: - brand_generic = 'G' - - ## Transaction profile states to use "ol_quantity * i_price" - ol_amount = ol_quantity * i_price - total += ol_amount - - ## createOrderLine - row_key = self.getRowKey([w_id, d_id, d_next_o_id, ol_number]) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["OL_I_ID"], Bytes.toBytes(Integer(ol_i_id))) - put.add(self.col_fam1, COLUMN_NAME["OL_SUPPLY_W_ID"], Bytes.toBytes(Integer(ol_supply_w_id))) - put.add(self.col_fam1, COLUMN_NAME["OL_DELIVERY_D"], Bytes.toBytes(String(str(o_entry_d)))) - put.add(self.col_fam1, COLUMN_NAME["OL_QUANTITY"], Bytes.toBytes(Integer(ol_quantity))) - put.add(self.col_fam1, COLUMN_NAME["OL_AMOUNT"], Bytes.toBytes(Float(ol_amount))) - put.add(self.col_fam1, COLUMN_NAME["OL_DIST_INFO"], Bytes.toBytes(String(s_dist_xx))) - self.order_line_tbl.put(put) - - ## Add the info to be returned - item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) - ## FOR - - ## Adjust the total for the discount - total *= (1 - c_discount) * (1 + w_tax + d_tax) - - ## Pack up values the client is missing (see TPC-C 2.4.3.5) - misc = [ (w_tax, d_tax, d_next_o_id, total) ] - - return [ customer_info, misc, item_data ] - - ## ============================================== - ## doOrderStatus - ## ============================================== - def doOrderStatus(self, params): - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - - assert w_id, pformat(params) - assert d_id, pformat(params) - - customer = None - if c_id != None: - ## getCustomerByCustomerId - row_key = self.getRowKey([w_id, d_id, c_id]) - get = Get(row_key) - get.addColumn(self.col_fam1, COLUMN_NAME["C_FIRST"]) - get.addColumn(self.col_fam1, COLUMN_NAME["C_MIDDLE"]) - get.addColumn(self.col_fam1, COLUMN_NAME["C_LAST"]) - get.addColumn(self.col_fam1, COLUMN_NAME["C_BALANCE"]) - customer = self.customer_tbl.get(get) - - else: - ## getCustomersByLastName - getCustomerRes = [ ] - start_row = self.getRowKey([w_id, d_id]) - s = Scan(start_row, self.getLexNextRowKey(start_row)) - s.addColumn(self.col_fam1, COLUMN_NAME["C_FIRST"]) - s.addColumn(self.col_fam1, COLUMN_NAME["C_MIDDLE"]) - s.addColumn(self.col_fam1, COLUMN_NAME["C_LAST"]) - s.addColumn(self.col_fam1, COLUMN_NAME["C_BALANCE"]) - scanner = self.customer_tbl.getScanner(s) - for res in scanner: - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_LAST"]) - current_c_last = Bytes.toString(byte_arr, 0, len(byte_arr)) - if current_c_last == c_last: - getCustomerRes.append(res) - ## FOR - scanner.close() - - assert len(getCustomerRes) > 0 - namecnt = len(getCustomerRes) - - getCustomerRes.sort(self.compareCustomerByFirst) - - index = (namecnt-1)/2 - customer = getCustomerRes[index] - c_id = Bytes.toInt(customer.getRow()[10:14]) - - assert customer != None - assert c_id != None - - byte_arr = customer.getValue(self.col_fam1,COLUMN_NAME["C_FIRST"]) - customer_first = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = customer.getValue(self.col_fam1,COLUMN_NAME["C_MIDDLE"]) - customer_middle = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = customer.getValue(self.col_fam1,COLUMN_NAME["C_LAST"]) - customer_last = Bytes.toString(byte_arr, 0, len(byte_arr)) - customer_balance = Bytes.toFloat(customer.getValue(self.col_fam1,COLUMN_NAME["C_BALANCE"])) - customer = [c_id, customer_first, customer_middle, customer_last, customer_balance] - - ## getLastOrder - getOrderRes = [ ] - start_row = self.getRowKey([w_id, d_id]) - s = Scan(start_row, self.getLexNextRowKey(start_row)) - s.addColumn(self.col_fam1, COLUMN_NAME["O_CARRIER_ID"]) - s.addColumn(self.col_fam1, COLUMN_NAME["O_ENTRY_D"]) - s.addColumn(self.col_fam1, COLUMN_NAME["O_C_ID"]) - scanner = self.order_tbl.getScanner(s) - for res in scanner: - current_o_c_id = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["O_C_ID"])) - if current_o_c_id == c_id: - getOrderRes.append(res) - ## FOR - scanner.close() - - getOrderRes.sort(self.compareOrderByOID) - order = None - if len(getOrderRes)>0 : - order = getOrderRes[0] - - orderLines = [ ] - if order != None: - order_o_id = Bytes.toInt(order.getRow()[10:14]) - byte_arr = order.getValue(self.col_fam1,COLUMN_NAME["O_CARRIER_ID"]) - order_o_carrier_id = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = order.getValue(self.col_fam1,COLUMN_NAME["O_ENTRY_D"]) - order_o_entry_d = Bytes.toString(byte_arr, 0, len(byte_arr)) - order = [order_o_id, order_o_carrier_id, order_o_entry_d] - ## getOrderLines - start_row = self.getRowKey([w_id, d_id, order_o_id]) - s = Scan(start_row, self.getLexNextRowKey(start_row)) - s.addColumn(self.col_fam1, COLUMN_NAME["OL_SUPPLY_W_ID"]) - s.addColumn(self.col_fam1, COLUMN_NAME["OL_I_ID"]) - s.addColumn(self.col_fam1, COLUMN_NAME["OL_QUANTITY"]) - s.addColumn(self.col_fam1, COLUMN_NAME["OL_AMOUNT"]) - s.addColumn(self.col_fam1, COLUMN_NAME["OL_DELIVERY_D"]) - scanner = self.order_line_tbl.getScanner(s) - for res in scanner: - current_ol_supply_w_id = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["OL_SUPPLY_W_ID"])) - current_ol_i_id = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["OL_I_ID"])) - current_ol_quantity = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["OL_QUANTITY"])) - current_ol_amount = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["OL_AMOUNT"])) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["OL_DELIVERY_D"]) - current_ol_delivery_d = Bytes.toString(byte_arr, 0, len(byte_arr)) - current_order_line = [current_ol_supply_w_id, current_ol_i_id, current_ol_quantity, current_ol_amount, current_ol_delivery_d] - orderLines.append(current_order_line) - ## FOR - scanner.close() - - else: - orderLines = [ ] - - return [ customer, order, orderLines ] - - ## ============================================== - ## doPayment - ## ============================================== - - def doPayment(self, params): - w_id = params["w_id"] - d_id = params["d_id"] - h_amount = params["h_amount"] - c_w_id = params["c_w_id"] - c_d_id = params["c_d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - h_date = params["h_date"] - - if c_id != None: - ## getCustomerByCustomerId - row_key = self.getRowKey([w_id, d_id, c_id]) - get = Get(row_key) - get.addFamily(self.col_fam1) - res = self.customer_tbl.get(get) - current_c_id = Bytes.toInt(res.getRow()[10:14]) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_FIRST"]) - current_c_first = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_MIDDLE"]) - current_c_middle = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_LAST"]) - current_c_last = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STREET_1"]) - current_c_street_1 = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STREET_2"]) - current_c_street_2 = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_CITY"]) - current_c_city = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STATE"]) - current_c_state = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_ZIP"]) - current_c_zip = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_PHONE"]) - current_c_phone = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_SINCE"]) - current_c_since = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_CREDIT"]) - current_c_credit = Bytes.toString(byte_arr, 0, len(byte_arr)) - current_c_credit_lim = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_CREDIT_LIM"])) - current_c_discount = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_DISCOUNT"])) - current_c_balance = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_BALANCE"])) - current_c_ytd_payment = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_YTD_PAYMENT"])) - current_c_payment_cnt = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["C_PAYMENT_CNT"])) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_DATA"]) - current_c_data = Bytes.toString(byte_arr, 0, len(byte_arr)) - customer = [current_c_id, current_c_first, current_c_middle, current_c_last, current_c_street_1, current_c_street_2, current_c_city, current_c_state, current_c_zip, current_c_phone, current_c_since, current_c_credit, current_c_credit_lim, current_c_discount, current_c_balance, current_c_ytd_payment, current_c_payment_cnt, current_c_data] - else: - ## getCustomersByLastName - start_row = self.getRowKey([w_id, d_id]) - s = Scan(start_row, self.getLexNextRowKey(start_row)) - s.addFamily(self.col_fam1) - scanner = self.customer_tbl.getScanner(s) - all_customers = [ ] - for res in scanner: - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_LAST"]) - current_c_last = Bytes.toString(byte_arr, 0, len(byte_arr)) - if current_c_last == c_last: - current_c_id = Bytes.toInt(res.getRow()[10:14]) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_FIRST"]) - current_c_first = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_MIDDLE"]) - current_c_middle = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_LAST"]) - current_c_last = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STREET_1"]) - current_c_street_1 = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STREET_2"]) - current_c_street_2 = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_CITY"]) - current_c_city = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_STATE"]) - current_c_state = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_ZIP"]) - current_c_zip = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_PHONE"]) - current_c_phone = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_SINCE"]) - current_c_since = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_CREDIT"]) - current_c_credit = Bytes.toString(byte_arr, 0, len(byte_arr)) - current_c_credit_lim = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_CREDIT_LIM"])) - current_c_discount = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_DISCOUNT"])) - current_c_balance = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_BALANCE"])) - current_c_ytd_payment = Bytes.toFloat(res.getValue(self.col_fam1,COLUMN_NAME["C_YTD_PAYMENT"])) - current_c_payment_cnt = Bytes.toInt(res.getValue(self.col_fam1,COLUMN_NAME["C_PAYMENT_CNT"])) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["C_DATA"]) - current_c_data = Bytes.toString(byte_arr, 0, len(byte_arr)) - - current_customer = [current_c_id, current_c_first, current_c_middle, current_c_last, current_c_street_1, current_c_street_2, current_c_city, current_c_state, current_c_zip, current_c_phone, current_c_since, current_c_credit, current_c_credit_lim, current_c_discount, current_c_balance, current_c_ytd_payment, current_c_payment_cnt, current_c_data] - all_customers.append(current_customer) - ## FOR - scanner.close() - - all_customers.sort(self.compareCustomerArrByFirst) - assert len(all_customers) > 0 - namecnt = len(all_customers) - index = (namecnt-1)/2 - customer = all_customers[index] - c_id = customer[0] - assert len(customer) > 0 - c_balance = customer[14] - h_amount - c_ytd_payment = customer[15] + h_amount - c_payment_cnt = customer[16] + 1 - c_data = customer[17] - - ## getWarehouse - row_key = self.getRowKey([w_id]) - get = Get(row_key) - get.addFamily(self.col_fam1) - get.addFamily(self.col_fam3) - res = self.warehouse_tbl.get(get) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_NAME"]) - current_w_name = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_STREET_1"]) - current_w_street_1 = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_STREET_2"]) - current_w_street_2 = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_CITY"]) - current_w_city = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_STATE"]) - current_w_state = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam1,COLUMN_NAME["W_ZIP"]) - current_w_zip = Bytes.toString(byte_arr, 0, len(byte_arr)) - ## pre-fetch w_ytd for updateWarehouseBalance - current_w_ytd = Bytes.toFloat(res.getValue(self.col_fam3,COLUMN_NAME["W_YTD"])) - warehouse = [current_w_name, current_w_street_1, current_w_street_2, current_w_city, current_w_state, current_w_zip] - - ## getDistrict - row_key = self.getRowKey([w_id, d_id]) - get = Get(row_key) - get.addFamily(self.col_fam2) - get.addFamily(self.col_fam3) - res = self.district_tbl.get(get) - byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_NAME"]) - current_d_name = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_STREET_1"]) - current_d_street_1 = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_STREET_2"]) - current_d_street_2 = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_CITY"]) - current_d_city = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_STATE"]) - current_d_state = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = res.getValue(self.col_fam2,COLUMN_NAME["D_ZIP"]) - current_d_zip = Bytes.toString(byte_arr, 0, len(byte_arr)) - ## pre-fetch d_ytd for updateDistrictBalance - current_d_ytd = Bytes.toFloat(res.getValue(self.col_fam3,COLUMN_NAME["D_YTD"])) - district = [current_d_name, current_d_street_1, current_d_street_2, current_d_city, current_d_state, current_d_zip] - - ## updateWarehouseBalance - row_key = self.getRowKey([w_id]) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam3, COLUMN_NAME["W_YTD"], Bytes.toBytes(Float(current_w_ytd + h_amount))) - self.warehouse_tbl.put(put) - - ## updateDistrictBalance - row_key = self.getRowKey([w_id, d_id]) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam3, COLUMN_NAME["D_YTD"], Bytes.toBytes(Float(current_d_ytd + h_amount))) - self.district_tbl.put(put) - - ## Customer Credit Information - if customer[11] == constants.BAD_CREDIT: - newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) - c_data = (newData + "|" + c_data) - if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] - ## updateBCCustomer - row_key = self.getRowKey([c_w_id, c_d_id, c_id]) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["C_BALANCE"], Bytes.toBytes(Float(c_balance))) - put.add(self.col_fam1, COLUMN_NAME["C_YTD_PAYMENT"], Bytes.toBytes(Float(c_ytd_payment))) - put.add(self.col_fam1, COLUMN_NAME["C_PAYMENT_CNT"], Bytes.toBytes(Integer(c_payment_cnt))) - put.add(self.col_fam1, COLUMN_NAME["C_DATA"], Bytes.toBytes(String(c_data))) - self.customer_tbl.put(put) - else: - c_data = "" - ## updateGCCustomer - row_key = self.getRowKey([c_w_id, c_d_id, c_id]) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["C_BALANCE"], Bytes.toBytes(Float(c_balance))) - put.add(self.col_fam1, COLUMN_NAME["C_YTD_PAYMENT"], Bytes.toBytes(Float(c_ytd_payment))) - put.add(self.col_fam1, COLUMN_NAME["C_PAYMENT_CNT"], Bytes.toBytes(Integer(c_payment_cnt))) - self.customer_tbl.put(put) - - # Concatenate w_name, four spaces, d_name - h_data = "%s %s" % (warehouse[0], district[0]) - # Create the history record - ## insertHistory - row_key = Bytes.toBytes(str(uuid.uuid1())) - put = Put(row_key) - put.setWriteToWAL(WRITE_TO_WAL) - put.add(self.col_fam1, COLUMN_NAME["H_C_ID"], Bytes.toBytes(Integer(c_id))) - put.add(self.col_fam1, COLUMN_NAME["H_C_D_ID"], Bytes.toBytes(Integer(c_d_id))) - put.add(self.col_fam1, COLUMN_NAME["H_C_W_ID"], Bytes.toBytes(Integer(c_w_id))) - put.add(self.col_fam1, COLUMN_NAME["H_D_ID"], Bytes.toBytes(Integer(d_id))) - put.add(self.col_fam1, COLUMN_NAME["H_W_ID"], Bytes.toBytes(Integer(w_id))) - put.add(self.col_fam1, COLUMN_NAME["H_DATE"], Bytes.toBytes(String(str(h_date)))) - put.add(self.col_fam1, COLUMN_NAME["H_AMOUNT"], Bytes.toBytes(Float(h_amount))) - put.add(self.col_fam1, COLUMN_NAME["H_DATA"], Bytes.toBytes(String(h_data))) - self.history_tbl.put(put) - - # TPC-C 2.5.3.3: Must display the following fields: - # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, - # D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, - # C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, - # C_DISCOUNT, C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = "BC"), - # H_AMOUNT, and H_DATE. - - # Hand back all the warehouse, district, and customer data - return [ warehouse, district, customer ] - - ## ============================================== - ## doStockLevel - ## ============================================== - def doStockLevel(self, params): - w_id = params["w_id"] - d_id = params["d_id"] - threshold = params["threshold"] - - ## getOId - row_key = self.getRowKey([w_id, d_id]) - get = Get(row_key) - get.addColumn(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"]) - res = self.district_tbl.get(get) - assert res.getRow() is not None - o_id = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["D_NEXT_O_ID"])) - - ## getStockCount - ol_i_id_set = set() - start_row = self.getRowKey([w_id, d_id]) - s = Scan(start_row, self.getLexNextRowKey(start_row)) - s.addColumn(self.col_fam1, COLUMN_NAME["OL_I_ID"]) - scanner = self.order_line_tbl.getScanner(s) - for res in scanner: - current_row = res.getRow() - ol_o_id = Bytes.toInt(current_row[10:14]) - if (ol_o_id < o_id) and (ol_o_id >= o_id - 20): - current_ol_i_id = Bytes.toInt(res.getValue(self.col_fam1, COLUMN_NAME["OL_I_ID"])) - row_key = self.getRowKey([w_id, current_ol_i_id]) - get = Get(row_key) - get.addColumn(self.col_fam1, COLUMN_NAME["S_QUANTITY"]) - resx = self.stock_tbl.get(get) - if resx.getRow() is not None: - current_s_quantity = Bytes.toInt(resx.getValue(self.col_fam1, COLUMN_NAME["S_QUANTITY"])) - if current_s_quantity < threshold: - ol_i_id_set.add(current_ol_i_id) - ## FOR - scanner.close() - - return len(ol_i_id_set) - - ## get a row key which is composite of primary keys - def getRowKey(self,keylist): - byte_arr = [ ] - length = len(keylist) - for i in range(length): - byte_arr.extend(Bytes.toBytes(Integer(keylist[i]))) - if i= 0 and next[j] < 127) or (next[j] >= -128 and next[j] < -1): - next[j] = next[j] + 1 - break - elif next[j] == 127: - next[j] = -128 - break - elif next[j] == -1: - next[j] = 0 - ## FOR - return next - - - ## compare function used in OrderStatus transaction: ASCE - def compareCustomerByFirst(self, i, j): - byte_arr = i.getValue(self.col_fam1,COLUMN_NAME["C_FIRST"]) - first_i = Bytes.toString(byte_arr, 0, len(byte_arr)) - byte_arr = j.getValue(self.col_fam1,COLUMN_NAME["C_FIRST"]) - first_j = Bytes.toString(byte_arr, 0, len(byte_arr)) - return cmp(first_i,first_j) - - ## compare function used in OrderStatus transaction: DESC - def compareOrderByOID(self, i, j): - oid_i = Bytes.toInt(i.getRow()[10:14]) - oid_j = Bytes.toInt(j.getRow()[10:14]) - return -cmp(oid_i,oid_j) - - ## compare function used in Payment transaction: ASCE - def compareCustomerArrByFirst(self, i, j): - first_i = i[1] - first_j = i[1] - return cmp(first_i,first_j) diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/membasedriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/membasedriver.py deleted file mode 100644 index 11317573..00000000 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/membasedriver.py +++ /dev/null @@ -1,932 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------- -# Copyright (C) 2011 -# Sunil Mallya, Silvia Zuffi -# http://www.cs.brown.edu/~sunilmallya/ -# http://www.cs.brown.edu/~zuffi/ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# ----------------------------------------------------------------------- - -from __future__ import with_statement - -import os,time -import logging -import commands -import memcache -from pprint import pprint,pformat - -import constants -from abstractdriver import * -MAX_CUSTOMER_ID = 3000 -MAX_ORDER_ID = 2999 - -# Appended with number of columns that needs to be used as the key -TABLE_COLUMNS = { - constants.TABLENAME_ITEM: [ - "I_ID", # INTEGER - "I_IM_ID", # INTEGER - "I_NAME", # VARCHAR - "I_PRICE", # FLOAT - "I_DATA", # VARCHAR - 1 - ], - constants.TABLENAME_WAREHOUSE: [ - "W_ID", # SMALLINT - "W_NAME", # VARCHAR - "W_STREET_1", # VARCHAR - "W_STREET_2", # VARCHAR - "W_CITY", # VARCHAR - "W_STATE", # VARCHAR - "W_ZIP", # VARCHAR - "W_TAX", # FLOAT - "W_YTD", # FLOAT - 1 - ], - constants.TABLENAME_DISTRICT: [ - "D_ID", # TINYINT - "D_W_ID", # SMALLINT - "D_NAME", # VARCHAR - "D_STREET_1", # VARCHAR - "D_STREET_2", # VARCHAR - "D_CITY", # VARCHAR - "D_STATE", # VARCHAR - "D_ZIP", # VARCHAR - "D_TAX", # FLOAT - "D_YTD", # FLOAT - "D_NEXT_O_ID", # INT - 2 - ], - constants.TABLENAME_CUSTOMER: [ - "C_ID", # INTEGER - "C_D_ID", # TINYINT - "C_W_ID", # SMALLINT - "C_FIRST", # VARCHAR - "C_MIDDLE", # VARCHAR - "C_LAST", # VARCHAR - "C_STREET_1", # VARCHAR - "C_STREET_2", # VARCHAR - "C_CITY", # VARCHAR - "C_STATE", # VARCHAR - "C_ZIP", # VARCHAR - "C_PHONE", # VARCHAR - "C_SINCE", # TIMESTAMP - "C_CREDIT", # VARCHAR - "C_CREDIT_LIM", # FLOAT - "C_DISCOUNT", # FLOAT - "C_BALANCE", # FLOAT - "C_YTD_PAYMENT", # FLOAT - "C_PAYMENT_CNT", # INTEGER - "C_DELIVERY_CNT", # INTEGER - "C_DATA", # VARCHAR - 3 - ], - constants.TABLENAME_STOCK: [ - "S_I_ID", # INTEGER - "S_W_ID", # SMALLINT - "S_QUANTITY", # INTEGER - "S_DIST_01", # VARCHAR - "S_DIST_02", # VARCHAR - "S_DIST_03", # VARCHAR - "S_DIST_04", # VARCHAR - "S_DIST_05", # VARCHAR - "S_DIST_06", # VARCHAR - "S_DIST_07", # VARCHAR - "S_DIST_08", # VARCHAR - "S_DIST_09", # VARCHAR - "S_DIST_10", # VARCHAR - "S_YTD", # INTEGER - "S_ORDER_CNT", # INTEGER - "S_REMOTE_CNT", # INTEGER - "S_DATA", # VARCHAR - 2 - ], - constants.TABLENAME_ORDERS: [ - "O_ID", # INTEGER - "O_C_ID", # INTEGER - "O_D_ID", # TINYINT - "O_W_ID", # SMALLINT - "O_ENTRY_D", # TIMESTAMP - "O_CARRIER_ID", # INTEGER - "O_OL_CNT", # INTEGER - "O_ALL_LOCAL", # INTEGER - 4 - ], - "ORDERS_ID": [ - "O_C_ID", # INTEGER - "O_D_ID", # TINYINT - "O_W_ID", # SMALLINT - "O_ID", # VARCHAR - 3 - ], - constants.TABLENAME_NEW_ORDER: [ - "NO_O_ID", # INTEGER - "NO_D_ID", # TINYINT - "NO_W_ID", # SMALLINT - 3 - ], - constants.TABLENAME_ORDER_LINE: [ - "OL_O_ID", # INTEGER - "OL_D_ID", # TINYINT - "OL_W_ID", # SMALLINT - "OL_NUMBER", # INTEGER - "OL_I_ID", # INTEGER - "OL_SUPPLY_W_ID", # SMALLINT - "OL_DELIVERY_D", # TIMESTAMP - "OL_QUANTITY", # INTEGER - "OL_AMOUNT", # FLOAT - "OL_DIST_INFO", # VARCHAR - 3 - ], - constants.TABLENAME_HISTORY: [ - "H_C_ID", # INTEGER - "H_C_D_ID", # TINYINT - "H_C_W_ID", # SMALLINT - "H_D_ID", # TINYINT - "H_W_ID", # SMALLINT - "H_DATE", # TIMESTAMP - "H_AMOUNT", # FLOAT - "H_DATA", # VARCHAR - 5 - ], -} - -def irange(sequence): - return zip(range(len(sequence)),sequence) - -def filter_table(sec_keys,table_values,table_name): - - columns = TABLE_COLUMNS[table_name] - return_list=[] - sec_key_indices = [] # [index, value] - for sec_key_name in sec_keys: - if sec_key_name[0]!= None: - index = columns.index(sec_key_name[0]) - sec_key_indices.append([index,sec_key_name[1]]) - - for tvalue in table_values: - copy_flag = 1 - for sec_key_tuple in sec_key_indices: - if tvalue[sec_key_tuple[0]] != sec_key_tuple[1]: - copy_flag =0 - - if copy_flag ==1: - return_list.append(tvalue) - - return return_list - - -def return_columns_single_record(extract_columns,table_values,table_name): - columns = TABLE_COLUMNS[table_name] - record =[] - for c in extract_columns: - c_index = columns.index(c) - record.append(table_values[c_index]) - - return record - -def return_columns(extract_columns,table_values,table_name): - #c contains the name of the columns to be extracted - columns = TABLE_COLUMNS[table_name] - - return_table=[] - for tvalue in table_values: - insert_tuple=[] - for c in extract_columns: - c_index = columns.index(c) - insert_tuple.append(tvalue[c_index]) - return_table.append(insert_tuple) - - return return_table - - -## ============================================== -## MembaseDriver -## ============================================== -class MembaseDriver(AbstractDriver): - DEFAULT_CONFIG = { - "host": ("The hostname to membase", "localhost" ), - "port": ("The port number to membase", 11211 ), - "name": ("Collection name", "tpcc"), - } - #This may be similar to the memcache configuration - - def __init__(self, ddl): - super(MembaseDriver, self).__init__("membase", ddl) - self.database = None - self.conn = None - self.load_time = 0 - - ## ---------------------------------------------- - ## makeDefaultConfig - ## ---------------------------------------------- - def makeDefaultConfig(self): - return MembaseDriver.DEFAULT_CONFIG - - ## ---------------------------------------------- - ## loadConfig - ## ---------------------------------------------- - def loadConfig(self, config): - for key in MembaseDriver.DEFAULT_CONFIG.keys(): - assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) - - connection_string = config['host'] +":"+ str(config['port']) - conn_list = [] ; conn_list.append(connection_string) - - self.conn = memcache.Client(conn_list) - #self.database = self.conn[str(config['name'])] - - ## ---------------------------------------------- - ## loadTuples into a csv file - ## ../py-tpcc/src/pytpcc/insert_data.csv - ## ---------------------------------------------- - - def loadTuples_to_csv(self, tableName, tuples): - - if len(tuples) == 0: return - f = open("insert_data.csv",'a') - for tuple_element in tuples: - primary_key = tuple_element[0] - key = tableName +"_"+ str(primary_key) - value = tuple_element[1:] - value = str(value) - value = value.strip('['); value = value.strip(']'); - f.write(key+','+value+"\n") - f.close() - - ## ---------------------------------------------- - ## loadTuples - ## ---------------------------------------------- - - def loadTuples(self, tableName, tuples): - if len(tuples) == 0: - print "NO DATA" - return - temp_max_id = self.conn.get(tableName+"_max_pkey") - if temp_max_id == None: - self.conn.set(tableName+"_max_pkey",0) - temp_max_id = 0 - - columns = TABLE_COLUMNS[tableName] - key_ids = columns[-1] - - key_val_pairs={} - start_time = time.time() - for tuple_element in tuples: - primary_key = tuple_element[0] - if temp_max_id < primary_key: - temp_max_id = primary_key - - key_postfix ="" - for idx in range(0,key_ids): - key_postfix = key_postfix + "_" + str(tuple_element[idx]) - - key = tableName + key_postfix - value = tuple_element - key_val_pairs[key] = value - #tmp_dict={} - #tmp_dict[key]=value - #key_val_pairs.append(tmp_dict) - - #self.conn.set(key,value) - - """ - if tableName == "ORDERS": - okey = "ORDERS_ID_"+str(tuple_element[1])+"_"+str(tuple_element[2])+"_"+str(tuple_element[3]) - ovalue = str(tuple_element[0])+":" - ret = self.conn.append(okey, ovalue) - if ret == False: - self.conn.set(okey, ovalue) - """ - if tableName == "ORDERS": - okey = "ORDERS_ID_"+str(tuple_element[1])+"_"+str(tuple_element[2])+"_"+str(tuple_element[3]) - ovalue = self.conn.get(okey) - if ovalue == None: - ovalue = str(tuple_element[0])+":" - else: - ovalue = ovalue + str(tuple_element[0])+":" - key_val_pairs[okey] = ovalue - - #Use multi_set to send all the keys to vbucket - self.conn.set_multi(key_val_pairs) - - #Define the implicit Schema - end_time = time.time() - self.load_time += (end_time - start_time) - self.conn.replace(tableName+"_max_pkey",temp_max_id) - print "[",tableName,"]"," Current Load Time: ",self.load_time - logging.debug("Loaded %d tuples for tableName %s" % (len(tuples), tableName)) - return - - ## ---------------------------------------------- - ## loadFinish - ## ---------------------------------------------- - - def loadFinish(self): - logging.info("Commiting changes to database") - ## ---------------------------------------------- - ## doOrderStatus - ## ---------------------------------------------- - - - ## ---------------------------------------------- - ## doDelivery - ## ---------------------------------------------- - def doDelivery(self, params): - - w_id = params["w_id"] - o_carrier_id = params["o_carrier_id"] - ol_delivery_d = params["o_carrier_id"] - - result = [ ] - ol_total =0 - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): - - #------------------------------------------------------------------------------------- - # The row in the NEW-ORDER table with matching NO_W_ID (equals W_ID) and NO_D_ID (equals D_ID) - # and with the lowest NO_O_ID value is selected. - #------------------------------------------------------------------------------------- - - for idx in range(1, MAX_ORDER_ID+1): - new_order = self.conn.get("NEW_ORDER_"+str(idx)+"_"+str(d_id)+"_"+str(w_id)) - if new_order != None: - break - - if new_order == None: - continue - - assert len(new_order) > 0 - columns = TABLE_COLUMNS["NEW_ORDER"] - o_id = new_order[ columns.index("NO_O_ID") ] - - #------------------------------------------------------------------------------------- - # The selected row in the NEW-ORDER table is deleted. - #------------------------------------------------------------------------------------- - - self.conn.delete("NEW_ORDER_"+str(o_id)+"_"+str(d_id)+"_"+str(w_id)) - - - #------------------------------------------------------------------------------------- - # The row in the ORDER table with matching O_W_ID (equals W_ ID), O_D_ID (equals D_ID), - # and O_ID (equals NO_O_ID) is selected, O_C_ID, the customer number, is retrieved, - # and O_CARRIER_ID is updated. - #------------------------------------------------------------------------------------- - - # getCId - for idx in range(1, MAX_CUSTOMER_ID+1): - order = self.conn.get("ORDERS_"+str(o_id)+"_"+str(idx)+"_"+str(d_id)+"_"+str(w_id)) - if order != None: - c_id = idx; - break - - - - # "updateOrders" - updated_order = self.update_single_record("O_CARRIER_ID", o_carrier_id, order,"ORDERS"); - self.conn.set("ORDERS_"+str(o_id)+"_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id),updated_order); - - #------------------------------------------------------------------------------------- - # All rows in the ORDER-LINE table with matching OL_W_ID (equals O_W_ID), OL_D_ID (equals O_D_ID), - # and OL_O_ID (equals O_ID) are selected. All OL_DELIVERY_D, the delivery dates, are updated to - # the current system time as returned by the operating system and the sum of all OL_AMOUNT is retrieved. - #------------------------------------------------------------------------------------- - - order_line = self.conn.get("ORDER_LINE_"+str(o_id)+"_"+str(d_id)+"_"+str(w_id)) - updated_order_line = self.update_single_record("OL_DELIVERY_D", ol_delivery_d, order_line,"ORDER_LINE") - self.conn.set("ORDER_LINE_"+str(o_id)+"_"+str(d_id)+"_"+str(w_id), updated_order_line) - - columns = TABLE_COLUMNS["ORDER_LINE"] - selected_order_line = order_line[ columns.index("OL_AMOUNT") ] - - ol_total += selected_order_line - - #------------------------------------------------------------------------------------- - # The row in the CUSTOMER table with matching C_W_ID (equals W_ID), C_D_ID (equals D_ID), - # and C_ID (equals O_C_ID) is selected and C_BALANCE is increased by the sum of all order-line - # amounts (OL_AMOUNT) previously retrieved. C_DELIVERY_CNT is incremented by 1. - #------------------------------------------------------------------------------------- - - # "updateCustomer": - customer = self.conn.get("CUSTOMER_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id)) - updated_customer = self.increment_field("C_BALANCE", ol_total, customer,"CUSTOMER") - updated_customer = self.increment_field("C_DELIVERY_CNT", 1, updated_customer,"CUSTOMER") - self.conn.set("CUSTOMER_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id), updated_customer); - - result.append((d_id, o_id)) - return result - - - ## ---------------------------------------------- - ## doNewOrder - ## ---------------------------------------------- - def doNewOrder(self, params): - def __getItem(items,item_id): - for index,item in irange(items): - if item[0] == item_id: - return index - - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - i_ids = params["i_ids"] - i_w_ids = params["i_w_ids"] - i_qtys = params["i_qtys"] - - o_entry_d = params["o_entry_d"] - s_dist_col = "S_DIST_%02d" % d_id - - assert len(i_ids) > 0 - assert len(i_ids) == len(i_w_ids) - assert len(i_ids) == len(i_qtys) - - items = [] - for item_index in i_ids: - item_key = "ITEM_"+str(item_index) - new_item = self.conn.get(item_key) - if new_item != None: - items.append(new_item) - - #------------------------------------------------------------------------------------- - # For each item on the order: - # The row in the ITEM table with matching I_ID (equals OL_I_ID) is selected and I_PRICE, - # the price of the item, I_NAME, the name of the item, and I_DATA are retrieved. - # If I_ID has an unused value, a "not-found" condition is signaled, - # resulting in a rollback of the database transaction. - #------------------------------------------------------------------------------------- - - if len(items) != len(i_ids): - return - - total = 1 - item_data = [] - - #------------------------------------------------------------------------------------- - - # The row in the WAREHOUSE table with matching W_ID is selected and W_TAX, the warehouse tax rate, is retrieved. - #------------------------------------------------------------------------------------- - warehouse = self.conn.get("WAREHOUSE_"+str(w_id)) - assert warehouse - columns = TABLE_COLUMNS["WAREHOUSE"] - w_tax = warehouse[ columns.index("W_TAX") ] - - #------------------------------------------------------------------------------------- - # The row in the DISTRICT table with matching D_W_ID and D_ ID is selected, D_TAX, the district tax rate, - # is retrieved, and D_NEXT_O_ID, the next available order number for the district, - # is retrieved and incremented by one. - #------------------------------------------------------------------------------------- - district = self.conn.get("DISTRICT_"+str(d_id)+"_"+str(w_id)) - assert district - selected_district = return_columns_single_record(["D_TAX", "D_NEXT_O_ID"], district, "DISTRICT") - d_tax = selected_district[0] - d_next_o_id = selected_district[1] - - updated_district = self.increment_field("D_NEXT_O_ID", 1, district, "DISTRICT"); - self.conn.set("DISTRICT_"+str(d_id)+"_"+str(w_id), updated_district); - - #------------------------------------------------------------------------------------- - # The row in the CUSTOMER table with matching C_W_ID, C_D_ID, and C_ID is selected and C_DISCOUNT, - # the customer's discount rate, C_LAST, the customer's last name, and C_CREDIT, the customer's credit status, - # are retrieved. - #------------------------------------------------------------------------------------- - customer = self.conn.get("CUSTOMER_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id)) - assert customer - selected_customer = return_columns_single_record(["C_DISCOUNT", "C_LAST", "C_CREDIT"], customer, "CUSTOMER") - c_discount = selected_customer[0] - - #------------------------------------------------------------------------------------- - # A new row is inserted into both the NEW-ORDER table and the ORDER table to reflect the creation - # of the new order. O_CARRIER_ID is set to a null value. If the order includes only home order-lines, - # then O_ALL_LOCAL is set to 1, otherwise O_ALL_LOCAL is set to 0. - # The number of items, O_OL_CNT, is computed to match ol_cnt. - #------------------------------------------------------------------------------------- - # from mongodb driver - all_local = (not i_w_ids or [w_id] * len(i_w_ids) == i_w_ids) - - o_all_local = 0 - if all_local: - o_all_local = 1; - ol_cnt = len(i_ids) - o_carrier_id = constants.NULL_CARRIER_ID - value = [o_entry_d, o_carrier_id, ol_cnt, all_local] - self.conn.set("ORDERS_"+str(d_next_o_id)+"_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id), value) - - # update the table that stores the orders id - ret = self.conn.append("ORDERS_ID_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id), str(d_next_o_id)+":") - if ret == None: - self.conn.set("ORDERS_ID_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id), str(d_next_o_id)+":") - - self.conn.set("NEW_ORDERS_"+str(d_next_o_id)+"_"+str(d_id)+"_"+str(w_id),value) - - for idx in i_ids: - - #------------------------------------------------------------------------------------- - # For each O_OL_CNT item on the order: - # The row in the ITEM table with matching I_ID (equals OL_I_ID) is selected and I_PRICE, - # the price of the item, I_NAME, the name of the item, and I_DATA are retrieved. - # If I_ID has an unused value, a "not-found" condition is signaled, - # resulting in a rollback of the database transaction. - #------------------------------------------------------------------------------------- - - i = __getItem(items,idx) - item = self.conn.get("ITEM_"+str(idx)); - if item == None: - return - - selected_item = return_columns_single_record(["I_PRICE", "I_NAME", "I_DATA"], item, "ITEM"); - i_price = selected_item[0] - i_name = selected_item[1] - i_data = selected_item[2] - - - #------------------------------------------------------------------------------------- - # The row in the STOCK table with matching S_I_ID (equals OL_I_ID) and S_W_ID (equals OL_SUPPLY_W_ID) - # is selected. S_QUANTITY, the quantity in stock, S_DIST_xx, where xx represents the district number, - # and S_DATA are retrieved. If the retrieved value for S_QUANTITY exceeds OL_QUANTITY by 10 or more, - # then S_QUANTITY is decreased by OL_QUANTITY; otherwise S_QUANTITY is updated to (S_QUANTITY - OL_QUANTITY)+91. - # S_YTD is increased by OL_QUANTITY and S_ORDER_CNT is incremented by 1. If the order-line is remote, then - # S_REMOTE_CNT is incremented by 1. - #------------------------------------------------------------------------------------- - - ol_supply_w_id = i_w_ids[i] - stock = self.conn.get("STOCK_"+str(idx)+"_"+str(ol_supply_w_id)) - assert stock - - selected_stock = return_columns_single_record(["S_QUANTITY", "S_DATA", "S_YTD", "S_ORDER_CNT", "S_REMOTE_CNT", s_dist_col ], stock, "STOCK") - s_quantity = selected_stock[0] - s_data = selected_stock[1] - ol_quantity = i_qtys[i] - if(s_quantity > (ol_quantity+10)): - s_quantity = s_quantity - ol_quantity - else: - s_quantity = (s_quantity - ol_quantity)+91 - - updated_stock = self.update_single_record("S_QUANTITY", s_quantity, stock,"STOCK"); - updated_stock = self.increment_field("S_YTD", ol_quantity, updated_stock, "STOCK") - updated_stock = self.increment_field("S_ORDER_CNT", 1, updated_stock, "STOCK") - if o_all_local == 1: - updated_stock = self.increment_field("S_REMOTE_CNT", 1, updated_stock, "STOCK") - - self.conn.set("STOCK_"+str(idx)+"_"+str(ol_supply_w_id), updated_stock); - - if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: - brand_generic = 'B' - else: - brand_generic = 'G' - item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_quantity) ) - - total *= (1 - c_discount) * (1 + w_tax + d_tax) - misc = [ (w_tax, d_tax, d_next_o_id, total) ] - - return [ customer, misc, item_data ] - - - ## ---------------------------------------------- - ## doOrderStatus - def doOrderStatus(self, params): - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - - - assert w_id, pformat(params) - assert d_id, pformat(params) - - #------------------------------------------------------------------------------------- - # Case 1, the customer is selected based on customer number: the row in the CUSTOMER table with matching - # C_W_ID, C_D_ID, and C_ID is selected and C_BALANCE, C_FIRST, C_MIDDLE, and C_LAST are retrieved. - #------------------------------------------------------------------------------------- - - if c_id != None: - customer = self.conn.get("CUSTOMER_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id)) - - #------------------------------------------------------------------------------------- - # Case 2, the customer is selected based on customer last name: all rows in the CUSTOMER table with - # matching C_W_ID, C_D_ID and C_LAST are selected sorted by C_FIRST in ascending order. - # Let n be the number of rows selected. C_BALANCE, C_FIRST, C_MIDDLE, and C_LAST are retrieved from - # the row at position n/2 rounded up in the sorted set of selected rows from the CUSTOMER table. - #------------------------------------------------------------------------------------- - else: - all_customers= [] - for idx in range(1, MAX_CUSTOMER_ID+1): - customer_record = self.conn.get("CUSTOMER_"+str(idx)+"_"+str(d_id)+"_"+str(w_id)) - all_customers.append(customer_record) - filtered_customer_table = filter_table([["C_LAST",c_last]],all_customers,"CUSTOMER") - ordered_customer_map = self.order_by("C_FIRST", filtered_customer_table, "CUSTOMER"); - - # Get the midpoint customer's id - namecnt = len(filtered_customer_table) - assert namecnt > 0 - index = ordered_customer_map[(namecnt-1)/2][0] - customer = filtered_customer_table[index] - - columns = TABLE_COLUMNS["CUSTOMER"] - c_index = columns.index('C_ID') - c_id = customer[c_index] - - selected_customer = return_columns_single_record(["C_BALANCE", "C_FIRST", "C_MIDDLE", "C_LAST"], customer, "CUSTOMER") - - #------------------------------------------------------------------------------------- - # The row in the ORDER table with matching O_W_ID (equals C_W_ID), O_D_ID (equals C_D_ID), O_C_ID (equals C_ID), - # and with the largest existing O_ID, is selected. This is the most recent order placed by that customer. - # O_ID, O_ENTRY_D, and O_CARRIER_ID are retrieved. - #-------------------------------------------------------------------------------------• - - # read all the orders id from table - all_orders_id_str = self.conn.get("ORDERS_ID_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id)) - assert all_orders_id_str - all_orders_id_array = all_orders_id_str.split(":"); - o_id = 0; - - # the last element is always empty because we append : at the end - for j in range(len(all_orders_id_array)-1): - o_str = all_orders_id_array[j] - o = int(o_str) - if o > o_id: - o_id = o - order = self.conn.get("ORDERS_"+str(o_id)+"_"+str(c_id)+"_"+str(d_id)+"_"+str(w_id)) - - - - #------------------------------------------------------------------------------------- - # All rows in the ORDER-LINE table with matching OL_W_ID (equals O_W_ID), OL_D_ID (equals O_D_ID), - # and OL_O_ID (equals O_ID) are selected and the corresponding sets of OL_I_ID, OL_SUPPLY_W_ID, - # OL_QUANTITY, OL_AMOUNT, and OL_DELIVERY_D are retrieved. - #------------------------------------------------------------------------------------- - - order_lines = self.conn.get("ORDER_LINE_"+str(o_id)+"_"+str(d_id)+"_"+str(w_id)) - filter_order_lines = return_columns_single_record(["OL_SUPPLY_W_ID", "OL_I_ID", "OL_QUANTITY", "OL_AMOUNT", "OL_DELIVERY_D"],order_lines, "ORDER_LINE") - return [ selected_customer, order, filter_order_lines ] - - - ## ---------------------------------------------- - ## doPayment - ## ---------------------------------------------- - def doPayment(self, params): - #print "PAYMENT" - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - c_w_id = params["c_w_id"] - c_d_id = params["c_d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - h_amount = params["h_amount"]; - h_date = params["h_date"] - # if w_id = c_w_id the transaction is "home", otherwise "remote" - - - #------------------------------------------------------------------------------------- - # The row in the WAREHOUSE table with matching W_ID is selected. - # W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, and W_ZIP are retrieved - # and W_YTD, the warehouse's year-to-date balance, is increased by H_ AMOUNT. - #------------------------------------------------------------------------------------- - - warehouse = self.conn.get("WAREHOUSE_"+str(w_id)) - updated_warehouse = self.increment_field("W_YTD", h_amount, warehouse,"WAREHOUSE") - self.conn.set("WAREHOUSE_"+str(w_id), updated_warehouse); - selected_warehouse = return_columns_single_record(["W_NAME", "W_STREET_1", "W_STREET_2", "W_CITY", "W_STATE", "W_ZIP"], updated_warehouse, "WAREHOUSE") - - #------------------------------------------------------------------------------------- - # The row in the DISTRICT table with matching D_W_ID and D_ID is selected. - # D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, and D_ZIP are retrieved and D_YTD, - # the district's year-to-date balance, is increased by H_AMOUNT. - #------------------------------------------------------------------------------------- - - district = self.conn.get("DISTRICT_"+str(d_id)+"_"+str(w_id)) - selected_district = return_columns_single_record(["D_NAME", "D_STREET_1", "D_STREET_2", "D_CITY", "D_STATE", "D_ZIP"], district, "DISTRICT") - updated_district = self.increment_field("D_YTD", h_amount, district,"DISTRICT") - self.conn.set("DISTRICT_"+str(d_id)+"_"+str(w_id),updated_district) - - #------------------------------------------------------------------------------------- - # Case 1, the customer is selected based on customer number: the row in the CUSTOMER table with - # matching C_W_ID, C_D_ID and C_ID is selected. C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, - # C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, and C_BALANCE are retrieved. - # C_BALANCE is decreased by H_AMOUNT. C_YTD_PAYMENT is increased by H_AMOUNT. C_PAYMENT_CNT is incremented by 1. - #------------------------------------------------------------------------------------- - - if c_id != None: - customer = self.conn.get("CUSTOMER_"+str(c_id)+"_"+str(c_d_id)+"_"+str(c_w_id)) - selected_customer = return_columns_single_record(["C_ID", "C_FIRST", "C_MIDDLE", "C_LAST", "C_STREET_1", "C_STREET_2", "C_CITY", "C_STATE", "C_ZIP", "C_PHONE", "C_SINCE", "C_CREDIT", "C_CREDIT_LIM", "C_DISCOUNT", "C_BALANCE", "C_YTD_PAYMENT", "C_PAYMENT_CNT", "C_DATA"], customer, "CUSTOMER") - #c_balance = selected_customer[14] - #c_ytd_payment = selected_customer[15] - #c_payment_cnt = selected_customer[16] - #c_data = selected_customer[17] - - updated_customer = self.increment_field("C_BALANCE", -1 * h_amount, customer, "CUSTOMER") - updated_customer = self.increment_field("C_YTD_PAYMENT", h_amount, updated_customer, "CUSTOMER") - updated_customer = self.increment_field("C_PAYMENT_CNT", 1, updated_customer, "CUSTOMER") - self.conn.set("CUSTOMER_"+str(c_id)+"_"+str(c_d_id)+"_"+str(c_w_id), updated_customer) - - #------------------------------------------------------------------------------------- - # Case 2, the customer is selected based on customer last name: all rows in the CUSTOMER table with matching - # C_W_ID, C_D_ID and C_LAST are selected sorted by C_FIRST in ascending order. Let n be the number of rows selected. - # C_ID, C_FIRST, C_MIDDLE, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, - # C_DISCOUNT, and C_BALANCE are retrieved from the row at position (n/2 rounded up to the next integer) - # in the sorted set of selected rows from the CUSTOMER table. C_BALANCE is decreased by H_AMOUNT. - # C_YTD_PAYMENT is increased by H_AMOUNT. C_PAYMENT_CNT is incremented by 1. - #------------------------------------------------------------------------------------- - - else: - - all_customers= [] - for idx in range(1, MAX_CUSTOMER_ID+1): - customer_record = self.conn.get("CUSTOMER_"+str(idx)+"_"+str(c_d_id)+"_"+str(c_w_id)) - if customer_record != None: - all_customers.append(customer_record) - - filter_customer_table = filter_table([["C_LAST",c_last]],all_customers,"CUSTOMER") - ordered_customer_map = self.order_by("C_FIRST", filter_customer_table, "CUSTOMER"); - - # Get the midpoint customer's id - namecnt = len(filter_customer_table) - assert namecnt > 0 - index = ordered_customer_map[(namecnt-1)/2][0] - customer = filter_customer_table[index] - - columns = TABLE_COLUMNS["CUSTOMER"] - c_index = columns.index('C_ID') - - c_id = customer[c_index] - - selected_customer = return_columns_single_record(["C_ID", "C_FIRST", "C_MIDDLE", "C_LAST", "C_STREET_1", "C_STREET_2", "C_CITY", "C_STATE", "C_ZIP", "C_PHONE", "C_SINCE", "C_CREDIT", "C_CREDIT_LIM", "C_DISCOUNT", "C_BALANCE", "C_YTD_PAYMENT", "C_PAYMENT_CNT", "C_DATA"], customer, "CUSTOMER") - - updated_customer = self.increment_field("C_BALANCE", -1* h_amount, customer, "CUSTOMER") - updated_customer = self.increment_field("C_YTD_PAYMENT", h_amount, updated_customer, "CUSTOMER") - updated_customer = self.increment_field("C_PAYMENT_CNT", 1, updated_customer, "CUSTOMER") - self.conn.set("CUSTOMER_"+str(c_id)+"_"+str(c_d_id)+"_"+str(c_w_id), updated_customer) - - assert len(customer) > 0 - c_data = selected_customer[17] - #------------------------------------------------------------------------------------- - # If the value of C_CREDIT is equal to "BC", then C_DATA is also retrieved from the selected customer and - # the following history information: C_ID, C_D_ID, C_W_ID, D_ID, W_ID, and H_AMOUNT, are inserted at the - # left of the C_DATA field by shifting the existing content of C_DATA to the right by an equal number of - # bytes and by discarding the bytes that are shifted out of the right side of the C_DATA field. - # The content of the C_DATA field never exceeds 500 characters. The selected customer is updated with - # the new C_DATA field. - #------------------------------------------------------------------------------------- - columns = TABLE_COLUMNS["CUSTOMER"] - credit_index = columns.index("C_CREDIT") - - if(customer[credit_index] == constants.BAD_CREDIT): - newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) - c_data = (newData + "|" + c_data) - if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] - - updated_customer = self.update_single_record("C_DATA", c_data, updated_customer,"CUSTOMER") - - #------------------------------------------------------------------------------------- - # H_DATA is built by concatenating W_NAME and D_NAME separated by 4 spaces. - # A new row is inserted into the HISTORY table with H_C_ID = C_ID, H_C_D_ID = C_D_ID, - # H_C_W_ID = C_W_ID, H_D_ID = D_ID, and H_W_ID = W_ID. - #------------------------------------------------------------------------------------- - columns = TABLE_COLUMNS["WAREHOUSE"] - w_index = columns.index("W_NAME") - columns = TABLE_COLUMNS["DISTRICT"] - d_index = columns.index("D_NAME") - - h_data = "%s %s" % (warehouse[w_index], district[d_index]) - value = [ d_id, w_id, h_date, h_amount, h_data ] - self.conn.set("HISTORY_"+str(c_id)+"_"+str(c_d_id)+"_"+str(c_w_id)+"_"+str(d_id)+"_"+str(w_id),value) - return [ selected_warehouse, selected_district, updated_customer ] - - ## ---------------------------------------------- - ## doStockLevel - ## ---------------------------------------------- - def doStockLevel(self, params): - w_id = params["w_id"] - d_id = params["d_id"] - threshold = params["threshold"] - - #------------------------------------------------------------------------------------- - # The row in the DISTRICT table with matching D_W_ID and D_ID is selected and D_NEXT_O_ID is retrieved. - #------------------------------------------------------------------------------------- - - district = self.conn.get("DISTRICT_"+str(d_id)+"_"+str(w_id)) - selected_district = return_columns_single_record(["D_NEXT_O_ID"], district, "DISTRICT") - next_o_id = selected_district[0] - #------------------------------------------------------------------------------------- - # All rows in the ORDER-LINE table with matching OL_W_ID (equals W_ID), OL_D_ID (equals D_ID), - # and OL_O_ID (lower than D_NEXT_O_ID and greater than or equal to D_NEXT_O_ID minus 20) are selected. - # They are the items for 20 recent orders of the district. - #------------------------------------------------------------------------------------- - - columns = TABLE_COLUMNS["ORDER_LINE"] - idx_O_ID = columns.index("OL_O_ID") - - # read the customers for the district and warehouse - """ - all_customers_idx = [] - for idx in range(1, MAX_CUSTOMER_ID+1): - customer_record = self.conn.get("CUSTOMER_"+str(idx)+"_"+str(d_id)+"_"+str(w_id)) - if customer_record != None: - all_customers_idx.append(idx) - """ - - # read all the orders for the customers - all_orders = [] - for idx in range(1,MAX_CUSTOMER_ID+1): - order_ids_str = self.conn.get("ORDERS_ID_"+str(idx)+"_"+str(d_id)+"_"+str(w_id)) - if order_ids_str != None: - order_ids = order_ids_str.split(":"); - for j in range(len(order_ids)-1): - o_id = int(order_ids[j]) - order_record = self.conn.get("ORDER_LINE_"+str(o_id)+"_"+str(d_id)+"_"+str(w_id)) - if order_record != None: - if order_record[idx_O_ID] < next_o_id or order_record[idx_O_ID] >= (next_o_id - 20): - all_orders.append(order_record) - - assert all_orders - order_lines_i = return_columns(["OL_O_ID"], all_orders, "ORDER_LINE") - - #------------------------------------------------------------------------------------- - # All rows in the STOCK table with matching S_I_ID (equals OL_I_ID) and S_W_ID (equals W_ID) - # from the list of distinct item numbers and with S_QUANTITY lower than threshold are counted (giving low_stock). - #------------------------------------------------------------------------------------- - - all_stock = [] - for idx in order_lines_i: - idx = idx[0] - stock = self.conn.get("STOCK_"+str(idx)+"_"+str(w_id)) - if stock != None: - quantity = return_columns_single_record(["S_QUANTITY"], stock, "STOCK") - if quantity[0] < threshold : - all_stock.append(stock) - - assert all_stock - if len(all_stock) ==1: - stock_i = return_columns_single_record(["S_I_ID"],all_stock,"STOCK") - else: - stock_i = return_columns(["S_I_ID"], all_stock, "STOCK") - - stock_i = self.count_distinct_values(stock_i) - return [ stock_i ] - - ## ---------------------------------------------- - ## Update single record - ## ---------------------------------------------- - - def update_single_record(self,field, value, record, table_name): - columns = TABLE_COLUMNS[table_name] - field_index = columns.index(field) - record[field_index] = value - return record - - - ## ---------------------------------------------- - ## increment_field - ## ---------------------------------------------- - def increment_field(self,field, value, record, table_name): - columns = TABLE_COLUMNS[table_name] - field_index = columns.index(field) - - record[field_index] += value - return record - - ## ---------------------------------------------- - ## count_distinct_values - ## ---------------------------------------------- - def count_distinct_values(self,values): - value_map={} - for val in values: - value_map[val[0]]=1 - - return len(value_map) - - ## ---------------------------------------------- - ## order_by - ## ---------------------------------------------- - #order_by("C_FIRST", filtered_customer_table, "CUSTOMER"); - - def order_by(self,field,records,table_name): - columns = TABLE_COLUMNS[table_name] - field_index = columns.index(field) - #filter_records = return_columns([field],records) - orderby_map={} - for idx,tvalue in irange(records): - field_value = tvalue[field_index] - orderby_map[idx] = field_value - - import operator - sorted_index_list = sorted(orderby_map.items(), key=operator.itemgetter(1),reverse=False) - - return sorted_index_list - #store index, value & then sort... diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/redisdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/redisdriver.py deleted file mode 100644 index 48108fdd..00000000 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/redisdriver.py +++ /dev/null @@ -1,1663 +0,0 @@ -import os, redis, time, sys -from datetime import datetime -from pprint import pprint,pformat -from abstractdriver import * - -#---------------------------------------------------------------------------- -# Redis TPC-C Driver -# -# @author Christopher Keith -# @author James Tavares -#---------------------------------------------------------------------------- -class RedisDriver(AbstractDriver): - - KEY_SEPARATOR = ':' - - TABLES = { - 'WAREHOUSE' : { - 'columns' : - [ - 'W_ID', - 'W_NAME', - 'W_STREET_1', - 'W_STREET_2', - 'W_CITY', - 'W_STATE', - 'W_ZIP', - 'W_TAX', - 'W_YTD', - ], - 'map' : - { - 'W_ID' : 0, - 'W_NAME' : 1, - 'W_STREET_1' : 2, - 'W_STREET_2' : 3, - 'W_CITY' : 4, - 'W_STATE' : 5, - 'W_ZIP' : 6, - 'W_TAX' : 7, - 'W_YTD' : 8, - }, - 'primary_key' : ['W_ID'], - 'indexes' : [ ], - }, - 'DISTRICT' : { - 'columns' : - [ - 'D_ID', - 'D_W_ID', - 'D_NAME', - 'D_STREET_1', - 'D_STREET_2', - 'D_CITY', - 'D_STATE', - 'D_ZIP', - 'D_TAX', - 'D_YTD', - 'D_NEXT_O_ID', - ], - 'map' : - { - 'D_ID' : 0, - 'D_W_ID' : 1, - 'D_NAME' : 2, - 'D_STREET_1' : 3, - 'D_STREET_2' : 4, - 'D_CITY' : 5, - 'D_STATE' : 6, - 'D_ZIP' : 7, - 'D_TAX' : 8, - 'D_YTD' : 9, - 'D_NEXT_O_ID' : 10, - }, - 'primary_key' : ['D_W_ID', 'D_ID'], - 'indexes' : [ ], - }, - 'ITEM' : { - 'columns' : - [ - 'I_ID', - 'I_IM_ID', - 'I_NAME', - 'I_PRICE', - 'I_DATA', - ], - 'map' : - { - 'I_ID' : 0, - 'I_IM_ID' : 1, - 'I_NAME' : 2, - 'I_PRICE' : 3, - 'I_DATA' : 4, - }, - 'primary_key' : ['I_ID'], - 'indexes' : [ ] - }, - 'CUSTOMER' : { - 'columns' : - [ - 'C_ID', - 'C_D_ID', - 'C_W_ID', - 'C_FIRST', - 'C_MIDDLE', - 'C_LAST', - 'C_STREET_1', - 'C_STREET_2', - 'C_CITY', - 'C_ZIP', - 'C_PHONE', - 'C_SINCE', - 'C_CREDIT', - 'C_CREDIT_LIM', - 'C_DISCOUNT', - 'C_BALANCE', - 'C_YTD_PAYMENT', - 'C_PAYMENT_CNT', - 'C_DELIVERY_CNT', - 'C_DATA', - ], - 'map' : - { - 'C_ID' : 0, - 'C_D_ID' : 1, - 'C_W_ID' : 2, - 'C_FIRST' : 3, - 'C_MIDDLE' : 4, - 'C_LAST' : 5, - 'C_STREET_1' : 6, - 'C_STREET_2' : 7, - 'C_CITY' : 8, - 'C_ZIP' : 9, - 'C_PHONE' : 10, - 'C_SINCE' : 11, - 'C_CREDIT' : 12, - 'C_CREDIT_LIM' : 13, - 'C_DISCOUNT' : 14, - 'C_BALANCE' : 15, - 'C_YTD_PAYMENT' : 16, - 'C_PAYMENT_CNT' : 17, - 'C_DELIVERY_CNT' : 18, - 'C_DATA' : 19, - }, - 'primary_key' : ['C_W_ID', 'C_D_ID', 'C_ID'], - 'indexes' : [ ], - }, - 'HISTORY' : { - 'columns' : - [ - 'H_C_ID', - 'H_C_D_ID', - 'H_C_W_ID', - 'H_D_ID', - 'H_W_ID', - 'H_DATE', - 'H_AMOUNT', - 'H_DATA', - ], - 'map' : - { - 'H_C_ID' : 0, - 'H_C_D_ID' : 1, - 'H_C_W_ID' : 2, - 'H_D_ID' : 3, - 'H_W_ID' : 4, - 'H_DATE' : 5, - 'H_AMOUNT' : 6, - 'H_DATA' : 7, - }, - 'primary_key' : [ ], - 'indexes' : [ ], - }, - 'STOCK' : { - 'columns' : - [ - 'S_I_ID', - 'S_W_ID', - 'S_QUANTITY', - 'S_DIST_01', - 'S_DIST_02', - 'S_DIST_03', - 'S_DIST_04', - 'S_DIST_05', - 'S_DIST_06', - 'S_DIST_07', - 'S_DIST_08', - 'S_DIST_09', - 'S_DIST_10', - 'S_YTD', - 'S_ORDER_CNT', - 'S_REMOTE_CNT', - 'S_DATA', - ], - 'map' : - { - 'S_I_ID' : 0, - 'S_W_ID' : 1, - 'S_QUANTITY' : 2, - 'S_DIST_01' : 3, - 'S_DIST_02' : 4, - 'S_DIST_03' : 5, - 'S_DIST_04' : 6, - 'S_DIST_05' : 7, - 'S_DIST_06' : 8, - 'S_DIST_07' : 9, - 'S_DIST_08' : 10, - 'S_DIST_09' : 11, - 'S_DIST_10' : 12, - 'S_YTD' : 13, - 'S_ORDER_CNT' : 14, - 'S_REMOTE_CNT' : 15, - 'S_DATA' : 16, - }, - 'primary_key' : ['S_W_ID', 'S_I_ID'], - 'indexes' : [ ], - }, - 'ORDERS' : { - 'columns' : - [ - 'O_ID', - 'O_D_ID', - 'O_W_ID', - 'O_C_ID', - 'O_ENTRY_D', - 'O_CARRIER_ID', - 'O_OL_CNT', - 'O_ALL_LOCAL', - ], - 'map' : - { - 'O_ID' : 0, - 'O_D_ID' : 1, - 'O_W_ID' : 2, - 'O_C_ID' : 3, - 'O_ENTRY_D' : 4, - 'O_CARRIER_ID' : 5, - 'O_OL_CNT' : 6, - 'O_ALL_LOCAL' : 7, - }, - 'primary_key' : ['O_ID', 'O_C_ID', 'O_D_ID', 'O_W_ID'], - 'indexes' : [ ], - }, - 'NEW_ORDER' : { - 'columns' : - [ - 'NO_O_ID', - 'NO_D_ID', - 'NO_W_ID', - ], - 'map' : - { - 'NO_O_ID' : 0, - 'NO_D_ID' : 1, - 'NO_W_ID' : 2, - }, - 'primary_key' : ['NO_D_ID', 'NO_W_ID', 'NO_O_ID'], - 'indexes' : [ ], - }, - 'ORDER_LINE' : { - 'columns' : - [ - 'OL_O_ID', - 'OL_D_ID', - 'OL_W_ID', - 'OL_NUMBER', - 'OL_I_ID', - 'OL_SUPPLY_W_ID', - 'OL_DELIVERY_D', - 'OL_QUANTITY', - 'OL_AMOUNT', - 'OL_DIST_INFO', - ], - 'map' : - { - 'OL_O_ID' : 0, - 'OL_D_ID' : 1, - 'OL_W_ID' : 2, - 'OL_NUMBER' : 3, - 'OL_I_ID' : 4, - 'OL_SUPPLY_W_ID' : 5, - 'OL_DELIVERY_D' : 6, - 'OL_QUANTITY' : 7, - 'OL_AMOUNT' : 8, - 'OL_DIST_INFO' : 9, - }, - 'primary_key' : ['OL_W_ID', 'OL_D_ID', 'OL_O_ID', 'OL_NUMBER'], - 'indexes' : [ ], - } - } - - DEFAULT_CONFIG = { - 'databases' : ("List of Redis Hosts", "127.0.0.1:6379"), - 'host-info' : ("Show information about hosts", 'Verbose'), - 'debug-load' : ("Show Loading Information", 'None'), - 'debug-delivery' : ("Show Delivery Performance", 'None'), - 'debug-new-order' : ("Show New Order Performance", 'None'), - 'debug-order-status' : ("Show Order Status Performance", 'None'), - 'debug-payment' : ("Show Payment Performance", 'None'), - 'debug-stock-level' : ("Show Stock Level Performance", 'None'), - } - - #------------------------------------------------------------------------ - # Class constructor - # - # @param string ddl (Data Definintion Language) - #------------------------------------------------------------------------ - def __init__(self, ddl) : - super(RedisDriver, self).__init__("redis", ddl) - self.databases = [ ] - self.t0 = 0 - self.db_count = 0 - self.metadata = None - self.r_pipes = [ ] - self.r_sizes = [ ] - self.w_pipes = [ ] - self.w_sizes = [ ] - self.debug = { - 'load' : 'None', - 'delivery' : 'None', - 'new-order' : 'None', - 'order-status' : 'None', - 'payment' : 'None', - 'stock-level' : 'None', - } - self.hosts = [ ] - # End __init__() - - #------------------------------------------------------------------------ - # Execute TPC-C Delivery Transaction - # - # @param dictionary params (transaction parameters) - # { - # "w_id" : value, - # "o_carrier_id" : value, - # "ol_delivery_d" : value, - # } - #------------------------------------------------------------------------ - def doDelivery(self, params) : - if self.debug['delivery'] != 'None' : - print 'TXN DELIVERY STARTING ------------------' - tt = time.time() - if self.debug['delivery'] == 'Verbose' : - t0 = tt - - # Initialize input parameters - w_id = params["w_id"] - o_carrier_id = params["o_carrier_id"] - ol_delivery_d = params["ol_delivery_d"] - - # Setup Redis pipelining - node = self.shard(w_id) - rdr = self.r_pipes[node] - wtr = self.w_pipes[node] - - # Initialize result set - result = [ ] - - #------------------------- - # Initialize Data Holders - #------------------------- - order_key = [ ] - ol_total = [ ] - customer_key = [ ] - ol_counts = [ ] - no_o_id = [ ] - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : - order_key.append(None) - ol_total.append(0) - customer_key.append(None) - ol_counts.append(0) - - #--------------------- - # Get New Order Query - #--------------------- - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : - cursor = d_id - 1 - # Get set of possible new order ids - index_key = self.safeKey([d_id, w_id]) - rdr.srandmember('NEW_ORDER.INDEXES.GETNEWORDER.' + index_key) - id_set = rdr.execute() - - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : - cursor = d_id - 1 - if id_set[cursor] == None : - rdr.get('NULL_VALUE') - else : - rdr.hget('NEW_ORDER.' + str(id_set[cursor]), 'NO_O_ID') - no_o_id = rdr.execute() - - if self.debug['delivery'] == 'Verbose' : - print 'New Order Query: ', time.time() - t0 - t0 = time.time() - - #----------------------- - # Get Customer ID Query - #----------------------- - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : - cursor = d_id - 1 - if no_o_id[cursor] == None : - order_key.insert(cursor, 'NO_KEY') - else : - order_key.insert( - cursor, - self.safeKey([w_id, d_id, no_o_id[0]]) - ) - rdr.hget('ORDERS.' + order_key[cursor], 'O_C_ID') - c_id = rdr.execute() - - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : - cursor = d_id - 1 - if no_o_id[cursor] == None or c_id[cursor] == None : - si_key = 'NO_KEY' - else : - si_key = self.safeKey([no_o_id[cursor], w_id, d_id]) - rdr.smembers('ORDER_LINE.INDEXES.SUMOLAMOUNT.' + si_key) - ol_ids = rdr.execute() - - if self.debug['delivery'] == 'Verbose' : - print 'Get Customer ID Query:', time.time() - t0 - t0 = time.time() - - #----------------------------- - # Sum Order Line Amount Query - #----------------------------- - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : - cursor = d_id - 1 - if no_o_id[cursor] == None or c_id[cursor] == None : - rdr.get('NULL_VALUE') - else : - for i in ol_ids[cursor] : - rdr.hget('ORDER_LINE.' + str(i), 'OL_AMOUNT') - ol_counts[cursor] += 1 - - pipe_results = rdr.execute() - index = 0 - counter = 0 - - for ol_amount in pipe_results : - counter += 1 - if counter > ol_counts[index] : - index += 1 - counter = 0 - elif ol_amount != None : - ol_total[index] += float(ol_amount) - - if self.debug['delivery'] == 'Verbose' : - print 'Sum Order Line Query:', time.time() - t0 - t0 = time.time() - - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : - cursor = d_id - 1 - if no_o_id[cursor] == None or c_id[cursor] == None : - ## No orders for this district: skip it. - ## Note: This must be reported if > 1% - continue - - #------------------------ - # Delete New Order Query - #------------------------ - no_key = self.safeKey([d_id, w_id, no_o_id[cursor]]) - no_si_key = self.safeKey([d_id, w_id]) - wtr.delete('NEW_ORDER.' + no_key) - wtr.srem('NEW_ORDER.IDS', no_key) - wtr.srem('NEW_ORDER.INDEXES.GETNEWORDER.' + no_si_key, no_key) - - if self.debug['delivery'] == 'Verbose' : - print 'Delete New Order Query:', time.time() - t0 - t0 = time.time() - - #--------------------- - # Update Orders Query - #--------------------- - wtr.hset( - 'ORDERS.' + order_key[cursor], - 'W_CARRIER_ID', - o_carrier_id - ) - - if self.debug['delivery'] == 'Verbose' : - print 'Update Orders Query:', time.time() - t0 - t0 = time.time() - - #------------------------- - # Update Order Line Query - #------------------------- - for i in ol_ids[cursor] : - wtr.hset( - 'ORDER_LINE.' + str(i), - 'OL_DELIVERY_D', - ol_delivery_d - ) - - if self.debug['delivery'] == 'Verbose' : - print 'Update Order Line Query:', time.time() - t0 - t0 = time.time() - - #----------------------- - # Update Customer Query - #----------------------- - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : - cursor = d_id - 1 - if no_o_id[cursor] == None or c_id[cursor] == None : - rdr.get('NULL_VALUE') - customer_key.insert(cursor, 'NO_KEY') - else : - customer_key.insert( - cursor, - self.safeKey([w_id, d_id, c_id[cursor]]) - ) - rdr.hget('CUSTOMER.' + customer_key[cursor], 'C_BALANCE') - old_balance = rdr.execute() - - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1) : - cursor = d_id - 1 - if no_o_id[cursor] == None or c_id[cursor] == None : - continue - else : - new_balance = float(old_balance[cursor]) + float(ol_total[cursor]) - wtr.hset( - 'CUSTOMER.' + customer_key[cursor], - 'C_BALANCE', - new_balance - ) - result.append((d_id, no_o_id[cursor])) - wtr.execute() - - if self.debug['delivery'] == 'Verbose' : - print 'Update Customer Query:', time.time() - t0 - if self.debug['delivery'] != 'None' : - print 'TXN DELIVERY:', time.time() - tt - - return result - # End doDelivery() - - #------------------------------------------------------------------------ - # Execute TPC-C New Order Transaction - # - # @param dictionary params (transaction parameters) - # { - # 'w_id' : value, - # 'd_id' : value, - # 'c_id' : value, - # 'o_entry_d' : value, - # 'i_ids' : value, - # 'i_w_ids' : value, - # 'i_qtys' : value, - # } - #------------------------------------------------------------------------ - def doNewOrder(self, params) : - if self.debug['new-order'] != 'None' : - print 'TXN NEW ORDER STARTING -----------------' - tt = time.time() - if self.debug['new-order'] == 'Verbose' : - t0 = tt - - # Initialize transaction parameters - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - o_entry_d = params["o_entry_d"] - i_ids = params["i_ids"] - i_w_ids = params["i_w_ids"] - i_qtys = params["i_qtys"] - - # Setup Redis pipelining - node = self.shard(w_id) - rdr = self.r_pipes[node] - wtr = self.w_pipes[node] - - # Validate transaction parameters - assert len(i_ids) > 0 - assert len(i_ids) == len(i_w_ids) - assert len(i_ids) == len(i_qtys) - - # Check if all items are local - all_local = True - items = [ ] - pipe_results = [ ] - for i in range(len(i_ids)): - all_local = all_local and i_w_ids[i] == w_id - rdr.hgetall('ITEM.' + str(i_ids[i])) - pipe_results = rdr.execute() - - for pr in pipe_results : - if len(pr) > 0 : - if pr['I_PRICE'] == None and pr['I_NAME'] == None and pr['I_DATA'] == None : - result = [ ] - items.append([ - pr['I_PRICE'], - pr['I_NAME'], - pr['I_DATA'], - ]) - else : - items.append([]) - - assert len(items) == len(i_ids) - - ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. - ## Note that this will happen with 1% of transactions on purpose. - for item in items : - if len(item) == 0 : - return - - #------------------------------------------------------------ - # Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER - #------------------------------------------------------------ - district_key = self.safeKey([w_id, d_id]) - customer_key = self.safeKey([w_id, d_id, c_id]) - - #------------------------------ - # Get Warehouse Tax Rate Query - #------------------------------ - rdr.hget('WAREHOUSE.' + str(w_id), 'W_TAX') - - if self.debug['new-order'] == 'Verbose' : - print 'Get Warehouse Tax Rate Query:', time.time() - t0 - t0 = time.time() - - #-------------------- - # Get District Query - #-------------------- - rdr.hgetall('DISTRICT.' + district_key) - - if self.debug['new-order'] == 'Verbose' : - print 'Get District Query:', time.time() - t0 - t0 = time.time() - - #-------------------- - # Get Customer Query - #-------------------- - rdr.hgetall('CUSTOMER.' + customer_key) - rdr_results = rdr.execute() - - w_tax = float(rdr_results[0]) - d_tax = float(rdr_results[1]['D_TAX']) - d_next_o_id = rdr_results[1]['D_NEXT_O_ID'] - customer_info = rdr_results[2] - c_discount = float(rdr_results[2]['C_DISCOUNT']) - - if self.debug['new-order'] == 'Verbose' : - print 'Get Customer Query:', time.time() - t0 - t0 = time.time() - - #-------------------------- - # Insert Order Information - #-------------------------- - ol_cnt = len(i_ids) - o_carrier_id = constants.NULL_CARRIER_ID - order_key = self.safeKey([w_id, d_id, d_next_o_id]) - new_order_key = self.safeKey([d_next_o_id, w_id, d_id]) - - #------------------------------- - # Increment Next Order ID Query - #------------------------------- - wtr.hincrby('DISTRICT.' + district_key, 'D_NEXT_O_ID') - - if self.debug['new-order'] == 'Verbose' : - print 'Increment Next Order ID Query:', time.time() - t0 - t0 = time.time() - - #-------------------- - # Create Order Query - #-------------------- - wtr.sadd('ORDERS.IDS', order_key) - wtr.hmset( - 'ORDERS.' + order_key, - { - 'O_ID' : d_next_o_id, - 'O_D_ID' : d_id, - 'O_W_ID' : w_id, - 'O_C_ID' : c_id, - 'O_ENTRY_D' : o_entry_d, - 'O_CARRIER_ID' : o_carrier_id, - 'O_OL_CNT' : ol_cnt, - 'O_ALL_LOCAL' : all_local - } - ) - - if self.debug['new-order'] == 'Verbose' : - print 'Create Order Query:', time.time() - t0 - t0 = time.time() - - # Add index for Order searching - si_key = self.safeKey([w_id, d_id, c_id]) - wtr.sadd('ORDERS.INDEXES.ORDERSEARCH', si_key) - - #------------------------ - # Create New Order Query - #------------------------ - wtr.sadd('NEW_ORDER.IDS', new_order_key) - wtr.hmset( - 'NEW_ORDER.' + new_order_key, - { - 'NO_O_ID' : d_next_o_id, - 'NO_D_ID' : d_id, - 'NO_W_ID' : w_id, - } - ) - - # Add index for New Order Searching - si_key = self.safeKey([d_id, w_id]) - wtr.sadd('NEW_ORDER.INDEXES.GETNEWORDER', si_key) - - if self.debug['new-order'] == 'Verbose' : - print 'Create New Order Query:', time.time() - t0 - t0 = time.time() - - #------------------------------- - # Insert Order Item Information - #------------------------------- - item_data = [ ] - total = 0 - - ol_number = [ ] - ol_quantity = [ ] - ol_supply_w_id = [ ] - ol_i_id = [ ] - i_name = [ ] - i_price = [ ] - i_data = [ ] - stock_key = [ ] - stock_info = [ ] - - for i in range(len(i_ids)) : - ol_number.append(i + 1) - ol_supply_w_id.append(i_w_ids[i]) - ol_i_id.append(i_ids[i]) - ol_quantity.append(i_qtys[i]) - - itemInfo = items[i] - i_name.append(itemInfo[1]) - i_data.append(itemInfo[2]) - i_price.append(float(itemInfo[0])) - - #----------------------------- - # Get Stock Information Query - #----------------------------- - stock_key.append(self.safeKey([ol_supply_w_id[i], ol_i_id[i]])) - rdr.hgetall('STOCK.' + stock_key[i]) - stock_info = rdr.execute() - - for index, si in enumerate(stock_info) : - if len(si) == 0 : - continue - s_quantity = float(si['S_QUANTITY']) - s_ytd = float(si['S_YTD']) - s_order_cnt = float(si['S_ORDER_CNT']) - s_remote_cnt = float(si['S_REMOTE_CNT']) - s_data = si['S_DATA'] - if len(str(d_id)) == 1 : - s_dist_xx = si['S_DIST_0' + str(d_id)] - else : - s_dist_xx = si['S_DIST_' + str(d_id)] - - if self.debug['new-order'] == 'Verbose' : - print 'Get Stock Information Query:', time.time() - t0 - t0 = time.time() - - #-------------------- - # Update Stock Query - #-------------------- - s_ytd += ol_quantity[index] - if s_quantity >= ol_quantity[index] + 10 : - s_quantity = s_quantity - ol_quantity[index] - else : - s_quantity = s_quantity + 91 - ol_quantity[index] - s_order_cnt += 1 - - if ol_supply_w_id[index] != w_id : s_remote_cnt += 1 - - wtr.hmset( - 'STOCK.' + stock_key[index], - { - 'S_QUANTITY' : s_quantity, - 'S_YTD' : s_ytd, - 'S_ORDER_CNT' : s_order_cnt, - 'S_REMOTE_CNT' : s_remote_cnt - } - ) - - if i_data[index].find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: - brand_generic = 'B' - else: - brand_generic = 'G' - - ## Transaction profile states to use "ol_quantity * i_price" - ol_amount = ol_quantity[index] * i_price[index] - total += ol_amount - - if self.debug['new-order'] == 'Verbose' : - print 'Update Stock Query:', time.time() - t0 - t0 = time.time() - - #------------------------- - # Create Order Line Query - #------------------------- - order_line_key = self.safeKey([w_id, d_id,d_next_o_id, ol_number]) - si_key = self.safeKey([d_next_o_id, d_id, w_id]) - wtr.sadd('ORDER_LINE.IDS', order_line_key) - wtr.hmset( - 'ORDER_LINE.' + order_line_key, - { - 'OL_O_ID' : d_next_o_id, - 'OL_D_ID' : d_id, - 'OL_W_ID' : w_id, - 'OL_NUMBER' : ol_number[index], - 'OL_I_ID' : ol_i_id[index], - 'OL_SUPPLY_W_ID' : ol_supply_w_id[index], - 'OL_DELIVERY_D' : o_entry_d, - 'OL_QUANTITY' : ol_quantity[index], - 'OL_AMOUNT' : ol_amount, - 'OL_DISTRICT_INFO' : s_dist_xx - } - ) - - # Create index for Order Line Searching - wtr.sadd( - 'ORDER_LINE.INDEXES.SUMOLAMOUNT.' + si_key, - order_line_key - ) - - if self.debug['new-order'] == 'Verbose' : - print 'Create Order Line Query:', time.time() - t0 - t0 = time.time() - - ## Add the info to be returned - item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) - ## End for i in range(len(i_ids)) : - - ## Commit! - wtr.execute() - - ## Adjust the total for the discount - total *= (1 - c_discount) * (1 + w_tax + d_tax) - - ## Pack up values the client is missing (see TPC-C 2.4.3.5) - misc = [ (w_tax, d_tax, d_next_o_id, total) ] - - if self.debug['new-order'] != 'None' : - print 'TXN NEW ORDER:', time.time() - tt - return [ customer_info, misc, item_data ] - - #------------------------------------------------------------------------ - # Execute TPC-C Do Order Status transaction - # - # @param dictionary params (transaction parameters) - # { - # 'w_id' : value, - # 'd_id' : value, - # 'c_id' : value, - # 'c_last' : value, - # } - #------------------------------------------------------------------------ - def doOrderStatus(self, params) : - if self.debug['order-status'] != 'None' : - print 'TXN ORDER STATUS STARTING --------------' - tt = time.time() - if self.debug['order-status'] == 'Verbose' : - t0 = tt - - # Initialize transactions parameters - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - - # Initialize Redis pipelining - node = self.shard(w_id) - rdr = self.databases[node].pipeline(False) - wtr = self.databases[node].pipeline(True) - - # Validate transaction parameters - assert w_id, pformat(params) - assert d_id, pformat(params) - - if c_id != None: - #----------------------------------- - # Get Customer By Customer ID Query - #----------------------------------- - customer_key = self.safeKey([w_id, d_id, c_id]) - rdr.hgetall('CUSTOMER.' + customer_key) - results = rdr.execute(); - customer = results[0] - - if self.debug['order-status'] == 'Verbose' : - print 'Get Customer By Customer ID Query:', time.time() - t0 - t0 = time.time() - else: - #---------------------------------- - # Get Customers By Last Name Query - #---------------------------------- - si_key = self.safeKey([w_id, d_id, c_last]) - rdr.smembers('CUSTOMER.INDEXES.NAMESEARCH.' + si_key) - results = rdr.execute(); - customer_id_set = results[0] - - customer_ids = [ ] - for customer_id in customer_id_set : - rdr.hgetall('CUSTOMER.' + str(customer_id)) - customer_ids.append(str(customer_id)) - - customers = [ ] - unsorted_customers = rdr.execute() - customers.append(unsorted_customers.pop()) - for cust in unsorted_customers : - for index in range(len(customers)) : - if cust['C_FIRST'] < customers[index]['C_FIRST'] : - customers.insert(index, cust) - continue - - assert len(customers) > 0 - - namecnt = len(customers) - index = (namecnt - 1)/2 - customer = customers[index] - customer_key = self.safeKey([ - customer['C_W_ID'], - customer['C_D_ID'], - customer['C_ID'] - ]) - c_id = customer['C_ID'] - - if self.debug['order-status'] == 'Verbose' : - print 'Get Customers By Last Name Query:', time.time() - t0 - t0 = time.time() - assert len(customer) > 0 - assert c_id != None - - #---------------------- - # Get Last Order Query - #---------------------- - search_key = self.safeKey([w_id, d_id, c_id]) - rdr.smembers('ORDERS.INDEXES.ORDERSEARCH.' + search_key) - results = rdr.execute() - order_id_set = results[0] - - if self.debug['order-status'] == 'Verbose' : - print 'Get Last Order Query:', time.time() - t0 - t0 = time.time() - - order = [ ] - if len(order_id_set) > 0 : - order_ids = sorted(list(order_id_set)) - order_key = str(order_ids[0]) - rdr.hgetall('ORDERS.' + order_key) - - result = rdr.execute() - order = [ - result[0]['O_ID'], - result[0]['O_CARRIER_ID'], - result[0]['O_ENTRY_D'], - ] - - #----------------------- - # Get Order Lines Query - #----------------------- - search_key = self.safeKey([order[0], d_id, w_id]) - rdr.smembers('ORDER_LINE.INDEXES.SUMOLAMOUNT.' + search_key) - results = rdr.execute() - line_ids = results[0] - - for line_id in line_ids : - rdr.hgetall('ORDER_LINE.' + str(line_id)) - - orderLines = [ ] - results = rdr.execute() - for r in results : - orderLines.append([ - r['OL_SUPPLY_W_ID'], - r['OL_I_ID'], - r['OL_QUANTITY'], - r['OL_AMOUNT'], - r['OL_DELIVERY_D'] - ]) - - if self.debug['order-status'] == 'Verbose' : - print 'Get Order Lines Query:', time.time() - t0 - else : - orderLines = [ ] - - if self.debug['order-status'] != 'None' : - print 'TXN ORDER STATUS:', time.time() - tt - - return [ customer, order, orderLines ] - - #------------------------------------------------------------------------ - # Execute TPC-C Do Payement Transaction - # - # @param dictionary params (transaction parameters) - # { - # 'w_id' : value, - # 'd_id' : value, - # 'h_amount' : value, - # 'c_w_id' : value, - # 'c_d_id' : value, - # 'c_id' : value, - # 'c_last' : value, - # 'h_date' : value, - # } - #------------------------------------------------------------------------ - def doPayment(self, params) : - if self.debug['payment'] != 'None' : - print 'TXN PAYMENT STARTING -------------------' - tt = time.time() - if self.debug['payment'] == 'Verbose' : - t0 = tt - - # Initialize transaction properties - w_id = params["w_id"] - d_id = params["d_id"] - h_amount = params["h_amount"] - c_w_id = params["c_w_id"] - c_d_id = params["c_d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - h_date = params["h_date"] - - # Initialize Redis pipeline - node = self.shard(w_id) - rdr = self.r_pipes[node] - wtr = self.w_pipes[node] - - t0 = time.time() - if c_id != None: - #-------------------------- - # Get Customer By ID Query - #-------------------------- - customer_key = self.safeKey([w_id, d_id, c_id]) - rdr.hgetall('CUSTOMER.' + customer_key) - results = rdr.execute() - customer = results[0] - - if self.debug['payment'] == 'Verbose' : - print 'Get Customer By ID Query:', time.time() - t0 - t0 = time.time() - else: - #---------------------------------- - # Get Customers By Last Name Query - #---------------------------------- - si_key = self.safeKey([w_id, d_id, c_last]) - rdr.smembers('CUSTOMER.INDEXES.NAMESEARCH.' + si_key) - results = rdr.execute() - customer_id_set = results[0] - - customer_ids = [ ] - for customer_id in customer_id_set : - rdr.hgetall('CUSTOMER.' + str(customer_id)) - customer_ids.append(str(customer_id)) - - customers = [ ] - unsorted_customers = rdr.execute() - customers.append(unsorted_customers.pop()) - for cust in unsorted_customers : - for index in range(len(customers)) : - if cust['C_FIRST'] < customers[index]['C_FIRST'] : - customers.insert(index, cust) - continue - - assert len(customers) > 0 - - namecnt = len(customers) - index = (namecnt - 1)/2 - customer = customers[index] - customer_key = self.safeKey([ - customer['C_W_ID'], - customer['C_D_ID'], - customer['C_ID'] - ]) - c_id = customer['C_ID'] - - if self.debug['payment'] == 'Verbose' : - print 'Get Customers By Last Name Query:', time.time() - t0 - t0 = time.time() - - assert len(customer) > 0 - assert c_id != None - - c_balance = float(customer['C_BALANCE']) - h_amount - c_ytd_payment = float(customer['C_YTD_PAYMENT']) + h_amount - c_payment_cnt = float(customer['C_PAYMENT_CNT']) + 1 - c_data = customer['C_DATA'] - - #--------------------- - # Get Warehouse Query - #--------------------- - rdr.hgetall('WAREHOUSE.' + str(w_id)) - - if self.debug['payment'] == 'Verbose' : - print 'Get Warehouse Query:', time.time() - t0 - t0 = time.time() - - #-------------------- - # Get District Query - #-------------------- - district_key = self.safeKey([w_id, d_id]) - rdr.hgetall('DISTRICT.' + district_key) - warehouse, district = rdr.execute() - - if self.debug['payment'] == 'Verbose' : - print 'Get District Query:', time.time() - t0 - t0 = time.time() - - #-------------------------------- - # Update Warehouse Balance Query - #-------------------------------- - wtr.set( - 'WAREHOUSE.' + str(w_id) + '.W_YTD', - float(warehouse['W_YTD']) + h_amount - ) - - if self.debug['payment'] == 'Verbose' : - print 'Update Warehouse Query:', time.time() - t0 - t0 = time.time() - - #------------------------------- - # Update District Balance Query - #------------------------------- - wtr.set( - 'DISTRICT.' + district_key + '.D_YTD', - float(district['D_YTD']) + h_amount - ) - - if self.debug['payment'] == 'Verbose' : - print 'Update District Balance Query:', time.time() - t0 - t0 = time.time() - - if customer['C_CREDIT'] == constants.BAD_CREDIT: - #---------------------------------- - # Update Bad Credit Customer Query - #---------------------------------- - newData = " ".join( - map( - str, - [c_id, c_d_id, c_w_id, d_id, w_id, h_amount] - ) - ) - - c_data = (newData + "|" + c_data) - if len(c_data) > constants.MAX_C_DATA : - c_data = c_data[:constants.MAX_C_DATA] - - wtr.hmset( - 'CUSTOMER.' + customer_key, - { - 'C_BALANCE' : c_balance, - 'C_YTD_PAYMENT' : c_ytd_payment, - 'C_PAYMENT_CNT' : c_payment_cnt, - 'C_DATA' : c_data, - } - ) - - if self.debug['payment'] == 'Verbose' : - print 'Update Bad Credit Customer Query:', time.time() - t0 - t0 = time.time() - else: - #----------------------------------- - # Update Good Credit Customer Query - #----------------------------------- - wtr.hmset( - 'CUSTOMER.' + customer_key, - { - 'C_BALANCE' : c_balance, - 'C_YTD_PAYMENT' : c_ytd_payment, - 'C_PAYMENT_CNT' : c_payment_cnt, - 'C_DATA' : '', - } - ) - if self.debug['payment'] == 'Verbose' : - print 'Update Good Credit Customer Query:', time.time() - t0 - t0 = time.time() - - wtr.execute() - - # Concatenate w_name, four spaces, d_name - h_data = "%s %s" % (warehouse['W_NAME'], district['D_NAME']) - - #---------------------- - # Insert History Query - #---------------------- - next_score = self.metadata.get('HISTORY.next_score') - self.metadata.incr('HISTORY.next_score') - - history_key = self.safeKey(next_score) - wtr.hmset( - 'HISTORY.' + history_key, - { - 'H_C_ID' : c_id, - 'H_C_D_ID' : c_d_id, - 'H_C_W_ID' : c_w_id, - 'H_D_ID' : d_id, - 'H_W_ID' : w_id, - 'H_DATE' : h_date, - 'H_AMOUNT' : h_amount, - 'H_DATA' : h_data, - } - ) - - if self.debug['payment'] == 'Verbose' : - print 'Insert History Query:', time.time() - t0 - t0 = time.time() - - wtr.execute() - - - # TPC-C 2.5.3.3: Must display the following fields: - # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, - # W_STATE, W_ZIP, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, - # C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, - # C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, - # C_BALANCE, the first 200 characters of C_DATA - # (only if C_CREDIT = "BC"), H_AMOUNT, and H_DATE. - - # Hand back all the warehouse, district, and customer data - - if self.debug['payment'] != 'None' : - print 'TXN PAYMENT:', time.time() - tt - - return [ warehouse, district, customer ] - - #------------------------------------------------------------------------ - # Execute TPC-C Stock Level Transaction - # - # @param dictionary params (transaction parameters) - # { - # 'w_id' : value, - # 'd_id' : value, - # 'threshold : value, - # } - #------------------------------------------------------------------------ - def doStockLevel(self, params) : - if self.debug['order-status'] != 'None' : - print 'TXN STOCK LEVEL STARTING ---------------' - tt = time.time() - if self.debug['stock-level'] == 'Verbose' : - t0 = tt - - # Initialize transaction parameters - w_id = params["w_id"] - d_id = params["d_id"] - threshold = params["threshold"] - - # Setup Redis pipelining - node = self.shard(w_id) - rdr = self.r_pipes[node] - wtr = self.w_pipes[node] - - #-------------------- - # Get Order ID Query - #-------------------- - district_key = self.safeKey([w_id, d_id]) - rdr.hget('DISTRICT.' + district_key, 'D_NEXT_O_ID') - results = rdr.execute() - o_id = results[0] - - if self.debug['stock-level'] == 'Verbose' : - print 'Get Order ID Query:', time.time() - t0 - t0 = time.time() - - #----------------------- - # Get Stock Count Query - #----------------------- - stock_counts = {} - si_key = self.safeKey([o_id, d_id, w_id]) - rdr.smembers('ORDER_LINE.INDEXES.SUMOLAMOUNT.' + si_key) - results = rdr.execute() - line_ids = results[0] - - for line_id in line_ids : - rdr.hgetall('ORDER_LINE.' + str(line_id)) - order_lines = rdr.execute() - - for line['OL_I_ID'] in order_lines : - stock_key = self.safeKey([w_id, line]) - rdr.hget('STOCK.' + stock_key, 'S_QUANTITY') - stocks = rdr.execute() - - for index in range(len(order_lines)) : - if int(order_lines[index]['OL_I_ID']) < int(o_id) and int(order_lines[index]['OL_I_ID']) > int(o_id) - 20 and float(stocks[index]) < threshold : - stock_counts[order_lines[index]['OL_I_ID']] = order_lines[index]['OL_I_ID'] - - if self.debug['stock-level'] == 'Verbose' : - print 'Get Stock Count Query:', time.time() - t0 - if self.debug['stock-level'] != 'None' : - print 'TXN STOCK LEVEL:', time.time() - tt - - return len(stock_counts) - - #------------------------------------------------------------------------ - # Load the specified configuration for Redis TPC-C run - # - # @param dictionary config (configuration options) - #------------------------------------------------------------------------ - def loadConfig(self, config) : - for key in RedisDriver.DEFAULT_CONFIG.keys() : - assert key in config, "Missing parameter '%s' in the %s configuration" % (key, self.name) - - hosts = config["databases"].split() - first = True - c_num = 0 - for host in hosts : - db, port_str = host.split(':') - port = int(port_str) - print 'Connectiong to host %s on port %s' % (db, port) - self.databases.append(redis.Redis(host=db, port=port, db=0)) - print str(self.databases[c_num].ping()) - self.r_pipes.append(self.databases[c_num].pipeline(False)) - self.r_sizes.append(0) - self.w_pipes.append(self.databases[c_num].pipeline(True)) - self.w_sizes.append(0) - if (first) : - first = False - self.metadata = redis.Redis(host=db, port=port, db=0) - c_num += 1 - self.db_count += 1 - - # Reset Databases if required - if config['reset'] : - for db in self.databases : - db.flushall() - - # Process Debugging Levels - self.debug['load'] = config['debug-load'] - self.debug['delivery'] = config['debug-delivery'] - self.debug['new-order'] = config['debug-new-order'] - self.debug['order-status'] = config['debug-order-status'] - self.debug['payment'] = config['debug-payment'] - self.debug['stock-level'] = config['debug-stock-level'] - - if config['host-info'] != 'None' : - print 'TPC-C Benchmark Running on Redis with %s nodes' % (len(hosts)) - if config['host-info'] == 'Verbose' : - for host in hosts : - db, port = host.split(':') - print 'Host: %s | Port: %s' % (db, port) - # End loadConfig() - - #------------------------------------------------------------------------ - # Post-processing function for data loading - #------------------------------------------------------------------------ - def loadFinish(self) : - # Check to see if pipelines need to be flushed - for index in range(len(self.w_pipes)) : - if self.w_sizes[index] > 0 : - if self.debug['load'] != 'None' : - print index, - self.w_pipes[index].execute() - self.w_sizes[index] = 0 - - elapsed = time.time() - self.t0 - print '' - print 'Loading Complete: ' + str(elapsed) + ' elapsed' - - # Store Metadata - for table, next in self.next_scores.items() : - self.metadata.set(table + '.next_score', next) - # End loadFinish() - - #------------------------------------------------------------------------ - # Pre-pocessing function for data loading - #------------------------------------------------------------------------ - def loadStart(self) : - if self.debug['load'] != 'None': - print 'Starting data load' - self.t0 = time.time() - self.next_scores = { - 'WAREHOUSE' : 1, - 'DISTRICT' : 1, - 'ITEM' : 1, - 'CUSTOMER' : 1, - 'HISTORY' : 1, - 'STOCK' : 1, - 'ORDERS' : 1, - 'NEW_ORDER' : 1, - 'ORDER_LINE' : 1, - } - # End loadStart() - - #------------------------------------------------------------------------ - # Load tuples into a table for TPC-C benchmarking - # - # @param string table name - # @param list of tuples corresponding to table schema - #------------------------------------------------------------------------ - def loadTuples(self, tableName, tuples) : - - # Instantiate Column-mapping - column_map = self.TABLES[tableName]['map'] - if self.debug['load'] == 'Verbose' : - print tableName, - - for record in tuples : - # Determine at which node to store this data - node = 'ALL' - if tableName == 'WAREHOUSE' : - node = self.shard(record[column_map['W_ID']]) - key = self.safeKey([record[0]]) - self.w_pipes[node].sadd('WAREHOUSE.IDS', key) - self.w_pipes[node].hmset( - 'WAREHOUSE.' + key, - { - 'W_ID' : record[0], - 'W_NAME' : record[1], - 'W_STREET_1' : record[2], - 'W_STREET_2' : record[3], - 'W_CITY' : record[4], - 'W_STATE' : record[5], - 'W_ZIP' : record[6], - 'W_TAX' : record[7], - 'W_YTD' : record[8], - } - ) - self.w_sizes[node] += 2 - elif tableName == 'DISTRICT' : - node = self.shard(record[column_map['D_W_ID']]) - key = self.safeKey([record[1], record[0]]) - self.w_pipes[node].sadd('DISTRICT.IDS', key) - self.w_pipes[node].hmset( - 'DISTRICT.' + key, - { - 'D_ID' : record[0], - 'D_W_ID' : record[1], - 'D_NAME' : record[2], - 'D_STREET_1' : record[3], - 'D_STREET_2' : record[4], - 'D_CITY' : record[5], - 'D_STATE' : record[6], - 'D_ZIP' : record[7], - 'D_TAX' : record[8], - 'D_YTD' : record[9], - 'D_NEXT_O_ID' : record[10], - } - ) - self.w_sizes[node] += 2 - elif tableName == 'CUSTOMER' : - node = self.shard(record[column_map['C_W_ID']]) - key = self.safeKey([record[2], record[1], record[0]]) - self.w_pipes[node].sadd('CUSTOMER.IDS', key) - self.w_pipes[node].hmset( - 'CUSTOMER.' + key, - { - 'C_ID' : record[0], - 'C_D_ID' : record[1], - 'C_W_ID' : record[2], - 'C_FIRST' : record[3], - 'C_MIDDLE' : record[4], - 'C_LAST' : record[5], - 'C_STREET_1' : record[6], - 'C_STREET_2' : record[7], - 'C_CITY' : record[8], - 'C_ZIP' : record[9], - 'C_PHONE' : record[10], - 'C_SINCE' : record[11], - 'C_CREDIT' : record[12], - 'C_CREDIT_LIM' : record[13], - 'C_DISCOUNT' : record[14], - 'C_BALANCE' : record[15], - 'C_YTD_PAYMENT' : record[16], - 'C_PAYMENT_CNT' : record[17], - 'C_DELIVERY_CNT' : record[18], - 'C_DATA' : record[19], - } - ) - - # Add Special Index for Customer Table - index_key = self.safeKey([record[2], record[1], record[5]]) - self.w_pipes[node].sadd( - 'CUSTOMER.INDEXES.NAMESEARCH.' + index_key, - key - ) - self.w_sizes[node] += 3 - elif tableName == 'HISTORY' : - node = self.shard(record[column_map['H_W_ID']]) - key = self.safeKey([self.next_scores['HISTORY']]) - self.w_pipes[node].sadd('HISTORY.IDS', key) - self.w_pipes[node].hmset( - 'HISTORY.' + key, - { - 'H_C_ID' : record[0], - 'H_C_D_ID' : record[1], - 'H_C_W_ID' : record[2], - 'H_D_ID' : record[3], - 'H_W_ID' : record[4], - 'H_DATE' : record[5], - 'H_AMOUNT' : record[6], - 'H_DATA' : record[7], - } - ) - self.w_sizes[node] += 2 - elif tableName == 'STOCK' : - node = self.shard(record[column_map['S_W_ID']]) - key = self.safeKey([record[1], record[0]]) - self.w_pipes[node].sadd('STOCK.IDS', key) - self.w_pipes[node].hmset( - 'STOCK.' + key, - { - 'S_I_ID' : record[0], - 'S_W_ID' : record[1], - 'S_QUANTITY' : record[2], - 'S_DIST_01' : record[3], - 'S_DIST_02' : record[4], - 'S_DIST_03' : record[5], - 'S_DIST_04' : record[6], - 'S_DIST_05' : record[7], - 'S_DIST_06' : record[8], - 'S_DIST_07' : record[9], - 'S_DIST_08' : record[10], - 'S_DIST_09' : record[11], - 'S_DIST_10' : record[12], - 'S_YTD' : record[13], - 'S_ORDER_CNT' : record[14], - 'S_REMOTE_CNT' : record[15], - 'S_DATA' : record[16], - - } - ) - self.w_sizes[node] += 2 - elif tableName == 'ORDERS' : - node = self.shard(record[column_map['O_W_ID']]) - key = self.safeKey([ - record[0], - record[3], - record[1], - record[2] - ]) - self.w_pipes[node].sadd('ORDER.IDS', key) - self.w_pipes[node].hmset( - 'ORDER.' + key, - { - 'O_ID' : record[0], - 'O_D_ID' : record[1], - 'O_W_ID' : record[2], - 'O_C_ID' : record[3], - 'O_ENTRY_D' : record[4], - 'O_CARRIER_ID' : record[5], - 'O_OL_CNT' : record[6], - 'O_ALL_LOCAL' : record[7], - } - ) - - # Add Special Index for Order Table - index_key = self.safeKey([record[2], record[1], record[3]]) - self.w_pipes[node].sadd( - 'ORDERS.INDEXES.ORDERSEARCH.' + index_key, - key - ) - self.w_sizes[node] += 3 - elif tableName == 'NEW_ORDER' : - node = self.shard(record[column_map['NO_W_ID']]) - key = self.safeKey([record[1], record[2], record[0]]) - self.w_pipes[node].sadd('NEW_ORDER.IDS', key) - self.w_pipes[node].hmset( - 'NEW_ORDER.' + key, - { - 'NO_O_ID' : record[0], - 'NO_D_ID' : record[1], - 'NO_W_ID' : record[2], - } - ) - - # Add Special Index for New Order Table - index_key = self.safeKey([record[1], record[2]]) - self.w_pipes[node].sadd( - 'NEW_ORDER.INDEXES.GETNEWORDER.' + index_key, - key - ) - self.w_sizes[node] += 3 - elif tableName == 'ORDER_LINE' : - node = self.shard(record[column_map['OL_W_ID']]) - key = self.safeKey([ - record[2], - record[1], - record[0], - record[3] - ]) - self.w_pipes[node].sadd('ORDER_LINE.IDS', key) - self.w_pipes[node].hmset( - 'ORDER_LINE.' + key, - { - 'OL_O_ID' : record[0], - 'OL_D_ID' : record[1], - 'OL_W_ID' : record[2], - 'OL_NUMBER' : record[3], - 'OL_I_ID' : record[4], - 'OL_SUPPLY_W_ID' : record[5], - 'OL_DELIVERY_D' : record[6], - 'OL_QUANTITY' : record[7], - 'OL_AMOUNT' : record[8], - 'OL_DIST_INFO' : record[9], - } - ) - - # Add Special Index for Order Line Table - index_key = self.safeKey([record[0], record[1], record[2]]) - self.w_pipes[node].sadd( - 'ORDER_LINE.INDEXES.SUMOLAMOUNT.' + index_key, - key - ) - self.w_sizes[node] += 3 - elif tableName == 'ITEMS' : - key = self.safeKey([record[0]]); - pi = 0 - for pipe in self.w_pipes : - pipe.sadd('ITEM.IDS', key) - pipe.hmset( - 'ITEM.' + key, - { - 'I_ID' : record[0], - 'I_IM_ID' : record[1], - 'I_NAME' : record[2], - 'I_PRICE' : record[3], - 'I_DATA' : record[4], - } - ) - self.w_sizes[pi] += 2 - pass - self.next_scores[tableName] += 1 - - #print key, - # Check to see if pipelines need to be flushed - for index in range(len(self.w_pipes)) : - if self.w_sizes[index] > 10000 : - if self.debug['load'] != 'None' : - print index, - sys.stdout.flush() - self.w_pipes[index].execute() - self.w_sizes[index] = 0 - - if self.debug['load'] == 'Verbose' : - print '' - # End loadTuples() - - #------------------------------------------------------------------------ - # Return default configuration when none is specified via command line - # - # @return dictionary configuration parameters - #------------------------------------------------------------------------ - def makeDefaultConfig(self) : - return self.DEFAULT_CONFIG - # End makeDefaultConfig() - - #------------------------------------------------------------------------ - # Create a safe key for Redis by removing invalid characters from - # input list - # - # @param list keys - #------------------------------------------------------------------------ - def safeKey(self, keys) : - new_keys = [] - for k in keys : - new_keys.append(str(k)) - return self.KEY_SEPARATOR.join(new_keys - ).replace('\n', '').replace(' ','') - # End safeKey() - - #------------------------------------------------------------------------ - # Sharding Method for determine which not to access - # - # @param string w_id warehouse id - # @return int - #------------------------------------------------------------------------ - def shard(self, w_id) : - return int(w_id) % self.db_count - # End shard() \ No newline at end of file diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/scalarisdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/scalarisdriver.py deleted file mode 100644 index 662f978e..00000000 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/scalarisdriver.py +++ /dev/null @@ -1,1039 +0,0 @@ -''' -Created on May 2, 2011 - -@author: Irina Calciu and Alex Gillmor - -Scalaris Driver for CS227 TPCC Benchmark -''' - -from abstractdriver import * - -import os, logging, commands, constants - -from collections import defaultdict - -from api.Scalaris import JSONConnection, Transaction, TransactionSingleOp, NotFoundException - - -#Table Definitions -TABLE_COLUMNS = { - constants.TABLENAME_ITEM: [ - "I_ID", # INTEGER - "I_IM_ID", # INTEGER - "I_NAME", # VARCHAR - "I_PRICE", # FLOAT - "I_DATA", # VARCHAR - ], - constants.TABLENAME_WAREHOUSE: [ - "W_ID", # SMALLINT - "W_NAME", # VARCHAR - "W_STREET_1", # VARCHAR - "W_STREET_2", # VARCHAR - "W_CITY", # VARCHAR - "W_STATE", # VARCHAR - "W_ZIP", # VARCHAR - "W_TAX", # FLOAT - "W_YTD", # FLOAT - ], - constants.TABLENAME_DISTRICT: [ - "D_ID", # TINYINT - "D_W_ID", # SMALLINT - "D_NAME", # VARCHAR - "D_STREET_1", # VARCHAR - "D_STREET_2", # VARCHAR - "D_CITY", # VARCHAR - "D_STATE", # VARCHAR - "D_ZIP", # VARCHAR - "D_TAX", # FLOAT - "D_YTD", # FLOAT - "D_NEXT_O_ID", # INT - ], - constants.TABLENAME_CUSTOMER: [ - "C_ID", # INTEGER - "C_D_ID", # TINYINT - "C_W_ID", # SMALLINT - "C_FIRST", # VARCHAR - "C_MIDDLE", # VARCHAR - "C_LAST", # VARCHAR - "C_STREET_1", # VARCHAR - "C_STREET_2", # VARCHAR - "C_CITY", # VARCHAR - "C_STATE", # VARCHAR - "C_ZIP", # VARCHAR - "C_PHONE", # VARCHAR - "C_SINCE", # TIMESTAMP - "C_CREDIT", # VARCHAR - "C_CREDIT_LIM", # FLOAT - "C_DISCOUNT", # FLOAT - "C_BALANCE", # FLOAT - "C_YTD_PAYMENT", # FLOAT - "C_PAYMENT_CNT", # INTEGER - "C_DELIVERY_CNT", # INTEGER - "C_DATA", # VARCHAR - ], - constants.TABLENAME_STOCK: [ - "S_I_ID", # INTEGER - "S_W_ID", # SMALLINT - "S_QUANTITY", # INTEGER - "S_DIST_01", # VARCHAR - "S_DIST_02", # VARCHAR - "S_DIST_03", # VARCHAR - "S_DIST_04", # VARCHAR - "S_DIST_05", # VARCHAR - "S_DIST_06", # VARCHAR - "S_DIST_07", # VARCHAR - "S_DIST_08", # VARCHAR - "S_DIST_09", # VARCHAR - "S_DIST_10", # VARCHAR - "S_YTD", # INTEGER - "S_ORDER_CNT", # INTEGER - "S_REMOTE_CNT", # INTEGER - "S_DATA", # VARCHAR - ], - constants.TABLENAME_ORDERS: [ - "O_ID", # INTEGER - "O_D_ID", # TINYINT - "O_W_ID", # SMALLINT - "O_C_ID", # INTEGER - "O_ENTRY_D", # TIMESTAMP - "O_CARRIER_ID", # INTEGER - "O_OL_CNT", # INTEGER - "O_ALL_LOCAL", # INTEGER - ], - constants.TABLENAME_NEW_ORDER: [ - "NO_O_ID", # INTEGER - "NO_D_ID", # TINYINT - "NO_W_ID", # SMALLINT - ], - constants.TABLENAME_ORDER_LINE: [ - "OL_O_ID", # INTEGER - "OL_D_ID", # TINYINT - "OL_W_ID", # SMALLINT - "OL_NUMBER", # INTEGER - "OL_I_ID", # INTEGER - "OL_SUPPLY_W_ID", # SMALLINT - "OL_DELIVERY_D", # TIMESTAMP - "OL_QUANTITY", # INTEGER - "OL_AMOUNT", # FLOAT - "OL_DIST_INFO", # VARCHAR - ], - constants.TABLENAME_HISTORY: [ - "H_C_ID", # INTEGER - "H_C_D_ID", # TINYINT - "H_C_W_ID", # SMALLINT - "H_D_ID", # TINYINT - "H_W_ID", # SMALLINT - "H_DATE", # TIMESTAMP - "H_AMOUNT", # FLOAT - "H_DATA", # VARCHAR - ], -} -TABLE_INDEXES = { - constants.TABLENAME_ITEM: [ - "I_ID", - ], - constants.TABLENAME_WAREHOUSE: [ - "W_ID", - ], - constants.TABLENAME_DISTRICT: [ - "D_ID", - "D_W_ID", - ], - constants.TABLENAME_CUSTOMER: [ - "C_ID", - "C_D_ID", - "C_W_ID", - ], - constants.TABLENAME_STOCK: [ - "S_I_ID", - "S_W_ID", - ], - constants.TABLENAME_ORDERS: [ - "O_ID", - "O_D_ID", - "O_W_ID", - "O_C_ID", - ], - constants.TABLENAME_NEW_ORDER: [ - "NO_O_ID", - "NO_D_ID", - "NO_W_ID", - ], - constants.TABLENAME_ORDER_LINE: [ - "OL_O_ID", - "OL_D_ID", - "OL_W_ID", - ], -} - -def createPrimaryKey(tableName, id, obj): - ''' - Helper method to create normalized primary keys - ''' - if tableName == constants.TABLENAME_ITEM: - return '%s.%s' % (constants.TABLENAME_ITEM, id) - elif tableName == constants.TABLENAME_WAREHOUSE: - return '%s.%s' %(constants.TABLENAME_WAREHOUSE, id) - elif tableName == constants.TABLENAME_DISTRICT: - return '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, obj['D_W_ID'], constants.TABLENAME_DISTRICT, id) - elif tableName == constants.TABLENAME_CUSTOMER: - return '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, obj['C_W_ID'], \ - constants.TABLENAME_DISTRICT, obj['C_D_ID'], constants.TABLENAME_CUSTOMER, id) - elif tableName == constants.TABLENAME_ORDERS: - return '%s.%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, obj['O_W_ID'], \ - constants.TABLENAME_DISTRICT, obj['O_D_ID'], constants.TABLENAME_CUSTOMER, obj['O_C_ID'], \ - constants.TABLENAME_ORDERS, id) - else: - return '%s.%s' % (tableName,id) - - - - - -class ScalarisDriver(AbstractDriver): - ''' - Scalaris Driver for CS227 TPCC benchmark - ''' - - DEFAULT_CONFIG = { - "database": ("The path to the main Scalaris Node", "http://localhost:8000" ), - } - - def __init__(self, ddl): - ''' - init - ''' - super(ScalarisDriver, self).__init__("scalaris", ddl) - - ## ---------------------------------------------- - ## makeDefaultConfig - ## ---------------------------------------------- - def makeDefaultConfig(self): - return ScalarisDriver.DEFAULT_CONFIG - - ## ---------------------------------------------- - ## loadConfig - ## ---------------------------------------------- - def loadConfig(self, config): - for key in ScalarisDriver.DEFAULT_CONFIG.keys(): - assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) - - self.database = str(config["database"]) - - - self.conn = JSONConnection(url=self.database) - - #self.tran = self.transactionFactory() - #single op is much faster for nearly all operations - self.tran = TransactionSingleOp(self.conn) - - def transactionFactory(self): - ''' - Transaction object factory method - ''' - return Transaction(self.conn) - - def loadTuples(self, tableName, tuples): - s = set([constants.TABLENAME_HISTORY, \ - constants.TABLENAME_NEW_ORDER, \ - constants.TABLENAME_ORDER_LINE,\ - constants.TABLENAME_STOCK]) - - if tableName in s: - self.loadComplexTuples(tableName,tuples) - else: - self.loadSimpleTuples(tableName, tuples) - - if tableName == constants.TABLENAME_ORDERS: - self.loadOrderCustomer(tableName, tuples) - if tableName == constants.TABLENAME_CUSTOMER: - self.loadWarehouseDistrictCustomers(tableName, tuples) - - def loadHistory(self,tuples): - ''' - Specialized method for history table. History is stored based on customer info. - ''' - history_d=defaultdict(lambda : defaultdict(lambda : defaultdict(list))) - tableDef = TABLE_COLUMNS[constants.TABLENAME_HISTORY] - for tuple in tuples: - history = dict(zip(tableDef,tuple)) - w_id = history["H_C_W_ID"] - d_id = history['H_C_D_ID'] - c_id = history["H_C_ID"] - - history_d[w_id][d_id][c_id].append(history) - - for w in history_d.keys(): - for d in history_d[w].keys(): - for o in history_d[w][d].keys(): - history_key = '%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ - constants.TABLENAME_DISTRICT, d, \ - constants.TABLENAME_CUSTOMER, o,\ - constants.TABLENAME_HISTORY) - self.tran.write(history_key, history_d[w][d][o]) - - def loadStock(self,tuples): - ''' - Specialized method for laoding stock - ''' - - #need to reestablish because some large values cause problems with scalaris ws - self.conn = JSONConnection(url=self.database) - self.tran = TransactionSingleOp(self.conn) - - stock_d = defaultdict(defaultdict) - tableDef = TABLE_COLUMNS[constants.TABLENAME_STOCK] - stock_idx = defaultdict(list) - - for tuple in tuples: - stock = dict(zip(tableDef,tuple)) - tuple_short = [tuple[0], tuple[2]] - stock_short = dict(zip(['S_I_ID', 'S_QUANTITY'],tuple_short)) - s_w_id = stock['S_W_ID'] - s_i_id = stock['S_I_ID'] - stock_d[s_w_id][s_i_id] = stock - stock_idx[s_w_id].append(stock_short) - - for s in stock_d.keys(): - s_key = '%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, s, constants.TABLENAME_STOCK) - print "key %s" % s_key - print "value %s" % stock_idx[s] - self.tran.write(s_key, stock_idx[s]) - - for w in stock_d.keys(): - for i in stock_d[w].keys(): - s_key = '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w, constants.TABLENAME_STOCK, i) - self.tran.write(s_key, stock_d[s][i]) - - - - def loadOrderLine(self,tuples): - ''' - Load order line objects: - WAREHOUSE.W_ID.DISTRICT.D_ID.ORDERLINE.O_ID -> OrderLine Object - WAREHOUSE.W_ID.DISTRICT.D_ID.ORDERLINE -> List of Order Ids for that District - ''' - ol_d=defaultdict(lambda : defaultdict(lambda : defaultdict(list))) - order_ids = defaultdict(lambda : defaultdict(list)) - tableDef = TABLE_COLUMNS[constants.TABLENAME_ORDER_LINE] - for tuple in tuples: - no = dict(zip(tableDef,tuple)) - w_id = no["OL_W_ID"] - d_id = no['OL_D_ID'] - o_id = no["OL_O_ID"] - - ol_d[w_id][d_id][o_id].append(no) - order_ids[w_id][d_id].append(str(o_id)) - - for w in ol_d.keys(): - for d in ol_d[w].keys(): - for o in ol_d[w][d].keys(): - ol_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ - constants.TABLENAME_DISTRICT, d, \ - constants.TABLENAME_ORDER_LINE, o) - self.tran.write(ol_key, ol_d[w][d][o]) - for w in order_ids.keys(): - for d in order_ids[w].keys(): - no_key = '%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ - constants.TABLENAME_DISTRICT, d, \ - constants.TABLENAME_ORDER_LINE) - self.tran.write(no_key, order_ids[w][d]) - - def loadNewOrder(self,tuples): - ''' - Load new order objects: - WAREHOUSE.W_ID.DISTRICT.D_ID.NEW_ORDER.O_ID -> New Order Object - WAREHOUSE.W_ID.DISTRICT.D_ID.NEW_ORDER -> List of Order Ids for that new_order - ''' - no_d=defaultdict(lambda : defaultdict(lambda : defaultdict(list))) - order_ids = defaultdict(lambda : defaultdict(list)) - tableDef = TABLE_COLUMNS[constants.TABLENAME_NEW_ORDER] - for tuple in tuples: - no = dict(zip(tableDef,tuple)) - w_id = no["NO_W_ID"] - d_id = no['NO_D_ID'] - o_id = no["NO_O_ID"] - - no_d[w_id][d_id][o_id].append(no) - order_ids[w_id][d_id].append(str(o_id)) - - for w in no_d.keys(): - for d in no_d[w].keys(): - for o in no_d[w][d].keys(): - no_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ - constants.TABLENAME_DISTRICT, d, \ - constants.TABLENAME_NEW_ORDER, o) - self.tran.write(no_key, no_d[w][d][o]) - - for w in order_ids.keys(): - for d in order_ids[w].keys(): - no_key = '%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ - constants.TABLENAME_DISTRICT, d, \ - constants.TABLENAME_NEW_ORDER) - self.tran.write(no_key, order_ids[w][d]) - - def loadComplexTuples(self, tableName, tuples): - ''' - Dispatching method for tuples that need secondary/advanced indexing - ''' - if tableName == constants.TABLENAME_ORDER_LINE: - self.loadOrderLine(tuples) - if tableName == constants.TABLENAME_NEW_ORDER: - self.loadNewOrder(tuples) - if tableName == constants.TABLENAME_HISTORY: - self.loadHistory(tuples) - if tableName == constants.TABLENAME_STOCK: - self.loadStock(tuples) - #self.tran.commit() - - - def loadOrderCustomer(self, tableName, tuples): - ''' - Load order objects: - WAREHOUSE.W_ID.DISTRICT.D_ID.ORDERS.O_ID -> Order Object - WAREHOUSE.W_ID.DISTRICT.D_ID.ORDERS-> List of Order Ids for that district - ''' - - #order case - tableDef = TABLE_COLUMNS[tableName] - o_d=defaultdict(lambda : defaultdict(lambda : defaultdict(list))) - - - for tuple in tuples: - value = dict(zip(tableDef,tuple)) - o_id = value['O_ID'] - c_id = value['O_C_ID'] - d_id = value['O_D_ID'] - w_id = value['O_W_ID'] - - oc_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE,w_id, constants.TABLENAME_DISTRICT,d_id, constants.TABLENAME_ORDERS,o_id) - self.tran.write(oc_key, c_id) - - o_d[w_id][d_id][c_id].append(str(o_id)) - - for k1 in o_d.keys(): - for k2 in o_d[k1].keys(): - for k3 in o_d[k1][k2].keys(): - orders_key = '%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, k1, constants.TABLENAME_DISTRICT, k2, \ - constants.TABLENAME_CUSTOMER, k3, constants.TABLENAME_ORDERS) - self.tran.write(orders_key,o_d[k1][k2][k3]) - - - def loadWarehouseDistrictCustomers(self, tableName, tuples): - ''' - Load warehouse district customers: - WAREHOUSE.W_ID.DISTRICT.CUSTOMERS -> List of Customer IDs - ''' - - #customers - custs = defaultdict(lambda : defaultdict(list)) - tableDef = TABLE_COLUMNS[tableName] - for tuple in tuples: - value = dict(zip(tableDef,tuple)) - - c_last = value['C_LAST'] - c_id = value['C_ID'] - c_idx = {} - c_idx['C_LAST'] = c_last - c_idx['C_ID'] = c_id - d_id = value['C_D_ID'] - w_id = value['C_W_ID'] - custs[w_id][d_id].append(c_idx) - - for w in custs.keys(): - for d in custs[w].keys(): - custs_key = '%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w,\ - constants.TABLENAME_DISTRICT, d, \ - 'CUSTOMERS') - self.tran.write(custs_key, custs[w][d]) - - - - - def loadSimpleTuples(self, tableName, tuples): - """Load a list of tuples into the target table""" - - self.conn = JSONConnection(url=self.database) - self.tran = TransactionSingleOp(self.conn) - - if len(tuples) == 0: return - - idx =0 - - tableDef = TABLE_COLUMNS[tableName] - - for tuple in tuples: - pId= tuple[0] - value = dict(zip(tableDef[1:],tuple[1:])) - primaryKey = createPrimaryKey(tableName, pId, value) - self.tran.write(primaryKey, value) - #self.tran.commit() - if idx == 500: - print '%s %s' % (tableName, primaryKey) - idx = 0 -# self.tran.commit() - idx+=1 - - #self.tran.commit() - - def loadFinish(self): - logging.info("Commiting changes to database") - #self.tran.commit() - - def executeStart(self): - self.conn = JSONConnection(url=self.database) - - #self.tran = self.transactionFactory() - self.tran = TransactionSingleOp(self.conn) - def executeFinish(self): - """Callback after the execution phase finishes""" -# self.tran.commit() - self.tran.closeConnection() - #self.tran.commit() - - ## ---------------------------------------------- - ## doDelivery - ## ---------------------------------------------- - def doDelivery(self, params): - w_id = params["w_id"] - o_carrier_id = params["o_carrier_id"] - ol_delivery_d = params["ol_delivery_d"] - - result = [ ] - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): - ## getNewOrder - ## WAREHOUSE.W_ID.DISTRICT.D_ID.NEW_ORDERS -> List of New Orders - no_key = '%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_NEW_ORDER) - - newOrders = None - newOrders = self.tran.read(no_key) #we expect a list of new order ifd - if newOrders == None: - ## No orders for this district: skip it. Note: This must be reported if > 1% - continue - assert len(newOrders) > 0 - - #just ids - no_o_id = min(newOrders) - newOrders.remove(no_o_id) - no_o_id = int(no_o_id) - #new order objects - - ## getCId - ## WAREHOUSE.W_ID.DISTRICT.D_ID.ORDER.ORDER_ID = List of Customers - customer = None - c_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ - constants.TABLENAME_DISTRICT, d_id, \ - constants.TABLENAME_ORDERS, no_o_id) - - customer = self.tran.read(c_key) - assert customer != None - c_id = customer - - ## sumOLAmount - ## WAREHOUSE.W_ID.DISTRICT.D_ID.ORDER_LINE.ORDER_ID -> List of OrderLine Objects or list of OL_NUMBERS - orderLines = [] - ol_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ - constants.TABLENAME_DISTRICT, d_id, \ - constants.TABLENAME_ORDER_LINE, no_o_id) - - orderLines = self.tran.read(ol_key) - - ol_total = 0.0 - for ol in orderLines: - ol_total += ol["OL_AMOUNT"] - ol['OL_DELIVERY_D'] = ol_delivery_d - ## FOR - - ## deleteNewOrder - #self.new_order.remove({"NO_D_ID": d_id, "NO_W_ID": w_id, "NO_O_ID": no_o_id}) - no_del_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_NEW_ORDER, no_o_id) - self.tran.write(no_del_key, None) - self.tran.write(no_key,newOrders) - - ## updateOrders - order_key = '%s.%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_CUSTOMER, c_id, constants.TABLENAME_ORDERS, no_o_id) - order = self.tran.read(order_key) - order['O_CARRIER_ID'] = o_carrier_id - self.tran.write(order_key, order) - #self.orders.update({"O_ID": no_o_id, "O_D_ID": d_id, "O_W_ID": w_id}, {"$set": {"O_CARRIER_ID": o_carrier_id}}, multi=False) - - ## updateOrderLine - #self.order_line.update({"OL_O_ID": no_o_id, "OL_D_ID": d_id, "OL_W_ID": w_id}, {"$set": {"OL_DELIVERY_D": ol_delivery_d}}, multi=False) - self.tran.write(ol_key, orderLines) - - - # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) - # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure - # them out - # If there are no order lines, SUM returns null. There should always be order lines. - assert ol_total != None, "ol_total is NULL: there are no order lines. This should not happen" - assert ol_total > 0.0 - - ## updateCustomer - customer_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ - constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_CUSTOMER, c_id) - - customer = self.tran.read(customer_key) - customer["C_BALANCE"]=customer["C_BALANCE"]+ol_total - - #self.customer.update({"C_ID": c_id, "C_D_ID": d_id, "C_W_ID": w_id}, {"$inc": {"C_BALANCE": ol_total}}) - - result.append((d_id, no_o_id)) - ## FOR - return result - - ## ---------------------------------------------- - ## doNewOrder - ## ---------------------------------------------- - def doNewOrder(self, params): - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - o_entry_d = params["o_entry_d"] - i_ids = params["i_ids"] - i_w_ids = params["i_w_ids"] - i_qtys = params["i_qtys"] - s_dist_col = "S_DIST_%02d" % d_id - - assert len(i_ids) > 0 - assert len(i_ids) == len(i_w_ids) - assert len(i_ids) == len(i_qtys) - - ## http://stackoverflow.com/q/3844931/ - all_local = (not i_w_ids or [w_id] * len(i_w_ids) == i_w_ids) - - ## GET ALL ITEMS WITH - ## ITEM.I_ID - items = [] - for id in i_ids: - i_key = '%s.%s' % (constants.TABLENAME_ITEM, id) - item = self.tran.read(i_key) - if item: - items.append(item) - - ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. - ## Note that this will happen with 1% of transactions on purpose. - if len(items) != len(i_ids): - ## TODO Abort here! - return - ## IF - - ## ---------------- - ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER - ## ---------------- - - # getWarehouseTaxRate - ## WAREHOUSE.W_ID -> warehouse object - w_key = '%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id) - w = self.tran.read(w_key) - assert w - w_tax = w["W_TAX"] - - # getDistrict - ## WAREHOUSE.W_ID.DISTRICT.D_ID -> district object - - d_key = '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id) - d = self.tran.read(d_key) - assert d - d_tax = d["D_TAX"] - d_next_o_id = d["D_NEXT_O_ID"] - - - # incrementNextOrderId - d["D_NEXT_O_ID"] = d["D_NEXT_O_ID"] + 1 - self.tran.write(d_key, d); - - - # getCustomer - ## WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMER.C_ID -> Customer Object - c_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ - constants.TABLENAME_CUSTOMER, c_id) - c = self.tran.read(c_key) - assert c - c_discount = c["C_DISCOUNT"] - - ## ---------------- - ## Insert Order Information - ## ---------------- - ol_cnt = len(i_ids) - o_carrier_id = constants.NULL_CARRIER_ID - - # createOrder - ## write to order object to WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMER.C_ID.ORDER.O_ID - o_key = '%s.%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ - constants.TABLENAME_CUSTOMER, c_id, constants.TABLENAME_ORDERS, d_next_o_id) - - order = {#"O_ID": d_next_o_id, - "O_D_ID": d_id, - "O_W_ID": w_id, - "O_C_ID": c_id, - "O_ENTRY_D": o_entry_d, - "O_CARRIER_ID": o_carrier_id, - "O_OL_CNT": ol_cnt, - "O_ALL_LOCAL": all_local} - - self.tran.write(o_key, order) - - - # createNewOrder - - ## write new order object to WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMER.C_ID.ORDER.O_ID.NEW_ORDER.NO_ID - ## newOrder - ## assumption - ## WAREHOUSE.W_ID.DISTRICT.D_ID.NEW_ORDER.ORDER_ID -> List of New Order Objects - new_order = {"NO_O_ID": d_next_o_id, "NO_D_ID": d_id, "NO_W_ID": w_id} - no_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_NEW_ORDER, d_next_o_id) - - try: - new_orders = self.tran.read(no_key) - except NotFoundException: - new_orders = [] - - new_orders.append(new_order) - - self.tran.write(no_key, new_orders) - - #self.new_order.insert({"NO_O_ID": d_next_o_id, "NO_D_ID": d_id, "NO_W_ID": w_id}) - - ## ---------------- - ## OPTIMIZATION: - ## If all of the items are at the same warehouse, then we'll issue a single - ## request to get their information - ## ---------------- - stockInfos = None - if all_local and False: - # getStockInfo - ##WAREHOUSE.W_ID.STOCK -returns-> List of Stock Objects w/ S_I_ID and S_QUANTITY - allStocks_key = '%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_STOCK) - allStocks = self.tran.read(allStocks_key) - - assert len(allStocks) == ol_cnt - stockInfos = { } - for si in allStocks: - stockInfos["S_I_ID"] = si # HACK - ## IF - - ## ---------------- - ## Insert Order Item Information - ## ---------------- - item_data = [ ] - total = 0 - for i in range(ol_cnt): - ol_number = i + 1 - ol_supply_w_id = i_w_ids[i] - ol_i_id = i_ids[i] - ol_quantity = i_qtys[i] - - itemInfo = items[i] - i_name = itemInfo["I_NAME"] - i_data = itemInfo["I_DATA"] - i_price = itemInfo["I_PRICE"] - - pformat = None - # getStockInfo - if all_local and stockInfos != None: - si = stockInfos[ol_i_id] - assert si["S_I_ID"] == ol_i_id, "S_I_ID should be %d\n%s" % (ol_i_id, pformat(si)) - else: - ## WAREHOUSE.W_ID.STOCK.S_ID - - allStocks_key = '%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_STOCK) - allStocks = self.tran.read(allStocks_key) - - for stock in allStocks: - if stock['S_I_ID'] == ol_i_id: - si = stock - break - - assert si, "Failed to find S_I_ID: %d\n%s" % (ol_i_id, pformat(itemInfo)) - - si_key = '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_STOCK, si['S_I_ID']) - stock= self.tran.read(si_key) - - - s_quantity = si["S_QUANTITY"] - s_ytd = stock["S_YTD"] - s_order_cnt = stock["S_ORDER_CNT"] - s_remote_cnt = stock["S_REMOTE_CNT"] - s_data = stock["S_DATA"] - s_dist_xx = stock[s_dist_col] # Fetches data from the s_dist_[d_id] column - - ## Update stock - s_ytd += ol_quantity - if s_quantity >= ol_quantity + 10: - s_quantity = s_quantity - ol_quantity - else: - s_quantity = s_quantity + 91 - ol_quantity - s_order_cnt += 1 - - if ol_supply_w_id != w_id: s_remote_cnt += 1 - - # updateStock - si["S_QUANTITY"] = s_quantity - stock["S_QUANTITY"] = s_quantity - stock["S_YTD"] = s_ytd - stock["S_ORDER_CNT"] = s_order_cnt - stock["S_REMOTE_CNT"] = s_remote_cnt - ## so this should be cools - self.tran.write(allStocks_key, allStocks) - self.tran.write(si_key, stock) - - - if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: - brand_generic = 'B' - else: - brand_generic = 'G' - ## Transaction profile states to use "ol_quantity * i_price" - ol_amount = ol_quantity * i_price - total += ol_amount - - order_line = {"OL_O_ID": d_next_o_id, - "OL_D_ID": d_id, - "OL_W_ID": w_id, - "OL_NUMBER": ol_number, - "OL_I_ID": ol_i_id, - "OL_SUPPLY_W_ID": ol_supply_w_id, - "OL_DELIVERY_D": o_entry_d, - "OL_QUANTITY": ol_quantity, - "OL_AMOUNT": ol_amount, - "OL_DIST_INFO": s_dist_xx} - - ##WAREHOUSE.W_ID.DISTRICT.D_ID.ORDER_LINE.ORDER_ID -> List of OrderLine Objects - ol_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ - constants.TABLENAME_ORDER_LINE, d_next_o_id) - - try: - order_lines = self.tran.read(ol_key) - except NotFoundException: - order_lines = [] - - order_lines.append(order_line) - self.tran.write(ol_key, order_lines) - - # createOrderLine - ## Add the info to be returned - item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) - ## FOR - - ## Adjust the total for the discount - #print "c_discount:", c_discount, type(c_discount) - #print "w_tax:", w_tax, type(w_tax) - #print "d_tax:", d_tax, type(d_tax) - total *= (1 - c_discount) * (1 + w_tax + d_tax) - - ## Pack up values the client is missing (see TPC-C 2.4.3.5) - misc = [ (w_tax, d_tax, d_next_o_id, total) ] - - return [ c, misc, item_data ] - - ## ---------------------------------------------- - ## doOrderStatus - ## ---------------------------------------------- - def doOrderStatus(self, params): - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - - - if c_id != None: - # getCustomerByCustomerId - cust_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ - constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_CUSTOMER, c_id) - c = self.tran.read(cust_key) - else: - # getCustomersByLastName - # Get the midpoint customer's id - #WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMERS -> List of Customers (or C_ID:C_LAST pairs) - all_customers_key = '%s.%s.%s.%s.CUSTOMERS' % (constants.TABLENAME_WAREHOUSE, w_id, \ - constants.TABLENAME_DISTRICT, d_id) - - all_customers = self.tran.read(all_customers_key) - all_customers = [customer for customer in all_customers if customer['C_LAST']==c_last] - - namecnt = len(all_customers) - assert namecnt > 0 - index = (namecnt-1)/2 - c = all_customers[index] - c_id = c["C_ID"] - cust_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ - constants.TABLENAME_DISTRICT, d_id, constants.TABLENAME_CUSTOMER, c_id) - c = self.tran.read(cust_key) - - # getLastOrder - ## WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMER.C_ID.ORDERS - > List of Orders - orders_key = '%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ - constants.TABLENAME_CUSTOMER, c_id, constants.TABLENAME_ORDERS) - orders = self.tran.read(orders_key) - - o_id = max(orders) - - order_key = '%s.%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ - constants.TABLENAME_CUSTOMER, c_id, constants.TABLENAME_ORDERS, o_id) - - - order = self.tran.read(order_key) - - if order: - # getOrderLines - - ## WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMER.ORDER_LINE.O_ID -> List of Orderline Objects - ol_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id, \ - constants.TABLENAME_ORDER_LINE, o_id) - orderLines = self.tran.read(ol_key) - else: - orderLines = [ ] - - return [ c, order, orderLines ] - - - ## ---------------------------------------------- - ## doPayment - ## ---------------------------------------------- - def doPayment(self, params): - - w_id = params["w_id"] - d_id = params["d_id"] - h_amount = params["h_amount"] - c_w_id = params["c_w_id"] - c_d_id = params["c_d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - h_date = params["h_date"] - - if c_id != None: - # getCustomerByCustomerId - cust_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, c_w_id, \ - constants.TABLENAME_DISTRICT, c_d_id, constants.TABLENAME_CUSTOMER, c_id) - c = self.tran.read(cust_key) - else: - # getCustomersByLastName - # Get the midpoint customer's id - #WAREHOUSE.W_ID.DISTRICT.D_ID.CUSTOMERS -> List of Customers (or C_ID:C_LAST pairs) - all_customers_key = '%s.%s.%s.%s.CUSTOMERS' % (constants.TABLENAME_WAREHOUSE, c_w_id, \ - constants.TABLENAME_DISTRICT, c_d_id) - - all_customers = self.tran.read(all_customers_key) - all_customers = [customer for customer in all_customers if customer['C_LAST']==c_last] - - namecnt = len(all_customers) - assert namecnt > 0 - index = (namecnt-1)/2 - c = all_customers[index] - c_id = c["C_ID"] - cust_key = '%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, c_w_id, \ - constants.TABLENAME_DISTRICT, c_d_id, constants.TABLENAME_CUSTOMER, c_id) - c = self.tran.read(cust_key) - - assert len(c) > 0 - assert c_id != None - c["C_BALANCE"] = c["C_BALANCE"] - h_amount - c["C_YTD_PAYMENT"] = c["C_YTD_PAYMENT"] + h_amount - c["C_PAYMENT_CNT"] = c["C_PAYMENT_CNT"] + 1 - c_data = c["C_DATA"] - - # getWarehouse - - ## SCALARIS - ## WAREHOUSE.W_ID -> Warehouse Object - w_key = '%s.%s' % (constants.TABLENAME_WAREHOUSE, c_w_id) - w = self.tran.read(w_key) - assert w - - # getDistrict - ## SCALARIS - ## WAREHOUSE.W_ID.DISTRICT.D_ID - > District Object - d_key = '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ - constants.TABLENAME_DISTRICT, d_id) - d = self.tran.read(d_key) - assert d - - # updateWarehouseBalance - w['W_YTD'] = w['W_YTD'] + h_amount - self.tran.write(w_key, w) - - # updateDistrictBalance - d['D_YTD'] = d['D_YTD'] + h_amount - self.tran.write(d_key, d) - - # Customer Credit Information - if c["C_CREDIT"] == constants.BAD_CREDIT: - newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) - c_data = (newData + "|" + c_data) - if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] - - c['C_DATA']=c_data - - - self.tran.write(cust_key, c) - # Concatenate w_name, four spaces, d_name - h_data = "%s %s" % (w["W_NAME"], d["D_NAME"]) - - # insertHistory - history = {#"H_C_ID": c_id, - #"H_C_D_ID": c_d_id, - #"H_C_W_ID": c_w_id, - "H_D_ID": d_id, - "H_W_ID": w_id, - "H_DATE": h_date, - "H_AMOUNT": h_amount, - "H_DATA": h_data} - - h_key = '%s.%s.%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, c_w_id, \ - constants.TABLENAME_DISTRICT, c_d_id, \ - constants.TABLENAME_CUSTOMER, c_id, - constants.TABLENAME_HISTORY) - histories = self.tran.read(h_key) - if histories == None: - histories = [] - histories.append(history) - self.tran.write(h_key, histories) - - # TPC-C 2.5.3.3: Must display the following fields: - # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, - # D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, - # C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, - # C_DISCOUNT, C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = "BC"), - # H_AMOUNT, and H_DATE. - - # Hand back all the warehouse, district, and customer data - return [ w, d, c ] - - ## ---------------------------------------------- - ## doStockLevel - ## ---------------------------------------------- - def doStockLevel(self, params): - w_id = params["w_id"] - d_id = params["d_id"] - threshold = params["threshold"] - - # getOId - ## WAREHOUSE.W_ID.DISTRICT.D_ID -returns-> District Object w/ attribute: D_NEXT_O_ID - ## - d_key = '%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, constants.TABLENAME_DISTRICT, d_id) - d = self.tran.read(d_key) - assert d - o_id = d["D_NEXT_O_ID"] - - ## WAREHOUSE.W_ID.DISTRICT.D_ID.ORDER_LINE.ORDER_ID -returns-> List of Order Line Objects w/ OL_I_ID - ## - ol_key = '%s.%s.%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ - constants.TABLENAME_DISTRICT, d_id, \ - constants.TABLENAME_ORDER_LINE) - - orderLines = self.tran.read(ol_key) - assert orderLines - ol_ids = set() - for ol in orderLines: - if ol < o_id and ol >= o_id-20: - ol_ids.add(ol) - - - ## WAREHOUSE.W_ID.STOCK -returns-> List of Stock Objects w/ S_I_ID and S_QUANTITY - s_key = '%s.%s.%s' % (constants.TABLENAME_WAREHOUSE, w_id, \ - constants.TABLENAME_STOCK) - stocks = self.tran.read(s_key) - result = len([stock for stock in stocks if stock['S_I_ID'] in ol_ids and stock['S_QUANTITY'] < threshold]) - - return int(result) - diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/tokyocabinetdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/tokyocabinetdriver.py deleted file mode 100644 index a561dbf7..00000000 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/tokyocabinetdriver.py +++ /dev/null @@ -1,1061 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------- -# Copyright (C) 2011 -# Marcelo Martins -# http://www.cs.brown.edu/~martins/ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# ----------------------------------------------------------------------- - -from __future__ import with_statement -from abstractdriver import * -from pprint import pprint, pformat -from pyrant import protocol - -import constants -import logging -import os -import pyrant -import sys - -TABLE_COLUMNS = { - constants.TABLENAME_ITEM: [ - "I_ID", # INTEGER - "I_IM_ID", # INTEGER - "I_NAME", # VARCHAR - "I_PRICE", # FLOAT - "I_DATA", # VARCHAR - ], - constants.TABLENAME_WAREHOUSE: [ - "W_ID", # SMALLINT - "W_NAME", # VARCHAR - "W_STREET_1", # VARCHAR - "W_STREEET_2", # VARCHAR - "W_CITY", # VARCHAR - "W_STATE", # VARCHAR - "W_ZIP", # VARCHAR - "W_TAX", # FLOAT - "W_YTD", # FLOAT - ], - constants.TABLENAME_DISTRICT: [ - "D_ID", # TINYINT - "D_W_ID", # SMALLINT - "D_NAME", # VARCHAR - "D_STREET_1", # VARCHAR - "D_STREET_2", # VARCHAR - "D_CITY", # VARCHAR - "D_STATE", # VARCHAR - "D_ZIP", # VARCHAR - "D_TAX", # FLOAT - "D_YTD", # FLOAT - "D_NEXT_O_ID", # INT - ], - constants.TABLENAME_CUSTOMER: [ - "C_ID", # INTEGER - "C_D_ID", # TINYINT - "C_W_ID", # SMALLINT - "C_FIRST", # VARCHAR - "C_MIDDLE", # VARCHAR - "C_LAST", # VARCHAR - "C_STREET_1", # VARCHAR - "C_STREET_2", # VARCHAR - "C_CITY", # VARCHAR - "C_STATE", # VARCHAR - "C_ZIP", # VARCHAR - "C_PHONE", # VARCHAR - "C_SINCE", # TIMESTAMP - "C_CREDIT", # VARCHAR - "C_CREDIT_LIM", # FLOAT - "C_DISCOUNT", # FLOAT - "C_BALANCE", # FLOAT - "C_YTD_PAYMENT", # FLOAT - "C_PAYMENT_CNT", # INTEGER - "C_DELIVERY_CNT", # INTEGER - "C_DATA", # VARCHAR - ], - constants.TABLENAME_STOCK: [ - "S_I_ID", # INTEGER - "S_W_ID", # SMALLINT - "S_QUANTITY", # INTEGER - "S_DIST_01", # VARCHAR - "S_DIST_02", # VARCHAR - "S_DIST_03", # VARCHAR - "S_DIST_04", # VARCHAR - "S_DIST_05", # VARCHAR - "S_DIST_06", # VARCHAR - "S_DIST_07", # VARCHAR - "S_DIST_08", # VARCHAR - "S_DIST_09", # VARCHAR - "S_DIST_10", # VARCHAR - "S_YTD", # INTEGER - "S_ORDER_CNT", # INTEGER - "S_REMOTE_CNT", # INTEGER - "S_DATA", # VARCHAR - ], - constants.TABLENAME_ORDERS: [ - "O_ID", # INTEGER - "O_C_ID", # INTEGER - "O_D_ID", # TINYINT - "O_W_ID", # SMALLINT - "O_ENTRY_ID", # TIMESTAMP - "O_CARRIER_ID", # INTEGER - "O_OL_CNT", # INTEGER - "O_ALL_LOCAL", # INTEGER - ], - constants.TABLENAME_NEW_ORDER: [ - "NO_O_ID", # INTEGER - "NO_D_ID", # TINYINT - "NO_W_ID", # SMALLINT - ], - constants.TABLENAME_ORDER_LINE: [ - "OL_O_ID", # INTEGER - "OL_D_ID", # TINYINT - "OL_W_ID", # SMALLINT - "OL_NUMBER", # INTEGER - "OL_I_ID", # INTEGER - "OL_SUPPLY_W_ID", # SMALLINT - "OL_DELIVERY_D", # TIMESTAMP - "OL_QUANTITY", # INTEGER - "OL_AMOUNT", # FLOAT - "OL_DIST_INFO", # VARCHAR - ], - constants.TABLENAME_HISTORY: [ - "H_C_ID", # INTEGER - "H_C_D_ID", # TINYINT - "H_C_W_ID", # SMALLINT - "H_D_ID", # TINYINT - "H_W_ID", # SMALLINT - "H_DATA", # TIMESTAMP - "H_AMOUNT", # FLOAT - "H_DATA", # VARCHAR - ], -} -TABLE_INDEXES = { - constants.TABLENAME_ITEM: [ - "I_ID", - ], - constants.TABLENAME_WAREHOUSE: [ - "W_ID", - ], - constants.TABLENAME_DISTRICT: [ - "D_ID", - "D_W_ID", - ], - constants.TABLENAME_CUSTOMER: [ - "C_ID", - "C_D_ID", - "C_W_ID", - ], - constants.TABLENAME_STOCK: [ - "S_I_ID", - "S_W_ID", - ], - constants.TABLENAME_ORDERS: [ - "O_ID", - "O_D_ID", - "O_W_ID", - "O_C_ID", - ], - constants.TABLENAME_NEW_ORDER: [ - "NO_O_ID", - "NO_D_ID", - "NO_W_ID", - ], - constants.TABLENAME_ORDER_LINE: [ - "OL_O_ID", - "OL_D_ID", - "OL_W_ID", - ], -} - -## ============================================== -## TokyocabinetDriver -## ============================================== -class TokyocabinetDriver(AbstractDriver): - - ## Tokyo Tyrant provides one connection per *table*, not per database. - - ## Config files have no hierarchy. Let's set up our own hierarchy as a - ## stringfied dictionary and evaluate it later. - - ## Each table connection defines the host and port location of the Tokyo - ## Tyrant server - - DEFAULT_CONFIG = { "servers": ("Tokyo Tyrant server configuration", '{ 0: { "' + - constants.TABLENAME_ITEM + '":{ "host": "localhost", "port": 1978, },"' + - constants.TABLENAME_WAREHOUSE + '" : { "host": "localhost", "port": 1979, },"' + - constants.TABLENAME_DISTRICT + '" : { "host": "localhost", "port": 1980, },"' + - constants.TABLENAME_CUSTOMER + '" : { "host": "localhost", "port": 1981, },"' + - constants.TABLENAME_STOCK + '" : { "host": "localhost", "port": 1982, },"' + - constants.TABLENAME_ORDERS + '" : { "host": "localhost", "port": 1983, },"' + - constants.TABLENAME_NEW_ORDER + '" : { "host": "localhost", "port": 1984, },"' + - constants.TABLENAME_ORDER_LINE+ '" : { "host": "localhost", "port": 1985, },"' + - constants.TABLENAME_HISTORY + '" : { "host": "localhost", "port": 1986, }, }, }' ), } - - def __init__(self, ddl): - super(TokyocabinetDriver, self).__init__("tokyocabinet", ddl) - self.databases = dict() - self.conn = dict() - self.numServers = 0 - - ##----------------------------------------------- - ## self.tupleToString - ##----------------------------------------------- - def tupleToString(self, tuple, sep=":"): - """Tokyo-Cabinet table-type databases only accept strings as keys. - This function transforms a compound key (tuple) into a string. - Tuples elements are separated by the sep char""" - return sep.join(str(t) for t in tuple) - - ##----------------------------------------------- - ## self.getServer - ##----------------------------------------------- - def getServer(self, warehouseID): - """Tokyo Cabinet does not support data partitioning. For the TPC-C - benchmark, we manually partition data based on the warehouse ID""" - return (warehouseID % self.numServers) - - ## ---------------------------------------------- - ## makeDefaultConfig - ## ---------------------------------------------- - def makeDefaultConfig(self): - return TokyocabinetDriver.DEFAULT_CONFIG - - ## ---------------------------------------------- - ## loadConfig - ## ---------------------------------------------- - def loadConfig(self, config): - for key in TokyocabinetDriver.DEFAULT_CONFIG.keys(): - assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) - - if config["servers"]: - # Whn reading from INI file, we need to convert the server - # description-object from string to a real dictionary - config["servers"] = eval(config["servers"]) - for serverId, tables in config["servers"].iteritems(): - self.databases[serverId] = tables - - # First connect to databases - for serverId, tables in self.databases.iteritems(): - self.conn[serverId] = dict() - for tab, values in tables.iteritems(): - self.conn[serverId][tab] = pyrant.Tyrant(values["host"], values["port"]) - ## FOR - - # Remove previous data - if config["reset"]: - for serverId, tables in self.conn.iteritems(): - for tab in tables.keys(): - logging.debug("Deleting database '%s' at server '%s'" % (tab, serverId)) - self.conn[serverId][tab].clear() - ## FOR - ## FOR - ## IF - - self.numServers = len(self.databases.keys()) - logging.info("Number of servers: %s" % self.numServers) - - ## ------------------------------------------- - ## loadTuples - ## ------------------------------------------- - def loadTuples(self, tableName, tuples): - """Load tuples into tables of database - Each table is a connection to a Tyrant server. Each record is a key-value pair, - where key = primary key, values = concatenation of columns (dictionary). If - key is compound we transform it into a string, since TC does not support - compound keys. Data partitioning occurs based on Warehouse ID.""" - - if len(tuples) == 0: return - - logging.debug("Loading %d tuples of tableName %s" % (len(tuples), tableName)) - - assert tableName in TABLE_COLUMNS, "Unexpected table %s" % tableName - columns = TABLE_COLUMNS[tableName] - num_columns = xrange(len(columns)) - records = list() - - if tableName == constants.TABLENAME_WAREHOUSE: - for t in tuples: - w_key = t[0] # W_ID - sID = self.getServer(w_key) - cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) - records.append((str(w_key), cols)) - ## FOR - - try: - self.conn[sID][tableName].multi_set(records) - except KeyError, err: - sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) - sys.exit(1) - - elif tableName == constants.TABLENAME_DISTRICT: - for t in tuples: - w_key = t[1] # W_ID - sID = self.getServer(w_key) - d_key = self.tupleToString(t[:2]) # D_ID, D_W_ID - cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) - records.append((d_key, cols)) - ## FOR - - try: - self.conn[sID][tableName].multi_set(records) - except KeyError, err: - sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) - sys.exit(1) - - ## Item table doesn't have a w_id for partition. Replicate it to all - ## servers - elif tableName == constants.TABLENAME_ITEM: - for t in tuples: - i_key = str(t[0]) - cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) - records.append((i_key, cols)) - ## FOR - - for i in xrange(self.numServers): - try: - self.conn[i][tableName].multi_set(records) - except KeyError, err: - sys.stderr.write("%s(%s): server ID doesn't exist or is offline\n" %(KeyError, err)) - sys.exit(1) - ## FOR - ## FOR - - elif tableName == constants.TABLENAME_CUSTOMER: - for t in tuples: - w_key = t[2] # W_ID - sID = self.getServer(w_key) - c_key = self.tupleToString(t[:3]) # C_ID, C_D_ID, C_W_ID - cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) - records.append((c_key, cols)) - ## FOR - - try: - self.conn[sID][tableName].multi_set(records) - except KeyError, err: - sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) - sys.exit(1) - - elif tableName == constants.TABLENAME_HISTORY: - for t in tuples: - w_key = t[4] # W_ID - # Not really a primary key, but we need to generate - # something tobe our key - h_key = self.tupleToString(t[:3]) # H_C_ID, H_C_D, H_C_W - sID = self.getServer(w_key) - cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) - records.append((h_key, cols)) - ## FOR - - try: - self.conn[sID][tableName].multi_set(records) - except KeyError, err: - sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) - sys.exit(1) - - elif tableName == constants.TABLENAME_STOCK: - for t in tuples: - w_key = t[1] # W_ID - sID = self.getServer(w_key) - s_key = self.tupleToString(t[:2]) # S_ID, S_W_ID - cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) - records.append((s_key, cols)) - ## FOR - - try: - self.conn[sID][tableName].multi_set(records) - except KeyError, err: - sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) - sys.exit(1) - - elif tableName == constants.TABLENAME_ORDERS: - for t in tuples: - w_key = t[3] # W_ID - sID = self.getServer(w_key) - o_key = self.tupleToString(t[1:4]) # O_ID, O_D_ID, O_W_ID - cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) - records.append((o_key, cols)) - ## FOR - - try: - self.conn[sID][tableName].multi_set(records) - except KeyError, err: - sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) - sys.exit(1) - - elif tableName == constants.TABLENAME_NEW_ORDER: - for t in tuples: - w_key = t[2] # W_ID - sID = self.getServer(w_key) - no_key = self.tupleToString(t[:3]) # NO_O_ID, NO_D_ID, NO_W_ID - cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) - records.append((no_key, cols)) - ## FOR - - try: - self.conn[sID][tableName].multi_set(records) - except KeyError, err: - sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) - sys.exit(1) - - elif tableName == constants.TABLENAME_ORDER_LINE: - for t in tuples: - w_key = t[2] # W_ID - sID = self.getServer(w_key) - ol_key = self.tupleToString(t[:4]) # OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER - cols = dict(map(lambda i: (columns[i], t[i]), num_columns)) - records.append((ol_key, cols)) - ## FOR - - try: - self.conn[sID][tableName].multi_set(records) - except KeyError, err: - sys.stderr.write("%s(%s): server ID does not exist or is offline\n" %(KeyError, err)) - sys.exit(1) - - logging.debug("Loaded %s tuples for tableName %s" % (len(tuples), tableName)) - return - - ## ------------------------------------------- - ## loadFinish - ## ------------------------------------------- - def loadFinish(self): - - conn = dict() - - logging.info("Creating indexes...") - # Add indexes to database after loading all data - for serverId, tables in self.databases.iteritems(): - conn[serverId] = dict() - for tab, connValues in tables.iteritems(): - conn[serverId][tab] = protocol.TyrantProtocol(connValues["host"], connValues["port"]) - for index_name in TABLE_COLUMNS[tab]: - conn[serverId][tab].add_index(index_name) - ## FOR - ## FOR - - logging.info("Optimizing indexes...") - # Optimize indexes for faster access - for serverId, tables in self.databases.iteritems(): - for tab, connValues in tables.iteritems(): - for index_name in TABLE_COLUMNS[tab]: - conn[serverId][tab].optimize_index(index_name) - ## FOR - ## FOR - - logging.info("Syncing to disk...") - # Finally, flush everything to disk - for tab in TABLE_COLUMNS.keys(): - for sID in self.conn.keys(): - self.conn[sID][tab].sync() - ## FOR - - logging.info("Finished loading tables") - - ## -------------------------------------------- - ## doDelivery - ## -------------------------------------------- - def doDelivery(self, params): - """Execute DELIVERY Transaction - Parameters Dict: - w_id - o_carrier_id - ol_delivery_id - """ - - w_id = params["w_id"] - o_carrier_id = params["o_carrier_id"] - ol_delivery_d = params["ol_delivery_d"] - - sID = self.getServer(w_id) - - newOrderQuery = self.conn[sID][constants.TABLENAME_NEW_ORDER].query - ordersQuery = self.conn[sID][constants.TABLENAME_ORDERS].query - orderLineQuery = self.conn[sID][constants.TABLENAME_ORDER_LINE].query - customerQuery = self.conn[sID][constants.TABLENAME_CUSTOMER].query - - results = [ ] - for d_id in xrange(1, constants.DISTRICTS_PER_WAREHOUSE+1): - - # getNewOrder - # SELECT NO_O_ID FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID > -1 LIMIT 1 - - newOrders = newOrderQuery.filter(NO_D_ID = d_id, NO_W_ID = w_id, NO_O_ID__gt = -1).columns("NO_O_ID") - if len(newOrders) == 0: - ## No orders for this district: skip it. Note: This must - ## reported if > 1% - continue - assert(newOrders) > 0 - no_o_id = int(newOrders[0]["NO_O_ID"]) - - # sumOLAmount - # SELECT SUM(OL_AMOUNT) FROM ORDER_LINE WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ? - - olines = orderLineQuery.filter(OL_O_ID = no_o_id, OL_D_ID = d_id, OL_W_ID = w_id).columns("OL_AMOUNT") - - # These must be logged in the "result file" according to TPC-C - # 2.7.22 (page 39) - # We remove the queued time, completed time, w_id, and - # o_carrier_id: the client can figure them out - # If there are no order lines, SUM returns null. There should - # always be order lines. - assert len(olines) > 0, "ol_total is NULL: there are no order lines. This should not happen" - - ol_total = sum(float(i["OL_AMOUNT"]) for i in olines) - - assert ol_total > 0.0 - - # deleteNewOrder - # DELETE FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID = ? - - orders = newOrderQuery.filter(NO_D_ID = d_id, NO_W_ID = w_id, NO_O_ID = no_o_id) - orders.delete(quick=True) - - # HACK: not transactionally safe - # getCId - # SELECT O_C_ID FROM ORDERS WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ? - - orders = ordersQuery.filter(O_ID = no_o_id, O_D_ID = d_id, O_W_ID = w_id) - assert len(orders) > 0 - c_id = int(orders.columns("O_C_ID")[0]["O_C_ID"]) - - # updateOrders - # UPDATE ORDERS SET O_CARRIER_ID = ? WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ? - - records = list() - for record in orders: - key, cols = record - cols["O_CARRIER_ID"] = o_carrier_id - records.append((key, cols)) - ## FOR - - self.conn[sID][constants.TABLENAME_ORDERS].multi_set(records) - - # updateOrderLine - # UPDATE ORDER_LINE SET OL_DELIVERY_D = ? WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ? - - orders = orderLineQuery.filter(OL_O_ID = no_o_id, OL_D_ID = d_id, OL_W_ID = w_id) - records = list() - for record in orders: - key, cols = record - cols["OL_DELIVERY_D"] = ol_delivery_d - records.append((key, cols)) - ## FOR - - self.conn[sID][constants.TABLENAME_ORDER_LINE].multi_set(records) - - # updateCustomer - # UPDATE CUSTOMER SET C_BALANCE = C_BALANCE + ? WHERE C_ID = ? AND C_D_ID = ? AND C_W_ID = ? - - customers = customerQuery.filter(C_ID = c_id, C_D_ID = d_id, C_W_ID = w_id) - records = list() - for record in customers: - key, cols = record - cols["C_BALANCE"] = float(cols["C_BALANCE"]) + ol_total - records.append((key, cols)) - ## FOR - - self.conn[sID][constants.TABLENAME_CUSTOMER].multi_add(records) - - results.append((d_id, no_o_id)) - ## FOR - - return results - - def doNewOrder(self, params): - """Execute NEW_ORDER Transaction - Parameters Dict: - w_id - d_id - c_id - o_entry_id - i_ids - i_w_ids - i_qtys - """ - - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - o_entry_d = params["o_entry_d"] - i_ids = params["i_ids"] - i_w_ids = params["i_w_ids"] - i_qtys = params["i_qtys"] - - assert len(i_ids) > 0 - assert len(i_ids) == len(i_w_ids) - assert len(i_ids) == len(i_qtys) - - sID = self.getServer(w_id) - - warehouseQuery = self.conn[sID][constants.TABLENAME_WAREHOUSE].query - districtQuery = self.conn[sID][constants.TABLENAME_DISTRICT].query - customerQuery = self.conn[sID][constants.TABLENAME_CUSTOMER].query - orderQuery = self.conn[sID][constants.TABLENAME_ORDERS].query - newOrderQuery = self.conn[sID][constants.TABLENAME_NEW_ORDER].query - itemQuery = self.conn[sID][constants.TABLENAME_ITEM].query - - all_local = True - items = [ ] - for i in xrange(len(i_ids)): - ## Determine if this is an all local order or not - all_local = all_local and i_w_ids[i] == w_id - # getItemInfo - retItems = itemQuery.filter(I_PRICE = i_ids[i]) - if len(items) > 0: - items.append(retItems.columns()) - else: - items.append([]) - assert len(items) == len(i_ids) - - ## TPCC define 1% of neworder gives a wrong itemid, causing rollback. - ## Note that this will happen with 1% of transactions on purpose. - for item in items: - if len(item) == 0: - ## TODO Abort here! - return - ## FOR - - ## ----------------- - ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER - ## ----------------- - - # getWarehouseTaxRate - # SELECT W_TAX FROM WAREHOUSE WHERE W_ID = ? - - taxes = warehouseQuery.filter(W_ID = w_id) - w_tax = float(taxes.columns("W_TAX")[0]["W_TAX"]) - - # getDistrict - # SELECT D_TAX, D_NEXT_O_ID FROM DISTRICT WHERE D_ID = ? AND D_W_ID = ? - - districts = districtQuery.filter(D_ID = d_id, D_W_ID = w_id) - districtInfo = districts.columns("D_TAX", "D_NEXT_O_ID")[0] - d_tax = float(districtInfo["D_TAX"]) - d_next_o_id = int(districtInfo["D_NEXT_O_ID"]) - - # incrementNextOrderId - # HACK: This is not transactionally safe! - # UPDATE DISTRICT SET D_NEXT_O_ID = ? WHERE D_ID = ? AND D_W_ID = ? - records = list() - for record in districts: - key, cols = record - cols["D_NEXT_O_ID"] = d_next_o_id + 1 - records.append((key, cols)) - ## FOR - - self.conn[sID][constants.TABLENAME_DISTRICT].multi_set(records) - - # getCustomer - # SELECT C_DISCOUNT, C_LAST, C_CREDIT FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? - - customers = customerQuery.filter(C_W_ID = w_id, C_D_ID = d_id, C_ID = c_id) - customerInfo = customers.columns("C_DISCOUNT", "C_LAST", "C_CREDIT")[0] - c_discount = float(customerInfo["C_DISCOUNT"]) - - ## ----------------- - ## Insert Order Information - ## ----------------- - ol_cnt = len(i_ids) - o_carrier_id = constants.NULL_CARRIER_ID - - # createOrder - # INSERT INTO ORDERS (O_ID, O_D_ID, O_W_ID, O_C_ID, O_ENTRY_D, O_CARRIER_ID, O_OL_CNT, - # O_ALL_LOCAL) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - - key = self.tupleToString((d_next_o_id, d_id, w_id)) - cols = {"O_ID": d_next_o_id, "O_D_ID": d_id, "O_W_ID": w_id, "O_C_ID": - c_id, "O_ENTRY_D": o_entry_d, "O_CARRIER_ID": - o_carrier_id, "O_OL_CNT": o_ol_cnt, "O_ALL_LOCAL": - all_local} - self.conn[sID][constants.TABLENAME_ORDERS].multi_set([(key, cols)]) - - # createNewOrder - # INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, NO_W_ID) VALUES (?, ?, ?) - - key = self.tupleToString((d_next_o_id, d_id, w_id)) - cols = {"NO_O_ID": d_next_o_id, "NO_D_ID": d_id, "NO_W_ID": w_id} - self.conn[sID][constants.TABLENAME_NEW_ORDER].multi_set([(key, cols)]) - - ## ------------------------------- - ## Insert Order Item Information - ## ------------------------------- - - item_data = [ ] - total = 0 - for i in xrange(len(i_id)): - ol_number = i+1 - ol_supply_w_id = i_w_ids[i] - ol_i_id = i_ids[i] - ol_quantity = i_qtys[i] - - # getItemInfo - # SELECT I_PRICE, I_NAME, I_DATA FROM ITEM WHERE I_ID = ? - - key, itemInfo = items[i] - i_price = float(itemInfo["I_PRICE"]) - i_name = itemInfo["I_NAME"] - i_data = itemInfo["I_DATA"] - - # getStockInfo - # SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DIST_%02d FROM STOCK - # WHERE S_I_ID = ? AND S_W_ID = ? - - stocks = stockQuery.filter(S_I_ID = ol_i_id, S_W_ID = ol_supply_w_id) - if len(stocks) == 0: - logging.warn("No STOCK record for (ol_i_id=%d, ol_supply_w_id=%d)" - % (ol_i_id, ol_supply_w_id)) - continue - stockInfo = stock.columns("S_QUANTITY", "S_DATA", "S_YTD", "S_ORDER_CNT", - "S_REMOTE_CNT", "S_DIST_%02d"%d_id)[0] - - s_quantity = int(stockInfo["S_QUANTITY"]) - s_ytd = float(stockInfo["S_YTD"]) - s_order_cnt = int(stockInfo["S_ORDER_CNT"]) - s_remote_cnt = int(stockInfo["S_REMOTE_CNT"]) - s_data = stockInfo["S_DATA"] - s_dist_xx = stockInfo["S_DIST_%02d"%d_id] # Fetches data from the - # s_dist_[d_id] column - - # updateStock - # UPDATE STOCK SET S_QUANTITY = ?, S_YTD = ?, S_ORDER_CNT = ?, S_REMOTE_CNT = ? - # WHERE S_I_ID = ? AND S_W_ID = ? - - s_ytd += ol_quantity - if s_quantity >= ol_quantity + 10: - s_quantity = s_quantity - ol_quantity - else: - s_quantity = s_quantity + 91 - ol_quantity - s_order_cnt += 1 - - if ol_supply_w_id != w_id: s_remote_cnt += 1 - - sSID = self.getServer(ol_supply_w_id) - stockQuery = self.conn[sID][constants.TABLENAME_STOCK].query - stocks = stockQuery.filter(S_I_ID = ol_i_id, S_W_ID = ol_supply_w_id) - records = list() - for record in stocks: - key, cols = record - cols["S_QUANTITY"] = s_quantity - cols["S_YTD"] = s_ytd - cols["S_ORDER_CNT"] = s_order_cnt - cols["S_REMOTE_CNT"] = s_remote_cnt - records.append((key, cols)) - ## FOR - - self.conn[sSID][constants.TABLENAME_STOCK].multi_set(records) - - if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: - brand_generic = 'B' - else: - brand_generic = 'G' - - ## Transaction profile states to use "ol_quantity * i_price" - ol_amount = ol_quantity * i_price - total += ol_amount - - # createOrderLine - # INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, - # OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - - key = tupletoString((d_next_o_id, d_id, w_id, ol_number)) - cols = {"OL_O_ID": d_next_o_id, "OL_D_ID": d_id, "OL_W_ID": w_id, - "OL_NUMBER": ol_number, "OL_I_ID": ol_i_id, - "OL_SUPPLY_W_ID": ol_supply_w_id, "OL_DELIVERY_D": - ol_entry_d, "OL_QUANTITY": ol_quantity, "OL_AMOUNT": - ol_amount, "OL_DIST_INFO": s_dist_xx} - self.conn[sID][constants.TABLENAME_ORDER_LINE].multi_set([(key, cols)]) - - ## Add the info to be returned - item_data.append((i_name, s_quantity, brand_generic, i_price, ol_amount)) - ## FOR - - ## Adjust the total for the discount - #print "c_discount:", c_discount, type(c_discount) - #print "w_tax:", w_tax, type(w_tax) - #print "d_tax:", d_tax, type(d_tax) - total *= (1 - c_discount) * (1 + w_tax + d_tax) - - ## Pack up values the client is missing (see TPC-C 2.4.3.5) - misc = [(w_tax, d_tax, d_next_o_id, total)] - - return [ customerInfo, misc, item_data ] - - def doOrderStatus(self, params): - """Execute ORDER_STATUS Transaction - Parameters Dict: - w_id - d_id - c_id - c_last - """ - w_id = params["w_id"] - d_id = params["d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - - assert w_id, pformat(params) - assert d_id, pformat(params) - - sID = self.getServer(w_id) - - customerQuery = self.conn[sID][constants.TABLENAME_CUSTOMER].query - orderQuery = self.conn[sID][constants.TABLENAME_ORDERS].query - orderLineQuery= self.conn[sID][constants.TABLENAME_ORDER_LINE].query - - if c_id != None: - # getCustomerByCustomerId - # SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER - # WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? - - customers = customerQuery.filter(C_W_ID = w_id, C_D_ID = d_id, C_ID = c_id) - customerInfo = customers.columns("C_ID", "C_FIRST", "C_MIDDLE", "C_LAST", "C_BALANCE")[0] - else: - # Get the midpoint customer's id - # getCustomersByLastName - # SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER - # WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST - - customers = customerQuery.filter(C_W_ID = w_id, C_D_ID = d_id, C_LAST__contains = c_last).order_by("C_FIRST", numeric=False) - all_customers = customers.columns("C_ID", "C_FIRST", "C_MIDDLE", "C_LAST", "C_BALANCE") - namecnt = len(all_customers) - assert namecnt > 0 - index = (namecnt-1)/2 - customerInfo = all_customers[index] - c_id = int(customerInfo["C_ID"]) - assert len(customerInfo) > 0 - - # getLastOrder TODO: LIMIT 1 - # SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS - # WHERE O_W_ID = ? AND O_D_ID = ? AND O_C_ID = ? ORDER BY O_ID DESC LIMIT 1 - - orders = orderQuery.filter(O_W_ID = w_id, O_D_ID = d_id, O_C_ID = c_id).order_by("-O_ID", numeric=True) - orderInfo = orders.columns("O_ID", "O_CARRIER_ID", "O_ENTRY_D")[0] - - # getOrderLines - # SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE - # WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID = ? - if len(orders) > 0: - o_id = int(orderInfo["O_ID"]) - orders = orderLineQuery.filter(OL_W_ID = w_id, OL_D_ID = d_id, OL_O_ID = o_id) - orderLines = orders.columns("OL_SUPPLY_W_ID", "OL_I_ID", "OL_QUANTITY", "OL_AMOUNT", "OL_DELIVERY_D") - else: - orderLines = [ ] - - return [customerInfo, orderInfo, orderLines] - - def doPayment(self, params): - """Execute PAYMENT Transaction - Parameters Dict: - w_id - d_id - h_amount - c_w_id - c_d_id - c_id - c_last - h_date - """ - - w_id = params["w_id"] - d_id = params["d_id"] - h_amount = params["h_amount"] - c_w_id = params["c_w_id"] - c_d_id = params["c_d_id"] - c_id = params["c_id"] - c_last = params["c_last"] - h_date = params["h_date"] - - sID = self.getServer(w_id) - cSID = self.getServer(c_w_id) - - customerQuery = self.conn[cSID][constants.TABLENAME_CUSTOMER].query - warehouseQuery = self.conn[sID][constants.TABLENAME_WAREHOUSE].query - districtQuery = self.conn[sID][constants.TABLENAME_DISTRICT].query - - if c_id != None: - # getCustomerByCustomerId - # SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, - # C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, - # C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER - # WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? - - customers = customerQuery.filter(C_W_ID = c_w_id, C_D_ID = d_id, C_ID = c_id) - customerInfo = customers.columns("C_ID", "C_BALANCE", "C_YTD_PAYMENT", "C_PAYMENT_CNT", "C_DATA")[0] - else: - # Get the midpoint customer's id - # getCustomersByLastName - # SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, - # C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, - # C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER - # WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST - - customers = customerQuery.filter(C_W_ID = c_w_id, C_D_ID = d_id, C_LAST__contains = c_last).order_by("C_FIRST", numeric=False) - all_customers = customers.columns("C_ID", "C_BALANCE", "C_YTD_PAYMENT", "C_PAYMENT_CNT", "C_DATA") - namecnt = len(all_customers) - assert namecnt > 0 - index = (namecnt-1)/2 - customerInfo = all_customers[index] - c_id = int(customerInfo["C_ID"]) - assert len(customerInfo) > 0 - - c_balance = float(customerInfo["C_BALANCE"]) - h_amount - c_ytd_payment = float(customerInfo["C_YTD_PAYMENT"]) + h_amount - c_payment_cnt = int(customerInfo["C_PAYMENT_CNT"]) + 1 - c_data = customerInfo["C_DATA"] - - # getWarehouse - # SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP FROM WAREHOUSE WHERE W_ID = ? - - warehouses = warehouseQuery.filter(W_ID = w_id) - warehouseInfo = warehouses.columns("W_NAME","W_STREET_1", "W_STREET_2", "W_CITY", "W_STATE", "W_ZIP")[0] - - # updateWarehouseBalance - # UPDATE WAREHOUSE SET W_YTD = W_YTD + ? WHERE W_ID = ? - - warehouses = warehouseQuery.filter(W_ID = w_id) - records = list() - for record in warehouses: - key, cols = record - cols["W_YTD"] = float(cols["W_YTD"]) + h_amount - records.append((key, cols)) - ## FOR - self.conn[sID][constants.TABLENAME_WAREHOUSE].multi_set(records) - - # getDistrict - # SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP FROM DISTRICT - # WHERE D_W_ID = ? AND D_ID = ? - - districts = districtQuery.filter(D_W_ID = w_id, D_ID = d_id) - districtInfo = districts.columns("D_NAME", "D_STREET_1", "D_STREET_2", "D_CITY", "D_STATE", "D_ZIP")[0] - - # updateDistrictBalance - # UPDATE DISTRICT SET D_YTD = D_YTD + ? WHERE D_W_ID = ? AND D_ID = ? - - districts = districtQuery.filter(W_ID = w_id, D_ID = d_id) - records = list() - for record in districts: - key, cols = record - cols["D_YTD"] = float(cols["D_YTD"]) + h_amount - records.append((key, cols)) - ## FOR - self.conn[sID][constants.TABLENAME_DISTRICT].multi_set(records) - - # Customer Credit Information - customers = customerQuery.filter(C_W_ID = c_w_id, C_D_ID = c_d_id, C_ID = c_id) - cInfo = customers.columns("C_CREDIT")[0] - - if cInfo["C_CREDIT"] == constants.BAD_CREDIT: - newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) - c_data = (newData + "|" + c_data) - if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] - - # updateBCCustomer - # UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ?, C_DATA = ? - # WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? - - records = list() - for record in customers: - key, cols = record - cols["C_BALANCE"] = c_balance - cols["C_YTD_PAYMENT"] = c_ytd_payment - cols["C_PAYMENT_CNT"] = c_payment_cnt - cols["C_DATA"] = c_data - records.append((key, cols)) - ## FOR - self.conn[cSID][constants.TABLENAME_CUSTOMER].multi_set(records) - else: - c_data = "" - - records = list() - # updateGCCustomer - # UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ? - # WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? - - for record in customers: - key, cols = record - cols["C_BALANCE"] = c_balance - cols["C_YTD_PAYMENT"] = c_ytd_payment - cols["C_PAYMENT_CNT"] = c_payment_cnt - records.append((key, cols)) - ## FOR - self.conn[cSID][constants.TABLENAME_CUSTOMER].multi_set(records) - - # Concatenate w_name, four space, d_name - h_data = "%s %s" % (warehouseInfo["W_NAME"], districtInfo["D_NAME"]) - - # Create the history record - # INSERT INTO HISTORY VALUES (?, ?, ?, ?, ?, ?, ?, ?) - - h_key = self.tupleToString((c_id, c_d_id, c_w_id)) - - cols = {"H_C_ID": c_id, "H_C_D_ID": c_d_id, "H_C_W_ID": c_w_id, "H_D_ID": - d_id, "H_W_ID": w_id, "H_DATE": h_date, "H_AMOUNT": - h_amount, "H_DATA": h_data} - self.conn[sID][constants.TABLENAME_HISTORY].multi_set([(key, cols)]) - - # TPC-C 2.5.3.3: Must display the following fields: - # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, - # W_STATE, W_ZIP, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, - # C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, - # C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, - # C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = - # "BC"), H_AMMOUNT, and H_DATE. - - # Hand back all the warehouse, district, and customer data - return [ warehouseInfo, districtInfo, customerInfo ] - - def doStockLevel(self, params): - """Execute STOCK_LEVEL Transaction - Parameters Dict: - w_id - d_id - threshold - """ - w_id = params["w_id"] - d_id = params["d_id"] - threshold = params["threshold"] - - sID = self.getServer(w_id) - - districtQuery = self.conn[sID][constants.TABLENAME_DISTRICT].query - orderLineQuery = self.conn[sID][constants.TABLENAME_ORDER_LINE].query - stockQuery = self.conn[sID][constants.TABLENAME_STOCK].query - - # getOId - # "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?" - - districts = districtQuery.filter(D_W_ID = w_id, D_ID = d_id) - o_id = int(districts.columns("D_NEXT_O_ID")[0]["D_NEXT_O_ID"]) - - # getStockCount - # SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK WHERE OL_W_ID = ? AND OL_D_ID = ? - # AND OL_O_ID < ? AND OL_O_ID >= ? AND S_W_ID = ? AND S_I_ID = OL_I_ID AND S_QUANTITY < ? - - orders = orderLineQuery.filter(OL_W_ID = w_id, OL_D_ID = d_id, OL_O_ID__between = [(o_id-20), (o_id-1)]) - ol_i_ids = set([i["OL_I_ID"] for i in orders.columns("OL_I_ID")]) - - stocks = stockQuery.filter(S_W_ID = w_id, S_I_ID__in = list(ol_i_ids), S_QUANTITY__lt = threshold).columns("S_W_ID") - - cnt = len(stocks) - - return cnt - -## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/message.py b/workloads/chbenchmark/py-tpcc/pytpcc/message.py index 78cca54b..8ef84ccc 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/message.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/message.py @@ -46,8 +46,8 @@ CMD_STOP = 3 LOAD_COMPLETED = 4 EXECUTE_COMPLETED = 5 - + class Message: - def __init__(self,header=EMPTY,data=None): - self.header=header - self.data=data + def __init__(self, header=EMPTY, data=None): + self.header = header + self.data = data diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py index 4e048c21..8606b43e 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py @@ -66,7 +66,7 @@ def execute(self, duration): val = self.driver.executeTransaction(txn, params) except KeyboardInterrupt: return -1 - except (Exception, AssertionError), ex: + except (Exception, AssertionError) as ex: logging.warn("Failed to execute Transaction '%s': %s" % (txn, ex)) if debug: traceback.print_exc(file=sys.stdout) if self.stop_on_error: raise diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py b/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py index 61041e82..dfde3e29 100755 --- a/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py @@ -114,7 +114,7 @@ def loaderFunc(driverClass, scaleParameters, args, config, w_ids, debug): driver.loadFinish() except KeyboardInterrupt: return -1 - except (Exception, AssertionError), ex: + except (Exception, AssertionError) as ex: logging.warn("Failed to load data: %s" % (ex)) #if debug: traceback.print_exc(file=sys.stdout) @@ -216,7 +216,7 @@ def executorFunc(driverClass, scaleParameters, args, config, debug): assert driver != None, "Failed to create '%s' driver" % args['system'] if args['print_config']: config = driver.makeDefaultConfig() - print driver.formatConfig(config) + print(driver.formatConfig(config)) print sys.exit(0) @@ -267,7 +267,7 @@ def executorFunc(driverClass, scaleParameters, args, config, debug): else: results = startExecution(driverClass, scaleParameters, args, config) assert results - print results.show(load_time) + print(results.show(load_time)) ## IF -## MAIN \ No newline at end of file +## MAIN diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/util/rand.py b/workloads/chbenchmark/py-tpcc/pytpcc/util/rand.py index 4e876659..faefbdcb 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/util/rand.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/util/rand.py @@ -32,21 +32,37 @@ import random import nurand -SYLLABLES = [ "BAR", "OUGHT", "ABLE", "PRI", "PRES", "ESE", "ANTI", "CALLY", "ATION", "EING" ] +SYLLABLES = [ + "BAR", + "OUGHT", + "ABLE", + "PRI", + "PRES", + "ESE", + "ANTI", + "CALLY", + "ATION", + "EING", +] + +nurandVar = None # NURand + -nurandVar = None # NURand def setNURand(nu): global nurandVar nurandVar = nu + + ## DEF + def NURand(a, x, y): """A non-uniform random number, as defined by TPC-C 2.1.6. (page 20).""" global nurandVar assert x <= y if nurandVar is None: - setNURand(nurand.makeForLoad()) - + setNURand(nurand.makeForLoad()) + if a == 255: c = nurandVar.cLast elif a == 1023: @@ -55,29 +71,39 @@ def NURand(a, x, y): c = nurandVar.orderLineItemId else: raise Exception("a = " + a + " is not a supported value") - + return (((number(0, a) | number(x, y)) + c) % (y - x + 1)) + x + + ## DEF + def number(minimum, maximum): value = random.randint(minimum, maximum) assert minimum <= value and value <= maximum return value + + ## DEF + def numberExcluding(minimum, maximum, excluding): """An in the range [minimum, maximum], excluding excluding.""" assert minimum < maximum assert minimum <= excluding and excluding <= maximum ## Generate 1 less number than the range - num = number(minimum, maximum-1) + num = number(minimum, maximum - 1) ## Adjust the numbers to remove excluding - if num >= excluding: num += 1 + if num >= excluding: + num += 1 assert minimum <= num and num <= maximum and num != excluding return num -## DEF + + +## DEF + def fixedPoint(decimal_places, minimum, maximum): assert decimal_places > 0 @@ -91,8 +117,11 @@ def fixedPoint(decimal_places, minimum, maximum): int_max = int(maximum * multiplier + 0.5) return float(number(int_min, int_max) / float(multiplier)) + + ## DEF + def selectUniqueIds(numUnique, minimum, maximum): rows = set() for i in range(0, numUnique): @@ -104,38 +133,56 @@ def selectUniqueIds(numUnique, minimum, maximum): ## FOR assert len(rows) == numUnique return rows + + ## DEF + def astring(minimum_length, maximum_length): """A random alphabetic string with length in range [minimum_length, maximum_length].""" - return randomString(minimum_length, maximum_length, 'a', 26) + return randomString(minimum_length, maximum_length, "a", 26) + + ## DEF + def nstring(minimum_length, maximum_length): """A random numeric string with length in range [minimum_length, maximum_length].""" - return randomString(minimum_length, maximum_length, '0', 10) + return randomString(minimum_length, maximum_length, "0", 10) + + ## DEF + def randomString(minimum_length, maximum_length, base, numCharacters): length = number(minimum_length, maximum_length) baseByte = ord(base) string = "" for i in range(length): - string += chr(baseByte + number(0, numCharacters-1)) + string += chr(baseByte + number(0, numCharacters - 1)) return string + + ## DEF + def makeLastName(number): """A last name as defined by TPC-C 4.3.2.3. Not actually random.""" global SYLLABLES assert 0 <= number and number <= 999 - indicies = [ number/100, (number/10)%10, number%10 ] + indicies = [number / 100, (number / 10) % 10, number % 10] return "".join(map(lambda x: SYLLABLES[x], indicies)) + + ## DEF + def makeRandomLastName(maxCID): """A non-uniform random last name, as defined by TPC-C 4.3.2.3. The name will be limited to maxCID.""" min_cid = 999 - if (maxCID - 1) < min_cid: min_cid = maxCID - 1 + if (maxCID - 1) < min_cid: + min_cid = maxCID - 1 return makeLastName(NURand(255, 0, min_cid)) + + ## DEF diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/worker.py b/workloads/chbenchmark/py-tpcc/pytpcc/worker.py index 5e2f9a62..02941973 100755 --- a/workloads/chbenchmark/py-tpcc/pytpcc/worker.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/worker.py @@ -61,12 +61,12 @@ def loaderFunc(driverClass, scaleParameters, args, config, w_ids, debug): driver = driverClass(args['ddl']) assert driver != None logging.debug("Starting client execution: %s [warehouses=%d]" % (driver, len(w_ids))) - + config['load'] = True config['execute'] = False config['reset'] = False driver.loadConfig(config) - + try: loadItems = (1 in w_ids) l = loader.Loader(driver, scaleParameters, w_ids, loadItems) @@ -75,7 +75,7 @@ def loaderFunc(driverClass, scaleParameters, args, config, w_ids, debug): driver.loadFinish() except KeyboardInterrupt: return -1 - except (Exception, AssertionError), ex: + except (Exception, AssertionError) as ex: logging.warn("Failed to load data: %s" % (ex)) #if debug: traceback.print_exc(file=sys.stdout) @@ -91,7 +91,7 @@ def executorFunc(driverClass, scaleParameters, args, config, debug): driver = driverClass(args['ddl']) assert driver != None logging.debug("Starting client execution: %s" % driver) - + config['execute'] = True config['reset'] = False driver.loadConfig(config) @@ -100,48 +100,47 @@ def executorFunc(driverClass, scaleParameters, args, config, debug): driver.executeStart() results = e.execute(args['duration']) driver.executeFinish() - + return results ## DEF ## MAIN if __name__=='__channelexec__': - driverClass=None + + driverClass = None for item in channel: - command=pickle.loads(item) - if command.header==message.CMD_LOAD: - scaleParameters=command.data[0] - args=command.data[1] - config=command.data[2] - w_ids=command.data[3] - - ## Create a handle to the target client driver at the client side - driverClass = createDriverClass(args['system']) - assert driverClass != None, "Failed to find '%s' class" % args['system'] - driver = driverClass(args['ddl']) - assert driver != None, "Failed to create '%s' driver" % args['system'] - - loaderFunc(driverClass,scaleParameters,args,config,w_ids,True) - m=message.Message(header=message.LOAD_COMPLETED) - channel.send(pickle.dumps(m,-1)) - elif command.header==message.CMD_EXECUTE: - scaleParameters=command.data[0] - args=command.data[1] - config=command.data[2] - - ## Create a handle to the target client driver at the client side - if driverClass==None: - driverClass = createDriverClass(args['system']) - assert driverClass != None, "Failed to find '%s' class" % args['system'] - driver = driverClass(args['ddl']) - assert driver != None, "Failed to create '%s' driver" % args['system'] - - results=executorFunc(driverClass,scaleParameters,args,config,True) - m=message.Message(header=message.EXECUTE_COMPLETED,data=results) - channel.send(pickle.dumps(m,-1)) - - elif command.header==message.CMD_STOP: - pass - else: - pass - \ No newline at end of file + command = pickle.loads(item) + if command.header == message.CMD_LOAD: + scaleParameters = command.data[0] + args = command.data[1] + config = command.data[2] + w_ids = command.data[3] + + ## Create a handle to the target client driver at the client side + driverClass = createDriverClass(args['system']) + assert driverClass != None, "Failed to find '%s' class" % args['system'] + driver = driverClass(args['ddl']) + assert driver != None, "Failed to create '%s' driver" % args['system'] + + loaderFunc(driverClass,scaleParameters,args,config,w_ids,True) + m = message.Message(header=message.LOAD_COMPLETED) + channel.send(pickle.dumps(m,-1)) + + elif command.header == message.CMD_EXECUTE: + scaleParameters = command.data[0] + args = command.data[1] + config = command.data[2] + + ## Create a handle to the target client driver at the client side + if driverClass == None: + driverClass = createDriverClass(args['system']) + assert driverClass != None, "Failed to find '%s' class" % args['system'] + driver = driverClass(args['ddl']) + assert driver != None, "Failed to create '%s' driver" % args['system'] + + results=executorFunc(driverClass,scaleParameters,args,config,True) + m=message.Message(header=message.EXECUTE_COMPLETED,data=results) + channel.send(pickle.dumps(m,-1)) + + elif command.header==message.CMD_STOP: + pass From b6cd2c7f998b07adc3f826899dbf7274e0cd772b Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Fri, 12 Apr 2024 17:01:42 -0400 Subject: [PATCH 3/5] Black reformat --- .../chbenchmark/py-tpcc/pytpcc/constants.py | 21 +- .../chbenchmark/py-tpcc/pytpcc/coordinator.py | 251 ++++-- .../py-tpcc/pytpcc/drivers/abstractdriver.py | 76 +- .../py-tpcc/pytpcc/drivers/couchdbdriver.py | 850 ++++++++++++------ .../py-tpcc/pytpcc/drivers/csvdriver.py | 59 +- .../py-tpcc/pytpcc/drivers/mongodbdriver.py | 762 ++++++++++------ .../py-tpcc/pytpcc/drivers/sqlitedriver.py | 257 +++--- .../chbenchmark/py-tpcc/pytpcc/message.py | 5 +- .../py-tpcc/pytpcc/runtime/executor.py | 144 ++- .../py-tpcc/pytpcc/runtime/loader.py | 320 +++++-- workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py | 308 ++++--- .../py-tpcc/pytpcc/util/__init__.py | 2 +- .../chbenchmark/py-tpcc/pytpcc/util/nurand.py | 6 +- .../py-tpcc/pytpcc/util/results.py | 63 +- .../py-tpcc/pytpcc/util/scaleparameters.py | 63 +- .../chbenchmark/py-tpcc/pytpcc/worker.py | 80 +- workloads/chbenchmark/py-tpcc/setup.py | 39 +- 17 files changed, 2172 insertions(+), 1134 deletions(-) diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/constants.py b/workloads/chbenchmark/py-tpcc/pytpcc/constants.py index 3918b3cd..52afd4dd 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/constants.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/constants.py @@ -129,15 +129,15 @@ ORIGINAL_STRING = "ORIGINAL" # Table Names -TABLENAME_ITEM = "ITEM" -TABLENAME_WAREHOUSE = "WAREHOUSE" -TABLENAME_DISTRICT = "DISTRICT" -TABLENAME_CUSTOMER = "CUSTOMER" -TABLENAME_STOCK = "STOCK" -TABLENAME_ORDERS = "ORDERS" -TABLENAME_NEW_ORDER = "NEW_ORDER" +TABLENAME_ITEM = "ITEM" +TABLENAME_WAREHOUSE = "WAREHOUSE" +TABLENAME_DISTRICT = "DISTRICT" +TABLENAME_CUSTOMER = "CUSTOMER" +TABLENAME_STOCK = "STOCK" +TABLENAME_ORDERS = "ORDERS" +TABLENAME_NEW_ORDER = "NEW_ORDER" TABLENAME_ORDER_LINE = "ORDER_LINE" -TABLENAME_HISTORY = "HISTORY" +TABLENAME_HISTORY = "HISTORY" ALL_TABLES = [ TABLENAME_ITEM, @@ -151,11 +151,14 @@ TABLENAME_HISTORY, ] + # Transaction Types def enum(*sequential, **named): enums = dict(map(lambda x: (x, x), sequential)) # dict(zip(sequential, range(len(sequential))), **named) - return type('Enum', (), enums) + return type("Enum", (), enums) + + TransactionTypes = enum( "DELIVERY", "NEW_ORDER", diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py b/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py index 9ddfb0dc..11e12341 100755 --- a/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/coordinator.py @@ -39,180 +39,239 @@ import worker import message from ConfigParser import SafeConfigParser -from pprint import pprint,pformat +from pprint import pprint, pformat from util import * from runtime import * import drivers -logging.basicConfig(level = logging.INFO, - format="%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s", - datefmt="%m-%d-%Y %H:%M:%S", - stream = sys.stdout) - +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s", + datefmt="%m-%d-%Y %H:%M:%S", + stream=sys.stdout, +) + + ## ============================================== ## createDriverClass ## ============================================== def createDriverClass(name): full_name = "%sDriver" % name.title() - mod = __import__('drivers.%s' % full_name.lower(), globals(), locals(), [full_name]) + mod = __import__("drivers.%s" % full_name.lower(), globals(), locals(), [full_name]) klass = getattr(mod, full_name) return klass + + ## DEF + ## ============================================== ## getDrivers ## ============================================== def getDrivers(): - drivers = [ ] - for f in map(lambda x: os.path.basename(x).replace("driver.py", ""), glob.glob("./drivers/*driver.py")): - if f != "abstract": drivers.append(f) - return (drivers) + drivers = [] + for f in map( + lambda x: os.path.basename(x).replace("driver.py", ""), + glob.glob("./drivers/*driver.py"), + ): + if f != "abstract": + drivers.append(f) + return drivers + + ## DEF + ## ============================================== ## startLoading ## ============================================== -def startLoading(scalParameters,args,config,channels): - #Split the warehouses into chunks +def startLoading(scalParameters, args, config, channels): + # Split the warehouses into chunks procs = len(channels) - w_ids = map(lambda x:[], range(procs)) - for w_id in range(scaleParameters.starting_warehouse, scaleParameters.ending_warehouse+1): + w_ids = map(lambda x: [], range(procs)) + for w_id in range( + scaleParameters.starting_warehouse, scaleParameters.ending_warehouse + 1 + ): idx = w_id % procs w_ids[idx].append(w_id) print(w_ids) - - load_start=time.time() + + load_start = time.time() for i in range(len(channels)): - m=message.Message(header=message.CMD_LOAD,data=[scalParameters,args,config,w_ids[i]]) - channels[i].send(pickle.dumps(m,-1)) + m = message.Message( + header=message.CMD_LOAD, data=[scalParameters, args, config, w_ids[i]] + ) + channels[i].send(pickle.dumps(m, -1)) for ch in channels: ch.receive() pass - return time.time()-load_start + return time.time() - load_start ## ============================================== ## startExecution ## ============================================== -def startExecution(scaleParameters, args, config,channels): +def startExecution(scaleParameters, args, config, channels): procs = len(channels) total_results = results.Results() - + for ch in channels: - m=message.Message(header=message.CMD_EXECUTE,data=[scaleParameters,args,config]) - ch.send(pickle.dumps(m,-1)) + m = message.Message( + header=message.CMD_EXECUTE, data=[scaleParameters, args, config] + ) + ch.send(pickle.dumps(m, -1)) for ch in channels: - r=pickle.loads(ch.receive()).data + r = pickle.loads(ch.receive()).data total_results.append(r) - return (total_results) + return total_results + + ## DEF ## ============================================== ## main ## ============================================== -if __name__ == '__main__': - aparser = argparse.ArgumentParser(description='Python implementation of the TPC-C Benchmark') - aparser.add_argument('system', choices=getDrivers(), - help='Target system driver') - aparser.add_argument('--config', type=file, - help='Path to driver configuration file') - aparser.add_argument('--reset', action='store_true', - help='Instruct the driver to reset the contents of the database') - aparser.add_argument('--scalefactor', default=1, type=float, metavar='SF', - help='Benchmark scale factor') - aparser.add_argument('--warehouses', default=4, type=int, metavar='W', - help='Number of Warehouses') - aparser.add_argument('--duration', default=60, type=int, metavar='D', - help='How long to run the benchmark in seconds') - aparser.add_argument('--ddl', default=os.path.realpath(os.path.join(os.path.dirname(__file__), "tpcc.sql")), - help='Path to the TPC-C DDL SQL file') +if __name__ == "__main__": + aparser = argparse.ArgumentParser( + description="Python implementation of the TPC-C Benchmark" + ) + aparser.add_argument("system", choices=getDrivers(), help="Target system driver") + aparser.add_argument( + "--config", type=file, help="Path to driver configuration file" + ) + aparser.add_argument( + "--reset", + action="store_true", + help="Instruct the driver to reset the contents of the database", + ) + aparser.add_argument( + "--scalefactor", + default=1, + type=float, + metavar="SF", + help="Benchmark scale factor", + ) + aparser.add_argument( + "--warehouses", default=4, type=int, metavar="W", help="Number of Warehouses" + ) + aparser.add_argument( + "--duration", + default=60, + type=int, + metavar="D", + help="How long to run the benchmark in seconds", + ) + aparser.add_argument( + "--ddl", + default=os.path.realpath(os.path.join(os.path.dirname(__file__), "tpcc.sql")), + help="Path to the TPC-C DDL SQL file", + ) ## number of processes per node - aparser.add_argument('--clientprocs', default=1, type=int, metavar='N', - help='Number of processes on each client node.') - - aparser.add_argument('--stop-on-error', action='store_true', - help='Stop the transaction execution when the driver throws an exception.') - aparser.add_argument('--no-load', action='store_true', - help='Disable loading the data') - aparser.add_argument('--no-execute', action='store_true', - help='Disable executing the workload') - aparser.add_argument('--print-config', action='store_true', - help='Print out the default configuration file for the system and exit') - aparser.add_argument('--debug', action='store_true', - help='Enable debug log messages') + aparser.add_argument( + "--clientprocs", + default=1, + type=int, + metavar="N", + help="Number of processes on each client node.", + ) + + aparser.add_argument( + "--stop-on-error", + action="store_true", + help="Stop the transaction execution when the driver throws an exception.", + ) + aparser.add_argument( + "--no-load", action="store_true", help="Disable loading the data" + ) + aparser.add_argument( + "--no-execute", action="store_true", help="Disable executing the workload" + ) + aparser.add_argument( + "--print-config", + action="store_true", + help="Print out the default configuration file for the system and exit", + ) + aparser.add_argument( + "--debug", action="store_true", help="Enable debug log messages" + ) args = vars(aparser.parse_args()) - if args['debug']: logging.getLogger().setLevel(logging.DEBUG) + if args["debug"]: + logging.getLogger().setLevel(logging.DEBUG) ## Arguments validation - assert args['reset'] == False or args['no_load'] == False, \ - "'--reset' and '--no-load' are incompatible with each other" + assert ( + args["reset"] == False or args["no_load"] == False + ), "'--reset' and '--no-load' are incompatible with each other" ## Create a handle to the target client driver - driverClass = createDriverClass(args['system']) - assert driverClass != None, "Failed to find '%s' class" % args['system'] - driver = driverClass(args['ddl']) - assert driver != None, "Failed to create '%s' driver" % args['system'] - if args['print_config']: + driverClass = createDriverClass(args["system"]) + assert driverClass != None, "Failed to find '%s' class" % args["system"] + driver = driverClass(args["ddl"]) + assert driver != None, "Failed to create '%s' driver" % args["system"] + if args["print_config"]: config = driver.makeDefaultConfig() print(driver.formatConfig(config)) print sys.exit(0) ## Load Configuration file - if args['config']: - logging.debug("Loading configuration file '%s'" % args['config']) + if args["config"]: + logging.debug("Loading configuration file '%s'" % args["config"]) cparser = SafeConfigParser() - cparser.read(os.path.realpath(args['config'].name)) - config = dict(cparser.items(args['system'])) + cparser.read(os.path.realpath(args["config"].name)) + config = dict(cparser.items(args["system"])) else: - logging.debug("Using default configuration for %s" % args['system']) + logging.debug("Using default configuration for %s" % args["system"]) defaultConfig = driver.makeDefaultConfig() config = dict(map(lambda x: (x, defaultConfig[x][1]), defaultConfig.keys())) - config['reset'] = args['reset'] - config['load'] = False - config['execute'] = False - if config['reset']: logging.info("Reseting database") + config["reset"] = args["reset"] + config["load"] = False + config["execute"] = False + if config["reset"]: + logging.info("Reseting database") driver.loadConfig(config) logging.info("Initializing TPC-C benchmark using %s" % driver) - - + ##Get a list of clientnodes from configuration file. - clients=[] - channels=[] - assert config['clients']!='' - clients=re.split(r"\s+",str(config['clients'])) - #print clients, len(clients),args['clientprocs'] + clients = [] + channels = [] + assert config["clients"] != "" + clients = re.split(r"\s+", str(config["clients"])) + # print clients, len(clients),args['clientprocs'] ##Create ssh channels to client nodes for node in clients: - cmd = 'ssh='+ node + cmd = "ssh=" + node cmd += r"//chdir=" - cmd += config['path'] - #print cmd - for i in range(args['clientprocs']): - gw=execnet.makegateway(cmd) - ch=gw.remote_exec(worker) + cmd += config["path"] + # print cmd + for i in range(args["clientprocs"]): + gw = execnet.makegateway(cmd) + ch = gw.remote_exec(worker) channels.append(ch) - + ## Create ScaleParameters - scaleParameters = scaleparameters.makeWithScaleFactor(args['warehouses'], args['scalefactor']) + scaleParameters = scaleparameters.makeWithScaleFactor( + args["warehouses"], args["scalefactor"] + ) nurand = rand.setNURand(nurand.makeForLoad()) - if args['debug']: logging.debug("Scale Parameters:\n%s" % scaleParameters) - + if args["debug"]: + logging.debug("Scale Parameters:\n%s" % scaleParameters) + ## DATA LOADER!!! load_time = None - if not args['no_load']: - load_time = startLoading(scaleParameters, args, config,channels) - #print load_time + if not args["no_load"]: + load_time = startLoading(scaleParameters, args, config, channels) + # print load_time ## IF - + ## WORKLOAD DRIVER!!! - if not args['no_execute']: - results = startExecution(scaleParameters, args, config,channels) + if not args["no_execute"]: + results = startExecution(scaleParameters, args, config, channels) assert results print(results.show(load_time)) ## IF - + ## MAIN diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/abstractdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/abstractdriver.py index d4bc137f..9c87f6f8 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/abstractdriver.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/abstractdriver.py @@ -28,6 +28,7 @@ import constants + ## ============================================== ## AbstractDriver ## ============================================== @@ -36,37 +37,42 @@ def __init__(self, name, ddl): self.name = name self.driver_name = "%sDriver" % self.name.title() self.ddl = ddl - + def __str__(self): return self.driver_name - + def makeDefaultConfig(self): """This function needs to be implemented by all sub-classes. It should return the items that need to be in your implementation's configuration file. Each item in the list is a triplet containing: ( , , ) """ - raise NotImplementedError("%s does not implement makeDefaultConfig" % (self.driver_name)) - + raise NotImplementedError( + "%s does not implement makeDefaultConfig" % (self.driver_name) + ) + def loadConfig(self, config): """Initialize the driver using the given configuration dict""" - raise NotImplementedError("%s does not implement loadConfig" % (self.driver_name)) - + raise NotImplementedError( + "%s does not implement loadConfig" % (self.driver_name) + ) + def formatConfig(self, config): """Return a formatted version of the config dict that can be used with the --config command line argument""" - ret = "# %s Configuration File\n" % (self.driver_name) + ret = "# %s Configuration File\n" % (self.driver_name) ret += "# Created %s\n" % (datetime.now()) ret += "[%s]" % self.name - + for name in config.keys(): desc, default = config[name] - if default == None: default = "" - ret += "\n\n# %s\n%-20s = %s" % (desc, name, default) - return (ret) - + if default == None: + default = "" + ret += "\n\n# %s\n%-20s = %s" % (desc, name, default) + return ret + def loadStart(self): """Optional callback to indicate to the driver that the data loading phase is about to begin.""" return None - + def loadFinish(self): """Optional callback to indicate to the driver that the data loading phase is finished.""" return None @@ -78,26 +84,28 @@ def loadFinishItem(self): def loadFinishWarehouse(self, w_id): """Optional callback to indicate to the driver that the data for the given warehouse is finished.""" return None - + def loadFinishDistrict(self, w_id, d_id): """Optional callback to indicate to the driver that the data for the given district is finished.""" return None - + def loadTuples(self, tableName, tuples): """Load a list of tuples into the target table""" - raise NotImplementedError("%s does not implement loadTuples" % (self.driver_name)) - + raise NotImplementedError( + "%s does not implement loadTuples" % (self.driver_name) + ) + def executeStart(self): """Optional callback before the execution phase starts""" return None - + def executeFinish(self): """Callback after the execution phase finishes""" return None - + def executeTransaction(self, txn, params): """Execute a transaction based on the given name""" - + if constants.TransactionTypes.DELIVERY == txn: result = self.doDelivery(params) elif constants.TransactionTypes.NEW_ORDER == txn: @@ -111,7 +119,7 @@ def executeTransaction(self, txn, params): else: assert False, "Unexpected TransactionType: " + txn return result - + def doDelivery(self, params): """Execute DELIVERY Transaction Parameters Dict: @@ -119,8 +127,10 @@ def doDelivery(self, params): o_carrier_id ol_delivery_d """ - raise NotImplementedError("%s does not implement doDelivery" % (self.driver_name)) - + raise NotImplementedError( + "%s does not implement doDelivery" % (self.driver_name) + ) + def doNewOrder(self, params): """Execute NEW_ORDER Transaction Parameters Dict: @@ -132,7 +142,9 @@ def doNewOrder(self, params): i_w_ids i_qtys """ - raise NotImplementedError("%s does not implement doNewOrder" % (self.driver_name)) + raise NotImplementedError( + "%s does not implement doNewOrder" % (self.driver_name) + ) def doOrderStatus(self, params): """Execute ORDER_STATUS Transaction @@ -142,7 +154,9 @@ def doOrderStatus(self, params): c_id c_last """ - raise NotImplementedError("%s does not implement doOrderStatus" % (self.driver_name)) + raise NotImplementedError( + "%s does not implement doOrderStatus" % (self.driver_name) + ) def doPayment(self, params): """Execute PAYMENT Transaction @@ -156,7 +170,9 @@ def doPayment(self, params): c_last h_date """ - raise NotImplementedError("%s does not implement doPayment" % (self.driver_name)) + raise NotImplementedError( + "%s does not implement doPayment" % (self.driver_name) + ) def doStockLevel(self, params): """Execute STOCK_LEVEL Transaction @@ -165,5 +181,9 @@ def doStockLevel(self, params): d_id threshold """ - raise NotImplementedError("%s does not implement doStockLevel" % (self.driver_name)) -## CLASS \ No newline at end of file + raise NotImplementedError( + "%s does not implement doStockLevel" % (self.driver_name) + ) + + +## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/couchdbdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/couchdbdriver.py index 0f1aab05..fbd5ee85 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/couchdbdriver.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/couchdbdriver.py @@ -49,124 +49,207 @@ # TPCC_SCM = { "WAREHOUSE": { - "db" : "warehouse", - "attrs" : ["W_ID", "W_NAME", "W_STREET_1", "W_STREET_2", "W_CITY", "W_STATE", "W_ZIP", "W_TAX", "W_YTD"], - "prim_key" : ["W_ID"], - "distr_key" : "W_ID", + "db": "warehouse", + "attrs": [ + "W_ID", + "W_NAME", + "W_STREET_1", + "W_STREET_2", + "W_CITY", + "W_STATE", + "W_ZIP", + "W_TAX", + "W_YTD", + ], + "prim_key": ["W_ID"], + "distr_key": "W_ID", }, - "DISTRICT": { - "db" : "district", - "attrs" : ["D_ID", "D_W_ID", "D_NAME", "D_STREET_1", "D_STREET_2", "D_CITY", "D_STATE", "D_ZIP", "D_TAX", - "D_YTD", "D_NEXT_O_ID"], - "prim_key" : ["D_W_ID", "D_ID"], - "distr_key" : "D_W_ID", + "DISTRICT": { + "db": "district", + "attrs": [ + "D_ID", + "D_W_ID", + "D_NAME", + "D_STREET_1", + "D_STREET_2", + "D_CITY", + "D_STATE", + "D_ZIP", + "D_TAX", + "D_YTD", + "D_NEXT_O_ID", + ], + "prim_key": ["D_W_ID", "D_ID"], + "distr_key": "D_W_ID", }, - "ITEM" : { - "db" : "item", - "attrs" : ["I_ID", "I_IM_ID", "I_NAME", "I_PRICE", "I_DATA"], - "prim_key" : ["I_ID"] + "ITEM": { + "db": "item", + "attrs": ["I_ID", "I_IM_ID", "I_NAME", "I_PRICE", "I_DATA"], + "prim_key": ["I_ID"], }, - "CUSTOMER": { - "db" : "customer", - "attrs" : ["C_ID", "C_D_ID", "C_W_ID", "C_FIRST", "C_MIDDLE", "C_LAST", "C_STREET_1", "C_STREET_2", - "C_CITY", "C_STATE", "C_ZIP", "C_PHONE", "C_SINCE", "C_CREDIT", "C_CREDIT_LIM", - "C_DISCOUNT", "C_BALANCE", "C_YTD_PAYMENT", "C_PAYMENT_CNT", "C_DELIVERY_CNT", "C_DATA"], - "prim_key" : ["C_W_ID" ,"C_D_ID", "C_ID"], - "distr_key" : "C_W_ID", - "indexes" : - { - "w_d_last" : { - "map" : """ + "CUSTOMER": { + "db": "customer", + "attrs": [ + "C_ID", + "C_D_ID", + "C_W_ID", + "C_FIRST", + "C_MIDDLE", + "C_LAST", + "C_STREET_1", + "C_STREET_2", + "C_CITY", + "C_STATE", + "C_ZIP", + "C_PHONE", + "C_SINCE", + "C_CREDIT", + "C_CREDIT_LIM", + "C_DISCOUNT", + "C_BALANCE", + "C_YTD_PAYMENT", + "C_PAYMENT_CNT", + "C_DELIVERY_CNT", + "C_DATA", + ], + "prim_key": ["C_W_ID", "C_D_ID", "C_ID"], + "distr_key": "C_W_ID", + "indexes": { + "w_d_last": { + "map": """ function(doc) { emit([doc.C_W_ID, doc.C_D_ID, doc.C_LAST], doc.C_FIRST); } """, - }, - } + }, + }, }, - "HISTORY" : { - "db" : "history", - "attrs" : ["H_C_ID", "H_C_D_ID", "H_C_W_ID", "H_D_ID", "H_W_ID", "H_DATE", "H_AMOUNT", "H_DATA"], - "prim_key" : [], - "distr_key" : "H_C_W_ID", + "HISTORY": { + "db": "history", + "attrs": [ + "H_C_ID", + "H_C_D_ID", + "H_C_W_ID", + "H_D_ID", + "H_W_ID", + "H_DATE", + "H_AMOUNT", + "H_DATA", + ], + "prim_key": [], + "distr_key": "H_C_W_ID", }, - "STOCK" : { - "db" : "stock", - "attrs" : ["S_I_ID", "S_W_ID", "S_QUANTITY", "S_DIST_01", "S_DIST_02", "S_DIST_03", "S_DIST_04", "S_DIST_05", - "S_DIST_06", "S_DIST_07", "S_DIST_08", "S_DIST_09", "S_DIST_10", "S_YTD", "S_ORDER_CNT", - "S_REMOTE_CNT", "S_DATA"], - "prim_key" : ["S_W_ID", "S_I_ID"], - "distr_key" : "S_W_ID", - "indexes" : - { - "w_i" : { - "map" : """ + "STOCK": { + "db": "stock", + "attrs": [ + "S_I_ID", + "S_W_ID", + "S_QUANTITY", + "S_DIST_01", + "S_DIST_02", + "S_DIST_03", + "S_DIST_04", + "S_DIST_05", + "S_DIST_06", + "S_DIST_07", + "S_DIST_08", + "S_DIST_09", + "S_DIST_10", + "S_YTD", + "S_ORDER_CNT", + "S_REMOTE_CNT", + "S_DATA", + ], + "prim_key": ["S_W_ID", "S_I_ID"], + "distr_key": "S_W_ID", + "indexes": { + "w_i": { + "map": """ function(doc) { emit([doc.S_W_ID, doc.S_I_ID], doc.S_QUANTITY); } """, - }, - } + }, + }, }, - "ORDERS" : { - "db" : "orders", - "attrs" : ["O_ID", "O_C_ID", "O_D_ID", "O_W_ID", "O_ENTRY_D", "O_CARRIER_ID", "O_OL_CNT", "O_ALL_LOCAL"], - "prim_key" : ["O_W_ID", "O_D_ID", "O_ID"], - "distr_key" : "O_W_ID", - "indexes" : - { - "w_d_c_o" : { - "map" : """ + "ORDERS": { + "db": "orders", + "attrs": [ + "O_ID", + "O_C_ID", + "O_D_ID", + "O_W_ID", + "O_ENTRY_D", + "O_CARRIER_ID", + "O_OL_CNT", + "O_ALL_LOCAL", + ], + "prim_key": ["O_W_ID", "O_D_ID", "O_ID"], + "distr_key": "O_W_ID", + "indexes": { + "w_d_c_o": { + "map": """ function(doc) { emit([doc.O_W_ID, doc.O_D_ID, doc.O_C_ID, doc.O_ID], null); } """, - }, - } + }, + }, }, "NEW_ORDER": { - "db" : "new_order", - "attrs" : ["NO_O_ID", "NO_D_ID", "NO_W_ID"], - "prim_key" : ["NO_D_ID", "NO_W_ID", "NO_O_ID"], - "distr_key" : "NO_W_ID", + "db": "new_order", + "attrs": ["NO_O_ID", "NO_D_ID", "NO_W_ID"], + "prim_key": ["NO_D_ID", "NO_W_ID", "NO_O_ID"], + "distr_key": "NO_W_ID", }, - "ORDER_LINE":{ - "db" : "order_line", - "attrs" : ["OL_O_ID", "OL_D_ID", "OL_W_ID", "OL_NUMBER", "OL_I_ID", "OL_SUPPLY_W_ID", "OL_DELIVERY_D", - "OL_QUANTITY", "OL_AMOUNT", "OL_DIST_INFO"], + "ORDER_LINE": { + "db": "order_line", + "attrs": [ + "OL_O_ID", + "OL_D_ID", + "OL_W_ID", + "OL_NUMBER", + "OL_I_ID", + "OL_SUPPLY_W_ID", + "OL_DELIVERY_D", + "OL_QUANTITY", + "OL_AMOUNT", + "OL_DIST_INFO", + ], "prim_key": ["OL_W_ID", "OL_D_ID", "OL_O_ID", "OL_NUMBER"], - "distr_key" : "OL_W_ID", - "indexes" : - { - "o_d_w" : { - "map" : """ + "distr_key": "OL_W_ID", + "indexes": { + "o_d_w": { + "map": """ function(doc) { emit([doc.OL_O_ID, doc.OL_D_ID, doc.OL_W_ID], doc.OL_AMOUNT); } """, - "reduce": """ + "reduce": """ function(keys, values, rereduce) { return sum(values); } """, - }, - "o_d_w_i" : { - "map" : """ + }, + "o_d_w_i": { + "map": """ function(doc) { emit([doc.OL_O_ID, doc.OL_D_ID, doc.OL_W_ID], doc.OL_I_ID); } """, - }, - } + }, + }, }, } + def db_from_table(table_name): """ Converts the name of the table to the corresponding CouchDB database name. Note, that CouchDB doesn't like CAPITAL database names. """ - return TPCC_SCM[table_name]['db'] + return TPCC_SCM[table_name]["db"] + def gen_pk_doc(table_name, doc): """ @@ -177,13 +260,14 @@ def gen_pk_doc(table_name, doc): It is usually recommended to generate an id on the client side. """ table_schema = TPCC_SCM[table_name] - if len(table_schema['prim_key']): - pk = '_'.join([str(doc[attr]) for attr in table_schema['prim_key']]) + if len(table_schema["prim_key"]): + pk = "_".join([str(doc[attr]) for attr in table_schema["prim_key"]]) else: pk = uuid4().hex return pk + def touch_view(db, view_name): """ Touches the 'view_name' view from the given db object. @@ -192,10 +276,15 @@ def touch_view(db, view_name): create it on the first query. We don't want that, since that would make things very slow during the actual transaction processing! """ - logging.debug("HACK: Fetching view '%s' from '%s' with 'limit = 1'" % (view_name, str(db))) + logging.debug( + "HACK: Fetching view '%s' from '%s' with 'limit = 1'" % (view_name, str(db)) + ) # the result is unimportant here, just use limit=1 - db.view('tpcc/%s' % view_name, limit = 1).rows - logging.debug("HACK: Fetched view '%s' from '%s' with 'limit = 1'" % (view_name, str(db))) + db.view("tpcc/%s" % view_name, limit=1).rows + logging.debug( + "HACK: Fetched view '%s' from '%s' with 'limit = 1'" % (view_name, str(db)) + ) + class TouchThread(threading.Thread): """ @@ -207,6 +296,7 @@ class TouchThread(threading.Thread): So, the thread just executes 'touch_view' function and then quits. """ + def __init__(self, *args): self._target = touch_view self._args = args @@ -215,18 +305,22 @@ def __init__(self, *args): def run(self): self._target(*self._args) + ## ============================================== ## CouchdbDriver ## ============================================== class CouchdbDriver(AbstractDriver): DEFAULT_CONFIG = { - "node_urls": ("CouchDB URL:", '["http://localhost:5984"]'), # usual "out-of-the-box" value + "node_urls": ( + "CouchDB URL:", + '["http://localhost:5984"]', + ), # usual "out-of-the-box" value } def __init__(self, ddl): super(CouchdbDriver, self).__init__("couchdb", ddl) - self.servers = [] # list of shards (couchdb server objects) - self.dbs = None # dict: 'db_name' -> (list of db_obj (shards)) + self.servers = [] # list of shards (couchdb server objects) + self.dbs = None # dict: 'db_name' -> (list of db_obj (shards)) ## ---------------------------------------------- ## makeDefaultConfig @@ -239,14 +333,17 @@ def makeDefaultConfig(self): ## ---------------------------------------------- def loadConfig(self, config): for key in CouchdbDriver.DEFAULT_CONFIG.keys(): - assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) + assert key in config, "Missing parameter '%s' in %s configuration" % ( + key, + self.name, + ) # open servers for srv_name in eval(config["node_urls"]): logging.debug("Got a CouchDB node from config: '%s'" % srv_name) # we use delayed commits here since we don't care much about durability # note, that couchdb would commit the data once per several seconds anyway - self.servers.append(couchdb.Server(url = srv_name, full_commit = False)) + self.servers.append(couchdb.Server(url=srv_name, full_commit=False)) db_names = [db_from_table(table) for table in TPCC_SCM.keys()] @@ -255,19 +352,25 @@ def loadConfig(self, config): for db in db_names: for srv in self.servers: if db in srv: - logging.debug("Deleting database '%s' on server '%s'" % (db, str(srv))) + logging.debug( + "Deleting database '%s' on server '%s'" % (db, str(srv)) + ) srv.delete(db) # creating databases self.dbs = dict() for db in db_names: - sdb = [] # list of shards for the db + sdb = [] # list of shards for the db for srv in self.servers: if not db in srv: - logging.debug("Creating database '%s' on server '%s'" % (db, str(srv))) + logging.debug( + "Creating database '%s' on server '%s'" % (db, str(srv)) + ) sdb.append(srv.create(db)) else: - logging.debug("Database exists: '%s', server: '%s'" % (db, str(srv))) + logging.debug( + "Database exists: '%s', server: '%s'" % (db, str(srv)) + ) sdb.append(srv[db]) self.dbs[db] = sdb @@ -300,13 +403,15 @@ def tuples_to_docs(self, table_name, tuples): docs = [list() for s in self.servers] tuple_len = len(tuples[0]) - assert(tuple_len == len(table_schema['attrs'])), "Number of attributes and the tuple length differ: %s" % table_name + assert tuple_len == len(table_schema["attrs"]), ( + "Number of attributes and the tuple length differ: %s" % table_name + ) for tup in tuples: doc = dict() # generate the doc as a simple dict - for i, attr in enumerate(table_schema['attrs']): + for i, attr in enumerate(table_schema["attrs"]): doc[attr] = tup[i] # determine the shard number we want to put the doc into. @@ -324,7 +429,7 @@ def tuples_to_docs(self, table_name, tuples): shard = -1 # emulate primary key with "id" or generate a random one - doc['_id'] = gen_pk_doc(table_name, doc) + doc["_id"] = gen_pk_doc(table_name, doc) # put the doc to the proper list. # '-1' means 'replicate to all' @@ -340,7 +445,8 @@ def tuples_to_docs(self, table_name, tuples): ## loadTuples ## ---------------------------------------------- def loadTuples(self, tableName, tuples): - if len(tuples) == 0: return + if len(tuples) == 0: + return # create docs for tuples docs = self.tuples_to_docs(tableName, tuples) @@ -349,7 +455,10 @@ def loadTuples(self, tableName, tuples): # load all documents in bulk on every node for srv_num, srv in enumerate(self.servers): if len(docs[srv_num]): - logging.debug("Loading tuples from the table '%s' into database '%s' on server '%s'" % (tableName, db_name, str(srv))) + logging.debug( + "Loading tuples from the table '%s' into database '%s' on server '%s'" + % (tableName, db_name, str(srv)) + ) # should we check the result here? we're assuming a fresh load. self.dbs[db_name][srv_num].update(docs[srv_num]) @@ -365,19 +474,22 @@ def loadFinish(self): """ view_touch_jobs = [] for table in TPCC_SCM.keys(): - if 'indexes' in TPCC_SCM[table]: + if "indexes" in TPCC_SCM[table]: for srv_num, srv in enumerate(self.servers): # load the design doc: _design/tpcc try: - logging.debug("Creating indexes for '%s' on server '%s'" % (table, str(srv))) + logging.debug( + "Creating indexes for '%s' on server '%s'" + % (table, str(srv)) + ) cdb = self.dbs[db_from_table(table)][srv_num] - design_doc = {'views' : TPCC_SCM[table]['indexes']} - cdb['_design/tpcc'] = design_doc + design_doc = {"views": TPCC_SCM[table]["indexes"]} + cdb["_design/tpcc"] = design_doc except couchdb.http.ResourceConflict: # happens if we have multiple loaders. This is okay. The design doc is still the same. pass finally: - for view_name in TPCC_SCM[table]['indexes'].keys(): + for view_name in TPCC_SCM[table]["indexes"].keys(): view_touch_jobs.append((cdb, view_name)) # we want actually to initialize views in parallel on all shard nodes @@ -407,12 +519,28 @@ def doDelivery(self, params): for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1): while True: # fetch any 'NEW_ORDER' doc ('0' as the 'NO_O_ID') - newOrder = self.dbs[db_from_table('NEW_ORDER')][self.shard_from_id(w_id)].view('_all_docs', limit = 1, - include_docs = 'true', startkey = gen_pk_doc('NEW_ORDER', {'NO_D_ID': d_id, 'NO_W_ID': w_id, 'NO_O_ID' : 0})).rows + newOrder = ( + self.dbs[db_from_table("NEW_ORDER")][self.shard_from_id(w_id)] + .view( + "_all_docs", + limit=1, + include_docs="true", + startkey=gen_pk_doc( + "NEW_ORDER", + {"NO_D_ID": d_id, "NO_W_ID": w_id, "NO_O_ID": 0}, + ), + ) + .rows + ) # it seems that we might fetch a deleted doc in case there are no more. Nice... - if newOrder[0]['value'].has_key('deleted') and newOrder[0]['value']['deleted'] == True: - logging.debug("No documents: _all_docs returned a deleted one. Skipping...") + if ( + newOrder[0]["value"].has_key("deleted") + and newOrder[0]["value"]["deleted"] == True + ): + logging.debug( + "No documents: _all_docs returned a deleted one. Skipping..." + ) newOrder = [] if len(newOrder) == 0: @@ -422,16 +550,22 @@ def doDelivery(self, params): newOrder = newOrder[0].doc try: - self.dbs[db_from_table('NEW_ORDER')][self.shard_from_id(w_id)].delete(newOrder) - no_o_ids.append((d_id, newOrder['NO_O_ID'])) + self.dbs[db_from_table("NEW_ORDER")][ + self.shard_from_id(w_id) + ].delete(newOrder) + no_o_ids.append((d_id, newOrder["NO_O_ID"])) break except couchdb.http.ResourceNotFound: # in case somebody got this order first, try to fetch another one - logging.debug('Pessimistic concurrency control: Delete failed: Restarting...') + logging.debug( + "Pessimistic concurrency control: Delete failed: Restarting..." + ) pass except couchdb.http.ResourceConflict: # in case somebody got this order first, try to fetch another one - logging.debug('Pessimistic concurrency control: Delete failed: Restarting...') + logging.debug( + "Pessimistic concurrency control: Delete failed: Restarting..." + ) pass if len(newOrder) == 0: @@ -441,61 +575,94 @@ def doDelivery(self, params): # Now we're "isolated" from concurrent transactions... # We're trying to fetch all info using as least requests as possible - order_keys = [gen_pk_doc('ORDERS', {'O_ID': no_o_id, 'O_W_ID': w_id, 'O_D_ID': d_id}) for d_id, no_o_id in no_o_ids] - order_docs = self.dbs[db_from_table('ORDERS')][self.shard_from_id(w_id)].view('_all_docs', - include_docs = 'true', - keys = order_keys).rows + order_keys = [ + gen_pk_doc("ORDERS", {"O_ID": no_o_id, "O_W_ID": w_id, "O_D_ID": d_id}) + for d_id, no_o_id in no_o_ids + ] + order_docs = ( + self.dbs[db_from_table("ORDERS")][self.shard_from_id(w_id)] + .view("_all_docs", include_docs="true", keys=order_keys) + .rows + ) order_docs = [od.doc for od in order_docs] # use the view for the sum aggregate - ol_totals = self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].view('tpcc/o_d_w', group = 'true', - keys = [[no_o_id, d_id, w_id] for d_id, no_o_id in no_o_ids]).rows + ol_totals = ( + self.dbs[db_from_table("ORDER_LINE")][self.shard_from_id(w_id)] + .view( + "tpcc/o_d_w", + group="true", + keys=[[no_o_id, d_id, w_id] for d_id, no_o_id in no_o_ids], + ) + .rows + ) # put the fetched information together for every client c_ids = [] for i in range(len(no_o_ids)): # find the total for the current (order, district, warehouse) # is there some way to find stuff in a list fast? - ol_total = filter(lambda x: x.key == [no_o_ids[i][1], no_o_ids[i][0], w_id], ol_totals)[0].value + ol_total = filter( + lambda x: x.key == [no_o_ids[i][1], no_o_ids[i][0], w_id], ol_totals + )[0].value # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure # them out # If there are no order lines, SUM returns null. There should always be order lines. - assert ol_total != None, "ol_total is NULL: there are no order lines. This should not happen" + assert ( + ol_total != None + ), "ol_total is NULL: there are no order lines. This should not happen" assert ol_total > 0.0 - c_ids.append((order_docs[i]['O_C_ID'], no_o_ids[i][0], ol_total)) + c_ids.append((order_docs[i]["O_C_ID"], no_o_ids[i][0], ol_total)) # this should be safe. no conflicts... for order_doc in order_docs: - order_doc['O_CARRIER_ID'] = o_carrier_id - self.dbs[db_from_table('ORDERS')][self.shard_from_id(w_id)].update(order_docs) + order_doc["O_CARRIER_ID"] = o_carrier_id + self.dbs[db_from_table("ORDERS")][self.shard_from_id(w_id)].update(order_docs) # ditto... # we must do the second retrieval from ORDER_LINES, since now we need docs, not aggregates - order_lines = self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].view('tpcc/o_d_w', - keys = [[no_o_id, d_id, w_id] for d_id, no_o_id in no_o_ids], - reduce = 'false', - include_docs = 'true').rows + order_lines = ( + self.dbs[db_from_table("ORDER_LINE")][self.shard_from_id(w_id)] + .view( + "tpcc/o_d_w", + keys=[[no_o_id, d_id, w_id] for d_id, no_o_id in no_o_ids], + reduce="false", + include_docs="true", + ) + .rows + ) order_lines = [r.doc for r in order_lines] for ol in order_lines: - ol['OL_DELIVERY_D'] = ol_delivery_d + ol["OL_DELIVERY_D"] = ol_delivery_d - self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].update(order_lines) + self.dbs[db_from_table("ORDER_LINE")][self.shard_from_id(w_id)].update( + order_lines + ) # again, updating clients may introduce conflicts. another bottleneck.... for c_id, d_id, ol_total in c_ids: while True: - customer_info = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].get(gen_pk_doc('CUSTOMER', - {'C_W_ID': w_id, 'C_D_ID': d_id, 'C_ID': c_id})) - customer_info['C_BALANCE'] += ol_total + customer_info = self.dbs[db_from_table("CUSTOMER")][ + self.shard_from_id(w_id) + ].get( + gen_pk_doc( + "CUSTOMER", {"C_W_ID": w_id, "C_D_ID": d_id, "C_ID": c_id} + ) + ) + customer_info["C_BALANCE"] += ol_total try: - self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].save(customer_info) + self.dbs[db_from_table("CUSTOMER")][self.shard_from_id(w_id)].save( + customer_info + ) break except couchdb.http.ResourceConflict: # in case somebody updated the customer first, try again with the new revision - logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + logging.debug( + "Pessimistic concurrency control: Update failed: Restarting..." + ) pass result = no_o_ids @@ -522,9 +689,11 @@ def doNewOrder(self, params): items = [] # retrieve and store info about all the items - item_data = self.dbs[db_from_table('ITEM')][self.shard_from_id(w_id)].view('_all_docs', - include_docs = 'true', - keys = [str(i) for i in i_ids]).rows + item_data = ( + self.dbs[db_from_table("ITEM")][self.shard_from_id(w_id)] + .view("_all_docs", include_docs="true", keys=[str(i) for i in i_ids]) + .rows + ) for i in range(len(i_ids)): ## Determine if this is an all local order or not @@ -540,34 +709,42 @@ def doNewOrder(self, params): ## TODO Abort here! return - items.append((doc['I_PRICE'], doc['I_NAME'], doc['I_DATA'])) + items.append((doc["I_PRICE"], doc["I_NAME"], doc["I_DATA"])) assert len(items) == len(i_ids) ## ---------------- ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER ## ---------------- - doc = self.dbs[db_from_table('WAREHOUSE')][self.shard_from_id(w_id)].get(str(w_id)) - w_tax = doc['W_TAX'] + doc = self.dbs[db_from_table("WAREHOUSE")][self.shard_from_id(w_id)].get( + str(w_id) + ) + w_tax = doc["W_TAX"] # conflict is possible. this is a bottleneck... while True: - district_info = self.dbs[db_from_table('DISTRICT')][self.shard_from_id(w_id)].get(gen_pk_doc('DISTRICT', - {'D_ID': d_id, 'D_W_ID': w_id})) - d_tax = district_info['D_TAX'] - d_next_o_id = district_info['D_NEXT_O_ID'] + district_info = self.dbs[db_from_table("DISTRICT")][ + self.shard_from_id(w_id) + ].get(gen_pk_doc("DISTRICT", {"D_ID": d_id, "D_W_ID": w_id})) + d_tax = district_info["D_TAX"] + d_next_o_id = district_info["D_NEXT_O_ID"] - district_info['D_NEXT_O_ID'] += 1 + district_info["D_NEXT_O_ID"] += 1 try: - self.dbs[db_from_table('DISTRICT')][self.shard_from_id(w_id)].save(district_info) + self.dbs[db_from_table("DISTRICT")][self.shard_from_id(w_id)].save( + district_info + ) break except couchdb.http.ResourceConflict: # want to get a unique order id! - logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + logging.debug( + "Pessimistic concurrency control: Update failed: Restarting..." + ) pass - customer_info = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].get(gen_pk_doc('CUSTOMER', - {'C_W_ID': w_id, 'C_D_ID': d_id, 'C_ID': c_id})) - c_discount = customer_info['C_DISCOUNT'] + customer_info = self.dbs[db_from_table("CUSTOMER")][ + self.shard_from_id(w_id) + ].get(gen_pk_doc("CUSTOMER", {"C_W_ID": w_id, "C_D_ID": d_id, "C_ID": c_id})) + c_discount = customer_info["C_DISCOUNT"] ol_cnt = len(i_ids) o_carrier_id = constants.NULL_CARRIER_ID @@ -591,14 +768,19 @@ def doNewOrder(self, params): # we have potential conflict for every stock while True: - stockInfo = self.dbs[db_from_table('STOCK')][self.shard_from_id(ol_supply_w_id)].get(gen_pk_doc('STOCK', - {'S_I_ID': ol_i_id, 'S_W_ID': ol_supply_w_id})) - s_quantity = stockInfo['S_QUANTITY'] - s_ytd = stockInfo['S_YTD'] - s_order_cnt = stockInfo['S_ORDER_CNT'] - s_remote_cnt = stockInfo['S_REMOTE_CNT'] - s_data = stockInfo['S_DATA'] - s_dist_xx = stockInfo['S_DIST_%02d' % d_id] # Fetches data from the s_dist_[d_id] column + stockInfo = self.dbs[db_from_table("STOCK")][ + self.shard_from_id(ol_supply_w_id) + ].get( + gen_pk_doc("STOCK", {"S_I_ID": ol_i_id, "S_W_ID": ol_supply_w_id}) + ) + s_quantity = stockInfo["S_QUANTITY"] + s_ytd = stockInfo["S_YTD"] + s_order_cnt = stockInfo["S_ORDER_CNT"] + s_remote_cnt = stockInfo["S_REMOTE_CNT"] + s_data = stockInfo["S_DATA"] + s_dist_xx = stockInfo[ + "S_DIST_%02d" % d_id + ] # Fetches data from the s_dist_[d_id] column ## Update stock s_ytd += ol_quantity @@ -608,27 +790,34 @@ def doNewOrder(self, params): s_quantity = s_quantity + 91 - ol_quantity s_order_cnt += 1 - if ol_supply_w_id != w_id: s_remote_cnt += 1 + if ol_supply_w_id != w_id: + s_remote_cnt += 1 # update stock - stockInfo['S_QUANTITY'] = s_quantity - stockInfo['S_YTD'] = s_ytd - stockInfo['S_ORDER_CNT'] = s_order_cnt - stockInfo['S_REMOTE_CNT'] = s_remote_cnt + stockInfo["S_QUANTITY"] = s_quantity + stockInfo["S_YTD"] = s_ytd + stockInfo["S_ORDER_CNT"] = s_order_cnt + stockInfo["S_REMOTE_CNT"] = s_remote_cnt try: - self.dbs[db_from_table('STOCK')][self.shard_from_id(ol_supply_w_id)].save(stockInfo) + self.dbs[db_from_table("STOCK")][ + self.shard_from_id(ol_supply_w_id) + ].save(stockInfo) break except couchdb.http.ResourceConflict: # if somebody had reserved the stock before us, repeat. - logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + logging.debug( + "Pessimistic concurrency control: Update failed: Restarting..." + ) pass - - if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: - brand_generic = 'B' + if ( + i_data.find(constants.ORIGINAL_STRING) != -1 + and s_data.find(constants.ORIGINAL_STRING) != -1 + ): + brand_generic = "B" else: - brand_generic = 'G' + brand_generic = "G" ## Transaction profile states to use "ol_quantity * i_price" ol_amount = ol_quantity * i_price @@ -636,9 +825,24 @@ def doNewOrder(self, params): # don't insert the order line right now # we'll do it in bulk later - order_line_row = dict(zip(TPCC_SCM['ORDER_LINE']['attrs'], [d_next_o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, - o_entry_d, ol_quantity, ol_amount, s_dist_xx])) - order_line_row['_id'] = gen_pk_doc('ORDER_LINE', order_line_row) + order_line_row = dict( + zip( + TPCC_SCM["ORDER_LINE"]["attrs"], + [ + d_next_o_id, + d_id, + w_id, + ol_number, + ol_i_id, + ol_supply_w_id, + o_entry_d, + ol_quantity, + ol_amount, + s_dist_xx, + ], + ) + ) + order_line_row["_id"] = gen_pk_doc("ORDER_LINE", order_line_row) order_line_docs.append(order_line_row) ## Add the info to be returned @@ -648,23 +852,49 @@ def doNewOrder(self, params): ## ---------------- ## Insert Order Information ## ---------------- - self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].update(order_line_docs) - - orders_row = dict(zip(TPCC_SCM['ORDERS']['attrs'], [d_next_o_id, c_id, d_id, w_id, o_entry_d, o_carrier_id, ol_cnt, all_local])) - orders_row['_id'] = gen_pk_doc('ORDERS', orders_row) - self.dbs[db_from_table('ORDERS')][self.shard_from_id(w_id)].save(orders_row) - - new_order_row = dict(zip(TPCC_SCM['NEW_ORDER']['attrs'], [d_next_o_id, d_id, w_id])) - new_order_row['_id'] = gen_pk_doc('NEW_ORDER', new_order_row) - self.dbs[db_from_table('NEW_ORDER')][self.shard_from_id(w_id)].save(new_order_row) + self.dbs[db_from_table("ORDER_LINE")][self.shard_from_id(w_id)].update( + order_line_docs + ) + + orders_row = dict( + zip( + TPCC_SCM["ORDERS"]["attrs"], + [ + d_next_o_id, + c_id, + d_id, + w_id, + o_entry_d, + o_carrier_id, + ol_cnt, + all_local, + ], + ) + ) + orders_row["_id"] = gen_pk_doc("ORDERS", orders_row) + self.dbs[db_from_table("ORDERS")][self.shard_from_id(w_id)].save(orders_row) + + new_order_row = dict( + zip(TPCC_SCM["NEW_ORDER"]["attrs"], [d_next_o_id, d_id, w_id]) + ) + new_order_row["_id"] = gen_pk_doc("NEW_ORDER", new_order_row) + self.dbs[db_from_table("NEW_ORDER")][self.shard_from_id(w_id)].save( + new_order_row + ) ## Adjust the total for the discount total *= (1 - c_discount) * (1 + w_tax + d_tax) ## Pack up values the client is missing (see TPC-C 2.4.3.5) - misc = [ (w_tax, d_tax, d_next_o_id, total) ] - customer_info = [ (customer_info['C_DISCOUNT'], customer_info['C_LAST'], customer_info['C_CREDIT']) ] - return [ customer_info, misc, item_data ] + misc = [(w_tax, d_tax, d_next_o_id, total)] + customer_info = [ + ( + customer_info["C_DISCOUNT"], + customer_info["C_LAST"], + customer_info["C_CREDIT"], + ) + ] + return [customer_info, misc, item_data] ## ---------------------------------------------- ## doOrderStatus @@ -679,49 +909,81 @@ def doOrderStatus(self, params): assert d_id, pformat(params) if c_id != None: - customer = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].get(gen_pk_doc('CUSTOMER', - {'C_W_ID': w_id, 'C_D_ID': d_id, 'C_ID': c_id})) + customer = self.dbs[db_from_table("CUSTOMER")][ + self.shard_from_id(w_id) + ].get( + gen_pk_doc("CUSTOMER", {"C_W_ID": w_id, "C_D_ID": d_id, "C_ID": c_id}) + ) else: # Get the midpoint customer's id - all_customers = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].view('tpcc/w_d_last', - key = [w_id, d_id, c_last], - reduce = 'false').rows - all_customers.sort(lambda x, y: cmp(x['value'], y['value'])) + all_customers = ( + self.dbs[db_from_table("CUSTOMER")][self.shard_from_id(w_id)] + .view("tpcc/w_d_last", key=[w_id, d_id, c_last], reduce="false") + .rows + ) + all_customers.sort(lambda x, y: cmp(x["value"], y["value"])) assert len(all_customers) > 0 namecnt = len(all_customers) index = (namecnt - 1) / 2 customer = all_customers[index] - customer = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].get(customer['id']) - c_id = customer['C_ID'] + customer = self.dbs[db_from_table("CUSTOMER")][ + self.shard_from_id(w_id) + ].get(customer["id"]) + c_id = customer["C_ID"] assert len(customer) > 0 assert c_id != None # get the last order from the customer - order = self.dbs[db_from_table('ORDERS')][self.shard_from_id(w_id)].view('tpcc/w_d_c_o', - limit = 1, - include_docs = 'true', - startkey = [w_id, d_id, c_id, 'a'], # 'a' is just to give all numbers - endkey = [w_id, d_id, c_id, -1], - descending = 'true', - reduce = 'false').rows + order = ( + self.dbs[db_from_table("ORDERS")][self.shard_from_id(w_id)] + .view( + "tpcc/w_d_c_o", + limit=1, + include_docs="true", + startkey=[w_id, d_id, c_id, "a"], # 'a' is just to give all numbers + endkey=[w_id, d_id, c_id, -1], + descending="true", + reduce="false", + ) + .rows + ) if len(order) > 0: order = order[0].doc - orderLines = self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].view('tpcc/o_d_w', - key = [order['O_ID'], d_id, w_id], - reduce = 'false', - include_docs = 'true').rows - - orderLines = [(o.doc['OL_SUPPLY_W_ID'], o.doc['OL_I_ID'], - o.doc['OL_QUANTITY'], o.doc['OL_AMOUNT'], - o.doc['OL_DELIVERY_D']) for o in orderLines] + orderLines = ( + self.dbs[db_from_table("ORDER_LINE")][self.shard_from_id(w_id)] + .view( + "tpcc/o_d_w", + key=[order["O_ID"], d_id, w_id], + reduce="false", + include_docs="true", + ) + .rows + ) + + orderLines = [ + ( + o.doc["OL_SUPPLY_W_ID"], + o.doc["OL_I_ID"], + o.doc["OL_QUANTITY"], + o.doc["OL_AMOUNT"], + o.doc["OL_DELIVERY_D"], + ) + for o in orderLines + ] else: orderLines = [] - customer = (customer['C_ID'], customer['C_FIRST'], customer['C_MIDDLE'], customer['C_LAST'], customer['C_BALANCE']) - order = (order['O_ID'], order['O_CARRIER_ID'], order['O_ENTRY_D']) - return [ customer, order, orderLines ] + customer = ( + customer["C_ID"], + customer["C_FIRST"], + customer["C_MIDDLE"], + customer["C_LAST"], + customer["C_BALANCE"], + ) + order = (order["O_ID"], order["O_CARRIER_ID"], order["O_ENTRY_D"]) + return [customer, order, orderLines] ## ---------------------------------------------- ## doPayment @@ -737,80 +999,109 @@ def doPayment(self, params): h_date = str(params["h_date"]) if c_id != None: - cus_doc_id = gen_pk_doc('CUSTOMER', {'C_W_ID': w_id, 'C_D_ID': d_id, 'C_ID': c_id}) + cus_doc_id = gen_pk_doc( + "CUSTOMER", {"C_W_ID": w_id, "C_D_ID": d_id, "C_ID": c_id} + ) else: # Get the midpoint customer's id - all_customers = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].view('tpcc/w_d_last', - key = [w_id, d_id, c_last], - reduce = 'false').rows - all_customers.sort(lambda x, y: cmp(x['value'], y['value'])) + all_customers = ( + self.dbs[db_from_table("CUSTOMER")][self.shard_from_id(w_id)] + .view("tpcc/w_d_last", key=[w_id, d_id, c_last], reduce="false") + .rows + ) + all_customers.sort(lambda x, y: cmp(x["value"], y["value"])) assert len(all_customers) > 0 namecnt = len(all_customers) index = (namecnt - 1) / 2 customer = all_customers[index] - cus_doc_id = customer['id'] + cus_doc_id = customer["id"] # try to update the customer record. conflicts expected. while True: - customer = self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].get(cus_doc_id) + customer = self.dbs[db_from_table("CUSTOMER")][ + self.shard_from_id(w_id) + ].get(cus_doc_id) assert len(customer) > 0 - c_id = customer['C_ID'] + c_id = customer["C_ID"] - c_balance = customer['C_BALANCE'] - h_amount - c_ytd_payment = customer['C_YTD_PAYMENT'] + h_amount - c_payment_cnt = customer['C_PAYMENT_CNT'] + 1 + c_balance = customer["C_BALANCE"] - h_amount + c_ytd_payment = customer["C_YTD_PAYMENT"] + h_amount + c_payment_cnt = customer["C_PAYMENT_CNT"] + 1 # Customer Credit Information try: - if customer['C_CREDIT'] == constants.BAD_CREDIT: - c_data = customer['C_DATA'] - newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) - c_data = (newData + "|" + c_data) - if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] - customer['C_DATA'] = c_data - - customer['C_BALANCE'] = c_balance - customer['C_YTD_PAYMENT'] = c_ytd_payment - customer['C_PAYMENT_CNT'] = c_payment_cnt - self.dbs[db_from_table('CUSTOMER')][self.shard_from_id(w_id)].save(customer) + if customer["C_CREDIT"] == constants.BAD_CREDIT: + c_data = customer["C_DATA"] + newData = " ".join( + map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount]) + ) + c_data = newData + "|" + c_data + if len(c_data) > constants.MAX_C_DATA: + c_data = c_data[: constants.MAX_C_DATA] + customer["C_DATA"] = c_data + + customer["C_BALANCE"] = c_balance + customer["C_YTD_PAYMENT"] = c_ytd_payment + customer["C_PAYMENT_CNT"] = c_payment_cnt + self.dbs[db_from_table("CUSTOMER")][self.shard_from_id(w_id)].save( + customer + ) break except couchdb.http.ResourceConflict: - logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + logging.debug( + "Pessimistic concurrency control: Update failed: Restarting..." + ) pass # conflicts when updating warehouse record and... while True: - warehouse = self.dbs[db_from_table('WAREHOUSE')][self.shard_from_id(w_id)].get(str(w_id)) - warehouse['W_YTD'] += h_amount + warehouse = self.dbs[db_from_table("WAREHOUSE")][ + self.shard_from_id(w_id) + ].get(str(w_id)) + warehouse["W_YTD"] += h_amount try: - self.dbs[db_from_table('WAREHOUSE')][self.shard_from_id(w_id)].save(warehouse) + self.dbs[db_from_table("WAREHOUSE")][self.shard_from_id(w_id)].save( + warehouse + ) break except couchdb.http.ResourceConflict: # pessimistic concurrency control... - logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + logging.debug( + "Pessimistic concurrency control: Update failed: Restarting..." + ) pass # the district record while True: - district = self.dbs[db_from_table('DISTRICT')][self.shard_from_id(w_id)].get(gen_pk_doc('DISTRICT', - {'D_ID': d_id, 'D_W_ID': w_id})) - district['D_YTD'] += h_amount + district = self.dbs[db_from_table("DISTRICT")][ + self.shard_from_id(w_id) + ].get(gen_pk_doc("DISTRICT", {"D_ID": d_id, "D_W_ID": w_id})) + district["D_YTD"] += h_amount try: - self.dbs[db_from_table('DISTRICT')][self.shard_from_id(w_id)].save(district) + self.dbs[db_from_table("DISTRICT")][self.shard_from_id(w_id)].save( + district + ) break except couchdb.http.ResourceConflict: # pessimistic concurrency control... - logging.debug('Pessimistic concurrency control: Update failed: Restarting...') + logging.debug( + "Pessimistic concurrency control: Update failed: Restarting..." + ) pass # Concatenate w_name, four spaces, d_name - h_data = "%s %s" % (warehouse['W_NAME'], district['D_NAME']) + h_data = "%s %s" % (warehouse["W_NAME"], district["D_NAME"]) # Create the history record - hist = dict(zip(TPCC_SCM['HISTORY']['attrs'],[c_id, c_d_id, c_w_id, d_id, w_id, h_date, h_amount, h_data])) - self.dbs[db_from_table('HISTORY')][self.shard_from_id(c_w_id)].save(hist) + hist = dict( + zip( + TPCC_SCM["HISTORY"]["attrs"], + [c_id, c_d_id, c_w_id, d_id, w_id, h_date, h_amount, h_data], + ) + ) + self.dbs[db_from_table("HISTORY")][self.shard_from_id(c_w_id)].save(hist) # TPC-C 2.5.3.3: Must display the following fields: # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, @@ -820,17 +1111,45 @@ def doPayment(self, params): # H_AMOUNT, and H_DATE. # Hand back all the warehouse, district, and customer data - warehouse = (warehouse['W_NAME'], warehouse['W_STREET_1'], warehouse['W_STREET_2'], - warehouse['W_CITY'], warehouse['W_STATE'], warehouse['W_ZIP']) - district = (district['D_NAME'], district['D_STREET_1'], district['D_STREET_2'], - district['D_CITY'], district['D_STATE'], district['D_ZIP']) - customer = (customer['C_ID'], customer['C_FIRST'], customer['C_MIDDLE'], customer['C_LAST'], customer['C_STREET_1'], - customer['C_STREET_2'], customer['C_CITY'], customer['C_STATE'], customer['C_ZIP'], customer['C_PHONE'], - customer['C_SINCE'], customer['C_CREDIT'], customer['C_CREDIT_LIM'], customer['C_DISCOUNT'], - customer['C_BALANCE'], customer['C_YTD_PAYMENT'], customer['C_PAYMENT_CNT'], customer['C_DATA']) + warehouse = ( + warehouse["W_NAME"], + warehouse["W_STREET_1"], + warehouse["W_STREET_2"], + warehouse["W_CITY"], + warehouse["W_STATE"], + warehouse["W_ZIP"], + ) + district = ( + district["D_NAME"], + district["D_STREET_1"], + district["D_STREET_2"], + district["D_CITY"], + district["D_STATE"], + district["D_ZIP"], + ) + customer = ( + customer["C_ID"], + customer["C_FIRST"], + customer["C_MIDDLE"], + customer["C_LAST"], + customer["C_STREET_1"], + customer["C_STREET_2"], + customer["C_CITY"], + customer["C_STATE"], + customer["C_ZIP"], + customer["C_PHONE"], + customer["C_SINCE"], + customer["C_CREDIT"], + customer["C_CREDIT_LIM"], + customer["C_DISCOUNT"], + customer["C_BALANCE"], + customer["C_YTD_PAYMENT"], + customer["C_PAYMENT_CNT"], + customer["C_DATA"], + ) # Hand back all the warehouse, district, and customer data - return [ warehouse, district, customer ] + return [warehouse, district, customer] ## ---------------------------------------------- ## doStockLevel @@ -840,23 +1159,33 @@ def doStockLevel(self, params): d_id = params["d_id"] threshold = params["threshold"] - result = self.dbs[db_from_table('DISTRICT')][self.shard_from_id(w_id)].get(gen_pk_doc('DISTRICT', - {'D_ID': d_id, 'D_W_ID': w_id})) + result = self.dbs[db_from_table("DISTRICT")][self.shard_from_id(w_id)].get( + gen_pk_doc("DISTRICT", {"D_ID": d_id, "D_W_ID": w_id}) + ) assert result - o_id = result['D_NEXT_O_ID'] + o_id = result["D_NEXT_O_ID"] # note, that we might get only parts of some orders because of isolation issues with NewOrder on 'D_NEXT_O_ID' # I really doubt anything can be done about it - orderLines = self.dbs[db_from_table('ORDER_LINE')][self.shard_from_id(w_id)].view('tpcc/o_d_w_i', - startkey = [o_id - 20, d_id, w_id], - endkey = [o_id - 1, d_id, w_id], - reduce = 'false').rows + orderLines = ( + self.dbs[db_from_table("ORDER_LINE")][self.shard_from_id(w_id)] + .view( + "tpcc/o_d_w_i", + startkey=[o_id - 20, d_id, w_id], + endkey=[o_id - 1, d_id, w_id], + reduce="false", + ) + .rows + ) # 'set' operation in the next line just filters out duplicates - stock_keys = [[w_id, i_id] for i_id in set([r['value'] for r in orderLines])] + stock_keys = [[w_id, i_id] for i_id in set([r["value"] for r in orderLines])] # do an index scan join! - stock_items = self.dbs[db_from_table('STOCK')][self.shard_from_id(w_id)].view('tpcc/w_i', - keys = stock_keys).rows + stock_items = ( + self.dbs[db_from_table("STOCK")][self.shard_from_id(w_id)] + .view("tpcc/w_i", keys=stock_keys) + .rows + ) count = 0 for item in stock_items: @@ -865,4 +1194,5 @@ def doStockLevel(self, params): return count + ## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/csvdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/csvdriver.py index e5253f56..1e0cc040 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/csvdriver.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/csvdriver.py @@ -27,63 +27,82 @@ import os import csv from datetime import datetime -from pprint import pprint,pformat +from pprint import pprint, pformat from abstractdriver import * + ## ============================================== ## CSVDriver ## ============================================== class CsvDriver(AbstractDriver): DEFAULT_CONFIG = { - "table_directory": ("The path to the directory to store the table CSV files", "/tmp/tpcc-tables" ), - "txn_directory": ("The path to the directory to store the txn CSV files", "/tmp/tpcc-txns" ), + "table_directory": ( + "The path to the directory to store the table CSV files", + "/tmp/tpcc-tables", + ), + "txn_directory": ( + "The path to the directory to store the txn CSV files", + "/tmp/tpcc-txns", + ), } - + def __init__(self, ddl): super(CsvDriver, self).__init__("csv", ddl) self.table_directory = None - self.table_outputs = { } + self.table_outputs = {} self.txn_directory = None - self.txn_outputs = { } - self.txn_params = { } + self.txn_outputs = {} + self.txn_params = {} + ## DEF - + def makeDefaultConfig(self): return CsvDriver.DEFAULT_CONFIG + ## DEF - + def loadConfig(self, config): for key in CsvDriver.DEFAULT_CONFIG.keys(): - assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) - + assert key in config, "Missing parameter '%s' in %s configuration" % ( + key, + self.name, + ) + self.table_directory = config["table_directory"] assert self.table_directory - if not os.path.exists(self.table_directory): os.makedirs(self.table_directory) - + if not os.path.exists(self.table_directory): + os.makedirs(self.table_directory) + self.txn_directory = config["txn_directory"] assert self.txn_directory - if not os.path.exists(self.txn_directory): os.makedirs(self.txn_directory) + if not os.path.exists(self.txn_directory): + os.makedirs(self.txn_directory) + ## DEF - + def loadTuples(self, tableName, tuples): if not tableName in self.table_outputs: path = os.path.join(self.table_directory, "%s.csv" % tableName) - self.table_outputs[tableName] = csv.writer(open(path, 'wb'), quoting=csv.QUOTE_ALL) + self.table_outputs[tableName] = csv.writer( + open(path, "wb"), quoting=csv.QUOTE_ALL + ) ## IF self.table_outputs[tableName].writerows(tuples) + ## DEF - + def executeTransaction(self, txn, params): if not txn in self.txn_outputs: path = os.path.join(self.txn_directory, "%s.csv" % txn) - self.txn_outputs[txn] = csv.writer(open(path, 'wb'), quoting=csv.QUOTE_ALL) + self.txn_outputs[txn] = csv.writer(open(path, "wb"), quoting=csv.QUOTE_ALL) self.txn_params[txn] = params.keys()[:] self.txn_outputs[txn].writerow(["Timestamp"] + self.txn_params[txn]) ## IF row = [datetime.now()] + [params[k] for k in self.txn_params[txn]] self.txn_outputs[txn].writerow(row) + ## DEF -## CLASS - + +## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/mongodbdriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/mongodbdriver.py index d4a73a82..59841be0 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/mongodbdriver.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/mongodbdriver.py @@ -35,121 +35,121 @@ import sys import logging import pymongo -from pprint import pprint,pformat +from pprint import pprint, pformat import constants from abstractdriver import * TABLE_COLUMNS = { constants.TABLENAME_ITEM: [ - "I_ID", # INTEGER - "I_IM_ID", # INTEGER - "I_NAME", # VARCHAR - "I_PRICE", # FLOAT - "I_DATA", # VARCHAR + "I_ID", # INTEGER + "I_IM_ID", # INTEGER + "I_NAME", # VARCHAR + "I_PRICE", # FLOAT + "I_DATA", # VARCHAR ], constants.TABLENAME_WAREHOUSE: [ - "W_ID", # SMALLINT - "W_NAME", # VARCHAR - "W_STREET_1", # VARCHAR - "W_STREET_2", # VARCHAR - "W_CITY", # VARCHAR - "W_STATE", # VARCHAR - "W_ZIP", # VARCHAR - "W_TAX", # FLOAT - "W_YTD", # FLOAT - ], + "W_ID", # SMALLINT + "W_NAME", # VARCHAR + "W_STREET_1", # VARCHAR + "W_STREET_2", # VARCHAR + "W_CITY", # VARCHAR + "W_STATE", # VARCHAR + "W_ZIP", # VARCHAR + "W_TAX", # FLOAT + "W_YTD", # FLOAT + ], constants.TABLENAME_DISTRICT: [ - "D_ID", # TINYINT - "D_W_ID", # SMALLINT - "D_NAME", # VARCHAR - "D_STREET_1", # VARCHAR - "D_STREET_2", # VARCHAR - "D_CITY", # VARCHAR - "D_STATE", # VARCHAR - "D_ZIP", # VARCHAR - "D_TAX", # FLOAT - "D_YTD", # FLOAT - "D_NEXT_O_ID", # INT + "D_ID", # TINYINT + "D_W_ID", # SMALLINT + "D_NAME", # VARCHAR + "D_STREET_1", # VARCHAR + "D_STREET_2", # VARCHAR + "D_CITY", # VARCHAR + "D_STATE", # VARCHAR + "D_ZIP", # VARCHAR + "D_TAX", # FLOAT + "D_YTD", # FLOAT + "D_NEXT_O_ID", # INT ], - constants.TABLENAME_CUSTOMER: [ - "C_ID", # INTEGER - "C_D_ID", # TINYINT - "C_W_ID", # SMALLINT - "C_FIRST", # VARCHAR - "C_MIDDLE", # VARCHAR - "C_LAST", # VARCHAR - "C_STREET_1", # VARCHAR - "C_STREET_2", # VARCHAR - "C_CITY", # VARCHAR - "C_STATE", # VARCHAR - "C_ZIP", # VARCHAR - "C_PHONE", # VARCHAR - "C_SINCE", # TIMESTAMP - "C_CREDIT", # VARCHAR - "C_CREDIT_LIM", # FLOAT - "C_DISCOUNT", # FLOAT - "C_BALANCE", # FLOAT - "C_YTD_PAYMENT", # FLOAT - "C_PAYMENT_CNT", # INTEGER - "C_DELIVERY_CNT", # INTEGER - "C_DATA", # VARCHAR + constants.TABLENAME_CUSTOMER: [ + "C_ID", # INTEGER + "C_D_ID", # TINYINT + "C_W_ID", # SMALLINT + "C_FIRST", # VARCHAR + "C_MIDDLE", # VARCHAR + "C_LAST", # VARCHAR + "C_STREET_1", # VARCHAR + "C_STREET_2", # VARCHAR + "C_CITY", # VARCHAR + "C_STATE", # VARCHAR + "C_ZIP", # VARCHAR + "C_PHONE", # VARCHAR + "C_SINCE", # TIMESTAMP + "C_CREDIT", # VARCHAR + "C_CREDIT_LIM", # FLOAT + "C_DISCOUNT", # FLOAT + "C_BALANCE", # FLOAT + "C_YTD_PAYMENT", # FLOAT + "C_PAYMENT_CNT", # INTEGER + "C_DELIVERY_CNT", # INTEGER + "C_DATA", # VARCHAR ], - constants.TABLENAME_STOCK: [ - "S_I_ID", # INTEGER - "S_W_ID", # SMALLINT - "S_QUANTITY", # INTEGER - "S_DIST_01", # VARCHAR - "S_DIST_02", # VARCHAR - "S_DIST_03", # VARCHAR - "S_DIST_04", # VARCHAR - "S_DIST_05", # VARCHAR - "S_DIST_06", # VARCHAR - "S_DIST_07", # VARCHAR - "S_DIST_08", # VARCHAR - "S_DIST_09", # VARCHAR - "S_DIST_10", # VARCHAR - "S_YTD", # INTEGER - "S_ORDER_CNT", # INTEGER - "S_REMOTE_CNT", # INTEGER - "S_DATA", # VARCHAR + constants.TABLENAME_STOCK: [ + "S_I_ID", # INTEGER + "S_W_ID", # SMALLINT + "S_QUANTITY", # INTEGER + "S_DIST_01", # VARCHAR + "S_DIST_02", # VARCHAR + "S_DIST_03", # VARCHAR + "S_DIST_04", # VARCHAR + "S_DIST_05", # VARCHAR + "S_DIST_06", # VARCHAR + "S_DIST_07", # VARCHAR + "S_DIST_08", # VARCHAR + "S_DIST_09", # VARCHAR + "S_DIST_10", # VARCHAR + "S_YTD", # INTEGER + "S_ORDER_CNT", # INTEGER + "S_REMOTE_CNT", # INTEGER + "S_DATA", # VARCHAR ], - constants.TABLENAME_ORDERS: [ - "O_ID", # INTEGER - "O_C_ID", # INTEGER - "O_D_ID", # TINYINT - "O_W_ID", # SMALLINT - "O_ENTRY_D", # TIMESTAMP - "O_CARRIER_ID", # INTEGER - "O_OL_CNT", # INTEGER - "O_ALL_LOCAL", # INTEGER + constants.TABLENAME_ORDERS: [ + "O_ID", # INTEGER + "O_C_ID", # INTEGER + "O_D_ID", # TINYINT + "O_W_ID", # SMALLINT + "O_ENTRY_D", # TIMESTAMP + "O_CARRIER_ID", # INTEGER + "O_OL_CNT", # INTEGER + "O_ALL_LOCAL", # INTEGER ], - constants.TABLENAME_NEW_ORDER: [ - "NO_O_ID", # INTEGER - "NO_D_ID", # TINYINT - "NO_W_ID", # SMALLINT + constants.TABLENAME_NEW_ORDER: [ + "NO_O_ID", # INTEGER + "NO_D_ID", # TINYINT + "NO_W_ID", # SMALLINT ], constants.TABLENAME_ORDER_LINE: [ - "OL_O_ID", # INTEGER - "OL_D_ID", # TINYINT - "OL_W_ID", # SMALLINT - "OL_NUMBER", # INTEGER - "OL_I_ID", # INTEGER - "OL_SUPPLY_W_ID", # SMALLINT - "OL_DELIVERY_D", # TIMESTAMP - "OL_QUANTITY", # INTEGER - "OL_AMOUNT", # FLOAT - "OL_DIST_INFO", # VARCHAR + "OL_O_ID", # INTEGER + "OL_D_ID", # TINYINT + "OL_W_ID", # SMALLINT + "OL_NUMBER", # INTEGER + "OL_I_ID", # INTEGER + "OL_SUPPLY_W_ID", # SMALLINT + "OL_DELIVERY_D", # TIMESTAMP + "OL_QUANTITY", # INTEGER + "OL_AMOUNT", # FLOAT + "OL_DIST_INFO", # VARCHAR ], - constants.TABLENAME_HISTORY: [ - "H_C_ID", # INTEGER - "H_C_D_ID", # TINYINT - "H_C_W_ID", # SMALLINT - "H_D_ID", # TINYINT - "H_W_ID", # SMALLINT - "H_DATE", # TIMESTAMP - "H_AMOUNT", # FLOAT - "H_DATA", # VARCHAR + constants.TABLENAME_HISTORY: [ + "H_C_ID", # INTEGER + "H_C_D_ID", # TINYINT + "H_C_W_ID", # SMALLINT + "H_D_ID", # TINYINT + "H_W_ID", # SMALLINT + "H_DATE", # TIMESTAMP + "H_AMOUNT", # FLOAT + "H_DATA", # VARCHAR ], } TABLE_INDEXES = { @@ -158,27 +158,27 @@ ], constants.TABLENAME_WAREHOUSE: [ "W_ID", - ], + ], constants.TABLENAME_DISTRICT: [ "D_ID", "D_W_ID", ], - constants.TABLENAME_CUSTOMER: [ + constants.TABLENAME_CUSTOMER: [ "C_ID", "C_D_ID", "C_W_ID", ], - constants.TABLENAME_STOCK: [ + constants.TABLENAME_STOCK: [ "S_I_ID", "S_W_ID", ], - constants.TABLENAME_ORDERS: [ + constants.TABLENAME_ORDERS: [ "O_ID", "O_D_ID", "O_W_ID", "O_C_ID", ], - constants.TABLENAME_NEW_ORDER: [ + constants.TABLENAME_NEW_ORDER: [ "NO_O_ID", "NO_D_ID", "NO_W_ID", @@ -190,15 +190,19 @@ ], } + ## ============================================== ## MongodbDriver ## ============================================== class MongodbDriver(AbstractDriver): DEFAULT_CONFIG = { - "host": ("The hostname to mongod", "localhost" ), - "port": ("The port number to mongod", 27017 ), - "name": ("Collection name", "tpcc"), - "denormalize": ("If set to true, then the CUSTOMER data will be denormalized into a single document", True), + "host": ("The hostname to mongod", "localhost"), + "port": ("The port number to mongod", 27017), + "name": ("Collection name", "tpcc"), + "denormalize": ( + "If set to true, then the CUSTOMER data will be denormalized into a single document", + True, + ), } DENORMALIZED_TABLES = [ constants.TABLENAME_CUSTOMER, @@ -206,38 +210,41 @@ class MongodbDriver(AbstractDriver): constants.TABLENAME_ORDER_LINE, constants.TABLENAME_HISTORY, ] - - + def __init__(self, ddl): super(MongodbDriver, self).__init__("mongodb", ddl) self.database = None self.conn = None self.denormalize = False - self.w_customers = { } - self.w_orders = { } - + self.w_customers = {} + self.w_orders = {} + ## Create member mapping to collections for name in constants.ALL_TABLES: self.__dict__[name.lower()] = None - + ## ---------------------------------------------- ## makeDefaultConfig ## ---------------------------------------------- def makeDefaultConfig(self): return MongodbDriver.DEFAULT_CONFIG - + ## ---------------------------------------------- ## loadConfig ## ---------------------------------------------- def loadConfig(self, config): for key in MongodbDriver.DEFAULT_CONFIG.keys(): - assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) - - self.conn = pymongo.MongoClient(config['host'], int(config['port'])) - self.database = self.conn[str(config['name'])] - self.denormalize = config['denormalize'] - if self.denormalize: logging.debug("Using denormalized data model") - + assert key in config, "Missing parameter '%s' in %s configuration" % ( + key, + self.name, + ) + + self.conn = pymongo.MongoClient(config["host"], int(config["port"])) + self.database = self.conn[str(config["name"])] + self.denormalize = config["denormalize"] + if self.denormalize: + logging.debug("Using denormalized data model") + if config["reset"]: logging.debug("Deleting database '%s'" % self.database.name) for name in constants.ALL_TABLES: @@ -245,84 +252,112 @@ def loadConfig(self, config): self.database.drop_collection(name) logging.debug("Dropped collection %s" % name) ## IF - + ## Setup! - load_indexes = ('execute' in config and not config['execute']) and \ - ('load' in config and not config['load']) + load_indexes = ("execute" in config and not config["execute"]) and ( + "load" in config and not config["load"] + ) for name in constants.ALL_TABLES: - if self.denormalize and name in MongodbDriver.DENORMALIZED_TABLES[1:]: continue - + if self.denormalize and name in MongodbDriver.DENORMALIZED_TABLES[1:]: + continue + ## Create member mapping to collections self.__dict__[name.lower()] = self.database[name] - + ## Create Indexes - if load_indexes and name in TABLE_INDEXES and \ - (self.denormalize or (self.denormalize == False and not name in MongodbDriver.DENORMALIZED_TABLES[1:])): + if ( + load_indexes + and name in TABLE_INDEXES + and ( + self.denormalize + or ( + self.denormalize == False + and not name in MongodbDriver.DENORMALIZED_TABLES[1:] + ) + ) + ): logging.debug("Creating index for %s" % name) for index in TABLE_INDEXES[name]: - self.database[name].create_index(index) + self.database[name].create_index(index) ## FOR - + ## ---------------------------------------------- ## loadTuples ## ---------------------------------------------- def loadTuples(self, tableName, tuples): - if len(tuples) == 0: return + if len(tuples) == 0: + return logging.debug("Loading %d tuples for tableName %s" % (len(tuples), tableName)) - + assert tableName in TABLE_COLUMNS, "Unexpected table %s" % tableName columns = TABLE_COLUMNS[tableName] num_columns = range(len(columns)) - - tuple_dicts = [ ] - + + tuple_dicts = [] + ## We want to combine all of a CUSTOMER's ORDERS, ORDER_LINE, and HISTORY records ## into a single document if self.denormalize and tableName in MongodbDriver.DENORMALIZED_TABLES: ## If this is the CUSTOMER table, then we'll just store the record locally for now if tableName == constants.TABLENAME_CUSTOMER: for t in tuples: - key = tuple(t[:3]) # C_ID, D_ID, W_ID - self.w_customers[key] = dict(map(lambda i: (columns[i], t[i]), num_columns)) + key = tuple(t[:3]) # C_ID, D_ID, W_ID + self.w_customers[key] = dict( + map(lambda i: (columns[i], t[i]), num_columns) + ) ## FOR - - ## If this is an ORDER_LINE record, then we need to stick it inside of the + + ## If this is an ORDER_LINE record, then we need to stick it inside of the ## right ORDERS record elif tableName == constants.TABLENAME_ORDER_LINE: for t in tuples: - o_key = tuple(t[:3]) # O_ID, O_D_ID, O_W_ID + o_key = tuple(t[:3]) # O_ID, O_D_ID, O_W_ID (c_key, o_idx) = self.w_orders[o_key] c = self.w_customers[c_key] assert o_idx >= 0 assert o_idx < len(c[constants.TABLENAME_ORDERS]) o = c[constants.TABLENAME_ORDERS][o_idx] - if not tableName in o: o[tableName] = [ ] - o[tableName].append(dict(map(lambda i: (columns[i], t[i]), num_columns[4:]))) + if not tableName in o: + o[tableName] = [] + o[tableName].append( + dict(map(lambda i: (columns[i], t[i]), num_columns[4:])) + ) ## FOR - + ## Otherwise we have to find the CUSTOMER record for the other tables ## and append ourselves to them else: if tableName == constants.TABLENAME_ORDERS: key_start = 1 - cols = num_columns[0:1] + num_columns[4:] # Removes O_C_ID, O_D_ID, O_W_ID + cols = ( + num_columns[0:1] + num_columns[4:] + ) # Removes O_C_ID, O_D_ID, O_W_ID else: key_start = 0 - cols = num_columns[3:] # Removes H_C_ID, H_C_D_ID, H_C_W_ID - + cols = num_columns[3:] # Removes H_C_ID, H_C_D_ID, H_C_W_ID + for t in tuples: - c_key = tuple(t[key_start:key_start+3]) # C_ID, D_ID, W_ID - assert c_key in self.w_customers, "Customer Key: %s\nAll Keys:\n%s" % (str(c_key), "\n".join(map(str, sorted(self.w_customers.keys())))) + c_key = tuple(t[key_start : key_start + 3]) # C_ID, D_ID, W_ID + assert ( + c_key in self.w_customers + ), "Customer Key: %s\nAll Keys:\n%s" % ( + str(c_key), + "\n".join(map(str, sorted(self.w_customers.keys()))), + ) c = self.w_customers[c_key] - - if not tableName in c: c[tableName] = [ ] + + if not tableName in c: + c[tableName] = [] c[tableName].append(dict(map(lambda i: (columns[i], t[i]), cols))) - + ## Since ORDER_LINE doesn't have a C_ID, we have to store a reference to ## this ORDERS record so that we can look it up later if tableName == constants.TABLENAME_ORDERS: - o_key = (t[0], t[2], t[3]) # O_ID, O_D_ID, O_W_ID - self.w_orders[o_key] = (c_key, len(c[tableName])-1) # CUSTOMER, ORDER IDX + o_key = (t[0], t[2], t[3]) # O_ID, O_D_ID, O_W_ID + self.w_orders[o_key] = ( + c_key, + len(c[tableName]) - 1, + ) # CUSTOMER, ORDER IDX ## FOR ## IF @@ -333,16 +368,21 @@ def loadTuples(self, tableName, tuples): ## FOR self.database[tableName].insert(tuple_dicts) ## IF - + return - + ## ---------------------------------------------- ## loadFinishDistrict ## ---------------------------------------------- def loadFinishDistrict(self, w_id, d_id): if self.denormalize: - logging.debug("Pushing %d denormalized CUSTOMER records for WAREHOUSE %d DISTRICT %d into MongoDB" % (len(self.w_customers), w_id, d_id)) - self.database[constants.TABLENAME_CUSTOMER].insert(self.w_customers.values()) + logging.debug( + "Pushing %d denormalized CUSTOMER records for WAREHOUSE %d DISTRICT %d into MongoDB" + % (len(self.w_customers), w_id, d_id) + ) + self.database[constants.TABLENAME_CUSTOMER].insert( + self.w_customers.values() + ) self.w_customers.clear() self.w_orders.clear() ## IF @@ -354,8 +394,11 @@ def loadFinish(self): logging.info("Finished loading tables") if logging.getLogger().isEnabledFor(logging.DEBUG): for name in constants.ALL_TABLES: - if self.denormalize and name in MongodbDriver.DENORMALIZED_TABLES[1:]: continue - logging.debug("%-12s%d records" % (name+":", self.database[name].count())) + if self.denormalize and name in MongodbDriver.DENORMALIZED_TABLES[1:]: + continue + logging.debug( + "%-12s%d records" % (name + ":", self.database[name].count()) + ) ## IF ## ---------------------------------------------- @@ -365,23 +408,32 @@ def doDelivery(self, params): w_id = params["w_id"] o_carrier_id = params["o_carrier_id"] ol_delivery_d = params["ol_delivery_d"] - - result = [ ] - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): + + result = [] + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1): ## getNewOrder - no = self.new_order.find_one({"NO_D_ID": d_id, "NO_W_ID": w_id}, {"NO_O_ID": 1}) + no = self.new_order.find_one( + {"NO_D_ID": d_id, "NO_W_ID": w_id}, {"NO_O_ID": 1} + ) if no == None: ## No orders for this district: skip it. Note: This must be reported if > 1% continue assert len(no) > 0 o_id = no["NO_O_ID"] - + if self.denormalize: ## getCId - c = self.customer.find_one({"ORDERS.O_ID": o_id, "C_D_ID": d_id, "C_W_ID": w_id}, {"C_ID": 1, "ORDERS.O_ID": 1, "ORDERS.ORDER_LINE": 1}) - assert c != None, "No customer record [O_ID=%d, D_ID=%d, W_ID=%d]" % (o_id, d_id, w_id) + c = self.customer.find_one( + {"ORDERS.O_ID": o_id, "C_D_ID": d_id, "C_W_ID": w_id}, + {"C_ID": 1, "ORDERS.O_ID": 1, "ORDERS.ORDER_LINE": 1}, + ) + assert c != None, "No customer record [O_ID=%d, D_ID=%d, W_ID=%d]" % ( + o_id, + d_id, + w_id, + ) c_id = c["C_ID"] - + ## sumOLAmount + updateOrderLine ol_total = 0 for o in c["ORDERS"]: @@ -393,7 +445,7 @@ def doDelivery(self, params): ol["OL_DELIVERY_D"] = ol_delivery_d break ## FOR - + if ol_total == 0: pprint(params) pprint(no) @@ -401,37 +453,63 @@ def doDelivery(self, params): sys.exit(1) ## updateOrders + updateCustomer - self.customer.update({"_id": c['_id'], "ORDERS.O_ID": o_id}, {"$set": {"ORDERS.$.O_CARRIER_ID": o_carrier_id, "ORDERS.$.ORDER_LINE": orderLines}, "$inc": {"C_BALANCE": ol_total}}, multi=False) + self.customer.update( + {"_id": c["_id"], "ORDERS.O_ID": o_id}, + { + "$set": { + "ORDERS.$.O_CARRIER_ID": o_carrier_id, + "ORDERS.$.ORDER_LINE": orderLines, + }, + "$inc": {"C_BALANCE": ol_total}, + }, + multi=False, + ) else: ## getCId - o = self.orders.find_one({"O_ID": o_id, "O_D_ID": d_id, "O_W_ID": w_id}, {"O_C_ID": 1}) + o = self.orders.find_one( + {"O_ID": o_id, "O_D_ID": d_id, "O_W_ID": w_id}, {"O_C_ID": 1} + ) assert o != None c_id = o["O_C_ID"] - + ## sumOLAmount - orderLines = self.order_line.find({"OL_O_ID": o_id, "OL_D_ID": d_id, "OL_W_ID": w_id}, {"OL_AMOUNT": 1}) + orderLines = self.order_line.find( + {"OL_O_ID": o_id, "OL_D_ID": d_id, "OL_W_ID": w_id}, + {"OL_AMOUNT": 1}, + ) assert orderLines != None ol_total = sum([ol["OL_AMOUNT"] for ol in orderLines]) - + ## updateOrders - self.orders.update(o, {"$set": {"O_CARRIER_ID": o_carrier_id}}, multi=False) - + self.orders.update( + o, {"$set": {"O_CARRIER_ID": o_carrier_id}}, multi=False + ) + ## updateOrderLine - self.order_line.update({"OL_O_ID": o_id, "OL_D_ID": d_id, "OL_W_ID": w_id}, {"$set": {"OL_DELIVERY_D": ol_delivery_d}}, multi=True) - + self.order_line.update( + {"OL_O_ID": o_id, "OL_D_ID": d_id, "OL_W_ID": w_id}, + {"$set": {"OL_DELIVERY_D": ol_delivery_d}}, + multi=True, + ) + ## updateCustomer - self.customer.update({"C_ID": c_id, "C_D_ID": d_id, "C_W_ID": w_id}, {"$inc": {"C_BALANCE": ol_total}}) + self.customer.update( + {"C_ID": c_id, "C_D_ID": d_id, "C_W_ID": w_id}, + {"$inc": {"C_BALANCE": ol_total}}, + ) ## IF ## deleteNewOrder - self.new_order.remove({"_id": no['_id']}) + self.new_order.remove({"_id": no["_id"]}) # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure # them out # If there are no order lines, SUM returns null. There should always be order lines. - assert ol_total != None, "ol_total is NULL: there are no order lines. This should not happen" + assert ( + ol_total != None + ), "ol_total is NULL: there are no order lines. This should not happen" assert ol_total > 0.0 result.append((d_id, o_id)) @@ -450,43 +528,51 @@ def doNewOrder(self, params): i_w_ids = params["i_w_ids"] i_qtys = params["i_qtys"] s_dist_col = "S_DIST_%02d" % d_id - + assert len(i_ids) > 0 assert len(i_ids) == len(i_w_ids) assert len(i_ids) == len(i_qtys) ## http://stackoverflow.com/q/3844931/ - all_local = (not i_w_ids or [w_id] * len(i_w_ids) == i_w_ids) - - items = self.item.find({"I_ID": {"$in": i_ids}}, {"I_ID": 1, "I_PRICE": 1, "I_NAME": 1, "I_DATA": 1}) + all_local = not i_w_ids or [w_id] * len(i_w_ids) == i_w_ids + + items = self.item.find( + {"I_ID": {"$in": i_ids}}, + {"I_ID": 1, "I_PRICE": 1, "I_NAME": 1, "I_DATA": 1}, + ) ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. ## Note that this will happen with 1% of transactions on purpose. if items.count() != len(i_ids): ## TODO Abort here! return ## IF - + ## ---------------- ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER ## ---------------- - + # getWarehouseTaxRate w = self.warehouse.find_one({"W_ID": w_id}, {"W_TAX": 1}) assert w w_tax = w["W_TAX"] - + # getDistrict - d = self.district.find_one({"D_ID": d_id, "D_W_ID": w_id}, {"D_TAX": 1, "D_NEXT_O_ID": 1}) + d = self.district.find_one( + {"D_ID": d_id, "D_W_ID": w_id}, {"D_TAX": 1, "D_NEXT_O_ID": 1} + ) assert d d_tax = d["D_TAX"] d_next_o_id = d["D_NEXT_O_ID"] - + # incrementNextOrderId # HACK: This is not transactionally safe! self.district.update(d, {"$inc": {"D_NEXT_O_ID": 1}}, multi=False) - + # getCustomer - c = self.customer.find_one({"C_ID": c_id, "C_D_ID": d_id, "C_W_ID": w_id}, {"C_DISCOUNT": 1, "C_LAST": 1, "C_CREDIT": 1}) + c = self.customer.find_one( + {"C_ID": c_id, "C_D_ID": d_id, "C_W_ID": w_id}, + {"C_DISCOUNT": 1, "C_LAST": 1, "C_CREDIT": 1}, + ) assert c c_discount = c["C_DISCOUNT"] @@ -497,19 +583,27 @@ def doNewOrder(self, params): o_carrier_id = constants.NULL_CARRIER_ID # createNewOrder - self.new_order.insert({"NO_O_ID": d_next_o_id, "NO_D_ID": d_id, "NO_W_ID": w_id}) + self.new_order.insert( + {"NO_O_ID": d_next_o_id, "NO_D_ID": d_id, "NO_W_ID": w_id} + ) - o = {"O_ID": d_next_o_id, "O_ENTRY_D": o_entry_d, "O_CARRIER_ID": o_carrier_id, "O_OL_CNT": ol_cnt, "O_ALL_LOCAL": all_local} + o = { + "O_ID": d_next_o_id, + "O_ENTRY_D": o_entry_d, + "O_CARRIER_ID": o_carrier_id, + "O_OL_CNT": ol_cnt, + "O_ALL_LOCAL": all_local, + } if self.denormalize: - o[constants.TABLENAME_ORDER_LINE] = [ ] + o[constants.TABLENAME_ORDER_LINE] = [] else: o["O_D_ID"] = d_id o["O_W_ID"] = w_id o["O_C_ID"] = c_id - + # createOrder - self.orders.insert(o) - + self.orders.insert(o) + ## ---------------- ## OPTIMIZATION: ## If all of the items are at the same warehouse, then we'll issue a single @@ -518,17 +612,28 @@ def doNewOrder(self, params): stockInfos = None if all_local and False: # getStockInfo - allStocks = self.stock.find({"S_I_ID": {"$in": i_ids}, "S_W_ID": w_id}, {"S_I_ID": 1, "S_QUANTITY": 1, "S_DATA": 1, "S_YTD": 1, "S_ORDER_CNT": 1, "S_REMOTE_CNT": 1, s_dist_col: 1}) + allStocks = self.stock.find( + {"S_I_ID": {"$in": i_ids}, "S_W_ID": w_id}, + { + "S_I_ID": 1, + "S_QUANTITY": 1, + "S_DATA": 1, + "S_YTD": 1, + "S_ORDER_CNT": 1, + "S_REMOTE_CNT": 1, + s_dist_col: 1, + }, + ) assert allStocks.count() == ol_cnt - stockInfos = { } + stockInfos = {} for si in allStocks: - stockInfos["S_I_ID"] = si # HACK + stockInfos["S_I_ID"] = si # HACK ## IF ## ---------------- ## Insert Order Item Information ## ---------------- - item_data = [ ] + item_data = [] total = 0 for i in range(ol_cnt): ol_number = i + 1 @@ -544,9 +649,23 @@ def doNewOrder(self, params): # getStockInfo if all_local and stockInfos != None: si = stockInfos[ol_i_id] - assert si["S_I_ID"] == ol_i_id, "S_I_ID should be %d\n%s" % (ol_i_id, pformat(si)) + assert si["S_I_ID"] == ol_i_id, "S_I_ID should be %d\n%s" % ( + ol_i_id, + pformat(si), + ) else: - si = self.stock.find_one({"S_I_ID": ol_i_id, "S_W_ID": w_id}, {"S_I_ID": 1, "S_QUANTITY": 1, "S_DATA": 1, "S_YTD": 1, "S_ORDER_CNT": 1, "S_REMOTE_CNT": 1, s_dist_col: 1}) + si = self.stock.find_one( + {"S_I_ID": ol_i_id, "S_W_ID": w_id}, + { + "S_I_ID": 1, + "S_QUANTITY": 1, + "S_DATA": 1, + "S_YTD": 1, + "S_ORDER_CNT": 1, + "S_REMOTE_CNT": 1, + s_dist_col: 1, + }, + ) assert si, "Failed to find S_I_ID: %d\n%s" % (ol_i_id, pformat(itemInfo)) s_quantity = si["S_QUANTITY"] @@ -554,7 +673,7 @@ def doNewOrder(self, params): s_order_cnt = si["S_ORDER_CNT"] s_remote_cnt = si["S_REMOTE_CNT"] s_data = si["S_DATA"] - s_dist_xx = si[s_dist_col] # Fetches data from the s_dist_[d_id] column + s_dist_xx = si[s_dist_col] # Fetches data from the s_dist_[d_id] column ## Update stock s_ytd += ol_quantity @@ -563,21 +682,44 @@ def doNewOrder(self, params): else: s_quantity = s_quantity + 91 - ol_quantity s_order_cnt += 1 - - if ol_supply_w_id != w_id: s_remote_cnt += 1 + + if ol_supply_w_id != w_id: + s_remote_cnt += 1 # updateStock - self.stock.update(si, {"$set": {"S_QUANTITY": s_quantity, "S_YTD": s_ytd, "S_ORDER_CNT": s_order_cnt, "S_REMOTE_CNT": s_remote_cnt}}) + self.stock.update( + si, + { + "$set": { + "S_QUANTITY": s_quantity, + "S_YTD": s_ytd, + "S_ORDER_CNT": s_order_cnt, + "S_REMOTE_CNT": s_remote_cnt, + } + }, + ) - if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: - brand_generic = 'B' + if ( + i_data.find(constants.ORIGINAL_STRING) != -1 + and s_data.find(constants.ORIGINAL_STRING) != -1 + ): + brand_generic = "B" else: - brand_generic = 'G' + brand_generic = "G" ## Transaction profile states to use "ol_quantity * i_price" ol_amount = ol_quantity * i_price total += ol_amount - ol = {"OL_O_ID": d_next_o_id, "OL_NUMBER": ol_number, "OL_I_ID": ol_i_id, "OL_SUPPLY_W_ID": ol_supply_w_id, "OL_DELIVERY_D": o_entry_d, "OL_QUANTITY": ol_quantity, "OL_AMOUNT": ol_amount, "OL_DIST_INFO": s_dist_xx} + ol = { + "OL_O_ID": d_next_o_id, + "OL_NUMBER": ol_number, + "OL_I_ID": ol_i_id, + "OL_SUPPLY_W_ID": ol_supply_w_id, + "OL_DELIVERY_D": o_entry_d, + "OL_QUANTITY": ol_quantity, + "OL_AMOUNT": ol_amount, + "OL_DIST_INFO": s_dist_xx, + } if self.denormalize: # createOrderLine @@ -585,28 +727,28 @@ def doNewOrder(self, params): else: ol["OL_D_ID"] = d_id ol["OL_W_ID"] = w_id - + # createOrderLine self.order_line.insert(ol) ## IF ## Add the info to be returned - item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) + item_data.append((i_name, s_quantity, brand_generic, i_price, ol_amount)) ## FOR - + ## Adjust the total for the discount - #print "c_discount:", c_discount, type(c_discount) - #print "w_tax:", w_tax, type(w_tax) - #print "d_tax:", d_tax, type(d_tax) + # print "c_discount:", c_discount, type(c_discount) + # print "w_tax:", w_tax, type(w_tax) + # print "d_tax:", d_tax, type(d_tax) total *= (1 - c_discount) * (1 + w_tax + d_tax) # createOrder self.customer.update({"_id": c["_id"]}, {"$push": {"ORDERS": o}}) ## Pack up values the client is missing (see TPC-C 2.4.3.5) - misc = [ (w_tax, d_tax, d_next_o_id, total) ] - - return [ c, misc, item_data ] + misc = [(w_tax, d_tax, d_next_o_id, total)] + + return [c, misc, item_data] ## ---------------------------------------------- ## doOrderStatus @@ -616,42 +758,51 @@ def doOrderStatus(self, params): d_id = params["d_id"] c_id = params["c_id"] c_last = params["c_last"] - + assert w_id, pformat(params) assert d_id, pformat(params) search_fields = {"C_W_ID": w_id, "C_D_ID": d_id} - return_fields = {"C_ID": 1, "C_FIRST": 1, "C_MIDDLE": 1, "C_LAST": 1, "C_BALANCE": 1} + return_fields = { + "C_ID": 1, + "C_FIRST": 1, + "C_MIDDLE": 1, + "C_LAST": 1, + "C_BALANCE": 1, + } if self.denormalize: - for f in ['O_ID', 'O_CARRIER_ID', 'O_ENTRY_D']: + for f in ["O_ID", "O_CARRIER_ID", "O_ENTRY_D"]: return_fields["%s.%s" % (constants.TABLENAME_ORDERS, f)] = 1 - for f in ['OL_SUPPLY_W_ID', 'OL_I_ID', 'OL_QUANTITY']: - return_fields["%s.%s.%s" % (constants.TABLENAME_ORDERS, constants.TABLENAME_ORDER_LINE, f)] = 1 + for f in ["OL_SUPPLY_W_ID", "OL_I_ID", "OL_QUANTITY"]: + return_fields[ + "%s.%s.%s" + % (constants.TABLENAME_ORDERS, constants.TABLENAME_ORDER_LINE, f) + ] = 1 ## IF - + if c_id != None: # getCustomerByCustomerId search_fields["C_ID"] = c_id c = self.customer.find_one(search_fields, return_fields) assert c - + else: # getCustomersByLastName # Get the midpoint customer's id - search_fields['C_LAST'] = c_last - + search_fields["C_LAST"] = c_last + all_customers = self.customer.find(search_fields, return_fields) namecnt = all_customers.count() assert namecnt > 0 - index = (namecnt-1)/2 + index = (namecnt - 1) / 2 c = all_customers[index] c_id = c["C_ID"] assert len(c) > 0 assert c_id != None - orderLines = [ ] + orderLines = [] order = None - + if self.denormalize: # getLastOrder if constants.TABLENAME_ORDERS in c: @@ -660,20 +811,35 @@ def doOrderStatus(self, params): orderLines = order[constants.TABLENAME_ORDER_LINE] else: # getLastOrder - order = self.orders.find({"O_W_ID": w_id, "O_D_ID": d_id, "O_C_ID": c_id}, {"O_ID": 1, "O_CARRIER_ID": 1, "O_ENTRY_D": 1}).sort("O_ID", direction=pymongo.DESCENDING).limit(1)[0] + order = ( + self.orders.find( + {"O_W_ID": w_id, "O_D_ID": d_id, "O_C_ID": c_id}, + {"O_ID": 1, "O_CARRIER_ID": 1, "O_ENTRY_D": 1}, + ) + .sort("O_ID", direction=pymongo.DESCENDING) + .limit(1)[0] + ) o_id = order["O_ID"] if order: # getOrderLines - orderLines = self.order_line.find({"OL_W_ID": w_id, "OL_D_ID": d_id, "OL_O_ID": o_id}, {"OL_SUPPLY_W_ID": 1, "OL_I_ID": 1, "OL_QUANTITY": 1, "OL_AMOUNT": 1, "OL_DELIVERY_D": 1}) + orderLines = self.order_line.find( + {"OL_W_ID": w_id, "OL_D_ID": d_id, "OL_O_ID": o_id}, + { + "OL_SUPPLY_W_ID": 1, + "OL_I_ID": 1, + "OL_QUANTITY": 1, + "OL_AMOUNT": 1, + "OL_DELIVERY_D": 1, + }, + ) ## IF - - return [ c, order, orderLines ] + return [c, order, orderLines] ## ---------------------------------------------- ## doPayment - ## ---------------------------------------------- + ## ---------------------------------------------- def doPayment(self, params): w_id = params["w_id"] d_id = params["d_id"] @@ -686,36 +852,38 @@ def doPayment(self, params): search_fields = {"C_W_ID": w_id, "C_D_ID": d_id} return_fields = {"C_BALANCE": 0, "C_YTD_PAYMENT": 0, "C_PAYMENT_CNT": 0} - + if c_id != None: # getCustomerByCustomerId search_fields["C_ID"] = c_id c = self.customer.find_one(search_fields, return_fields) assert c - + else: # getCustomersByLastName # Get the midpoint customer's id - search_fields['C_LAST'] = c_last + search_fields["C_LAST"] = c_last all_customers = self.customer.find(search_fields, return_fields) namecnt = all_customers.count() assert namecnt > 0 - index = (namecnt-1)/2 + index = (namecnt - 1) / 2 c = all_customers[index] c_id = c["C_ID"] assert len(c) > 0 assert c_id != None - + if c_id != None: # getCustomerByCustomerId c = self.customer.find_one({"C_W_ID": w_id, "C_D_ID": d_id, "C_ID": c_id}) else: # getCustomersByLastName # Get the midpoint customer's id - all_customers = self.customer.find({"C_W_ID": w_id, "C_D_ID": d_id, "C_LAST": c_last}) + all_customers = self.customer.find( + {"C_W_ID": w_id, "C_D_ID": d_id, "C_LAST": c_last} + ) namecnt = all_customers.count() assert namecnt > 0 - index = (namecnt-1)/2 + index = (namecnt - 1) / 2 c = all_customers[index] c_id = c["C_ID"] assert len(c) > 0 @@ -723,33 +891,66 @@ def doPayment(self, params): c_data = c["C_DATA"] # getWarehouse - w = self.warehouse.find_one({"W_ID": w_id}, {"W_NAME": 1, "W_STREET_1": 1, "W_STREET_2": 1, "W_CITY": 1, "W_STATE": 1, "W_ZIP": 1}) + w = self.warehouse.find_one( + {"W_ID": w_id}, + { + "W_NAME": 1, + "W_STREET_1": 1, + "W_STREET_2": 1, + "W_CITY": 1, + "W_STATE": 1, + "W_ZIP": 1, + }, + ) assert w - + # updateWarehouseBalance self.warehouse.update({"_id": w["_id"]}, {"$inc": {"W_YTD": h_amount}}) # getDistrict - d = self.district.find_one({"D_W_ID": w_id, "D_ID": d_id}, {"D_NAME": 1, "D_STREET_1": 1, "D_STREET_2": 1, "D_CITY": 1, "D_STATE": 1, "D_ZIP": 1}) + d = self.district.find_one( + {"D_W_ID": w_id, "D_ID": d_id}, + { + "D_NAME": 1, + "D_STREET_1": 1, + "D_STREET_2": 1, + "D_CITY": 1, + "D_STATE": 1, + "D_ZIP": 1, + }, + ) assert d - + # updateDistrictBalance - self.district.update({"_id": d["_id"]}, {"$inc": {"D_YTD": h_amount}}) + self.district.update({"_id": d["_id"]}, {"$inc": {"D_YTD": h_amount}}) + + # Build CUSTOMER update command + customer_update = { + "$inc": { + "C_BALANCE": h_amount * -1, + "C_YTD_PAYMENT": h_amount, + "C_PAYMENT_CNT": 1, + } + } - # Build CUSTOMER update command - customer_update = {"$inc": {"C_BALANCE": h_amount*-1, "C_YTD_PAYMENT": h_amount, "C_PAYMENT_CNT": 1}} - # Customer Credit Information if c["C_CREDIT"] == constants.BAD_CREDIT: newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) - c_data = (newData + "|" + c_data) - if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] + c_data = newData + "|" + c_data + if len(c_data) > constants.MAX_C_DATA: + c_data = c_data[: constants.MAX_C_DATA] customer_update["$set"] = {"C_DATA": c_data} ## IF # Concatenate w_name, four spaces, d_name h_data = "%s %s" % (w["W_NAME"], d["D_NAME"]) - h = {"H_D_ID": d_id, "H_W_ID": w_id, "H_DATE": h_date, "H_AMOUNT": h_amount, "H_DATA": h_data} + h = { + "H_D_ID": d_id, + "H_W_ID": w_id, + "H_DATE": h_date, + "H_AMOUNT": h_amount, + "H_DATA": h_data, + } if self.denormalize: # insertHistory + updateCustomer customer_update["$push"] = {constants.TABLENAME_HISTORY: h} @@ -757,7 +958,7 @@ def doPayment(self, params): else: # updateCustomer self.customer.update({"_id": c["_id"]}, customer_update) - + # insertHistory self.history.insert(h) @@ -769,41 +970,62 @@ def doPayment(self, params): # H_AMOUNT, and H_DATE. # Hand back all the warehouse, district, and customer data - return [ w, d, c ] - + return [w, d, c] + ## ---------------------------------------------- ## doStockLevel - ## ---------------------------------------------- + ## ---------------------------------------------- def doStockLevel(self, params): w_id = params["w_id"] d_id = params["d_id"] threshold = params["threshold"] - + # getOId d = self.district.find_one({"D_W_ID": w_id, "D_ID": d_id}, {"D_NEXT_O_ID": 1}) assert d o_id = d["D_NEXT_O_ID"] - + # getStockCount # Outer Table: ORDER_LINE # Inner Table: STOCK if self.denormalize: - c = self.customer.find({"C_W_ID": w_id, "C_D_ID": d_id, "ORDERS.O_ID": {"$lt": o_id, "$gte": o_id-20}}, {"ORDERS.ORDER_LINE.OL_I_ID": 1}) + c = self.customer.find( + { + "C_W_ID": w_id, + "C_D_ID": d_id, + "ORDERS.O_ID": {"$lt": o_id, "$gte": o_id - 20}, + }, + {"ORDERS.ORDER_LINE.OL_I_ID": 1}, + ) assert c - orderLines = [ ] + orderLines = [] for ol in c: assert "ORDER_LINE" in ol["ORDERS"][0] orderLines.extend(ol["ORDERS"][0]["ORDER_LINE"]) else: - orderLines = self.order_line.find({"OL_W_ID": w_id, "OL_D_ID": d_id, "OL_O_ID": {"$lt": o_id, "$gte": o_id-20}}, {"OL_I_ID": 1}) - + orderLines = self.order_line.find( + { + "OL_W_ID": w_id, + "OL_D_ID": d_id, + "OL_O_ID": {"$lt": o_id, "$gte": o_id - 20}, + }, + {"OL_I_ID": 1}, + ) + assert orderLines ol_ids = set() for ol in orderLines: ol_ids.add(ol["OL_I_ID"]) ## FOR - result = self.stock.find({"S_W_ID": w_id, "S_I_ID": {"$in": list(ol_ids)}, "S_QUANTITY": {"$lt": threshold}}).count() - + result = self.stock.find( + { + "S_W_ID": w_id, + "S_I_ID": {"$in": list(ol_ids)}, + "S_QUANTITY": {"$lt": threshold}, + } + ).count() + return int(result) - + + ## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/sqlitedriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/sqlitedriver.py index bec71091..623affd8 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/sqlitedriver.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/sqlitedriver.py @@ -35,55 +35,52 @@ import sqlite3 import logging import commands -from pprint import pprint,pformat +from pprint import pprint, pformat import constants from abstractdriver import * TXN_QUERIES = { "DELIVERY": { - "getNewOrder": "SELECT NO_O_ID FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID > -1 LIMIT 1", # - "deleteNewOrder": "DELETE FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID = ?", # d_id, w_id, no_o_id - "getCId": "SELECT O_C_ID FROM ORDERS WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ?", # no_o_id, d_id, w_id - "updateOrders": "UPDATE ORDERS SET O_CARRIER_ID = ? WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ?", # o_carrier_id, no_o_id, d_id, w_id - "updateOrderLine": "UPDATE ORDER_LINE SET OL_DELIVERY_D = ? WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ?", # o_entry_d, no_o_id, d_id, w_id - "sumOLAmount": "SELECT SUM(OL_AMOUNT) FROM ORDER_LINE WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ?", # no_o_id, d_id, w_id - "updateCustomer": "UPDATE CUSTOMER SET C_BALANCE = C_BALANCE + ? WHERE C_ID = ? AND C_D_ID = ? AND C_W_ID = ?", # ol_total, c_id, d_id, w_id + "getNewOrder": "SELECT NO_O_ID FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID > -1 LIMIT 1", # + "deleteNewOrder": "DELETE FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID = ?", # d_id, w_id, no_o_id + "getCId": "SELECT O_C_ID FROM ORDERS WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ?", # no_o_id, d_id, w_id + "updateOrders": "UPDATE ORDERS SET O_CARRIER_ID = ? WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ?", # o_carrier_id, no_o_id, d_id, w_id + "updateOrderLine": "UPDATE ORDER_LINE SET OL_DELIVERY_D = ? WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ?", # o_entry_d, no_o_id, d_id, w_id + "sumOLAmount": "SELECT SUM(OL_AMOUNT) FROM ORDER_LINE WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ?", # no_o_id, d_id, w_id + "updateCustomer": "UPDATE CUSTOMER SET C_BALANCE = C_BALANCE + ? WHERE C_ID = ? AND C_D_ID = ? AND C_W_ID = ?", # ol_total, c_id, d_id, w_id }, "NEW_ORDER": { - "getWarehouseTaxRate": "SELECT W_TAX FROM WAREHOUSE WHERE W_ID = ?", # w_id - "getDistrict": "SELECT D_TAX, D_NEXT_O_ID FROM DISTRICT WHERE D_ID = ? AND D_W_ID = ?", # d_id, w_id - "incrementNextOrderId": "UPDATE DISTRICT SET D_NEXT_O_ID = ? WHERE D_ID = ? AND D_W_ID = ?", # d_next_o_id, d_id, w_id - "getCustomer": "SELECT C_DISCOUNT, C_LAST, C_CREDIT FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id - "createOrder": "INSERT INTO ORDERS (O_ID, O_D_ID, O_W_ID, O_C_ID, O_ENTRY_D, O_CARRIER_ID, O_OL_CNT, O_ALL_LOCAL) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", # d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, o_ol_cnt, o_all_local - "createNewOrder": "INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, NO_W_ID) VALUES (?, ?, ?)", # o_id, d_id, w_id - "getItemInfo": "SELECT I_PRICE, I_NAME, I_DATA FROM ITEM WHERE I_ID = ?", # ol_i_id - "getStockInfo": "SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DIST_%02d FROM STOCK WHERE S_I_ID = ? AND S_W_ID = ?", # d_id, ol_i_id, ol_supply_w_id - "updateStock": "UPDATE STOCK SET S_QUANTITY = ?, S_YTD = ?, S_ORDER_CNT = ?, S_REMOTE_CNT = ? WHERE S_I_ID = ? AND S_W_ID = ?", # s_quantity, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id - "createOrderLine": "INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", # o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info + "getWarehouseTaxRate": "SELECT W_TAX FROM WAREHOUSE WHERE W_ID = ?", # w_id + "getDistrict": "SELECT D_TAX, D_NEXT_O_ID FROM DISTRICT WHERE D_ID = ? AND D_W_ID = ?", # d_id, w_id + "incrementNextOrderId": "UPDATE DISTRICT SET D_NEXT_O_ID = ? WHERE D_ID = ? AND D_W_ID = ?", # d_next_o_id, d_id, w_id + "getCustomer": "SELECT C_DISCOUNT, C_LAST, C_CREDIT FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id + "createOrder": "INSERT INTO ORDERS (O_ID, O_D_ID, O_W_ID, O_C_ID, O_ENTRY_D, O_CARRIER_ID, O_OL_CNT, O_ALL_LOCAL) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", # d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, o_ol_cnt, o_all_local + "createNewOrder": "INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, NO_W_ID) VALUES (?, ?, ?)", # o_id, d_id, w_id + "getItemInfo": "SELECT I_PRICE, I_NAME, I_DATA FROM ITEM WHERE I_ID = ?", # ol_i_id + "getStockInfo": "SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DIST_%02d FROM STOCK WHERE S_I_ID = ? AND S_W_ID = ?", # d_id, ol_i_id, ol_supply_w_id + "updateStock": "UPDATE STOCK SET S_QUANTITY = ?, S_YTD = ?, S_ORDER_CNT = ?, S_REMOTE_CNT = ? WHERE S_I_ID = ? AND S_W_ID = ?", # s_quantity, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id + "createOrderLine": "INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", # o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info }, - "ORDER_STATUS": { - "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id - "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last - "getLastOrder": "SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS WHERE O_W_ID = ? AND O_D_ID = ? AND O_C_ID = ? ORDER BY O_ID DESC LIMIT 1", # w_id, d_id, c_id - "getOrderLines": "SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID = ?", # w_id, d_id, o_id + "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id + "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last + "getLastOrder": "SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS WHERE O_W_ID = ? AND O_D_ID = ? AND O_C_ID = ? ORDER BY O_ID DESC LIMIT 1", # w_id, d_id, c_id + "getOrderLines": "SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID = ?", # w_id, d_id, o_id }, - "PAYMENT": { - "getWarehouse": "SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP FROM WAREHOUSE WHERE W_ID = ?", # w_id - "updateWarehouseBalance": "UPDATE WAREHOUSE SET W_YTD = W_YTD + ? WHERE W_ID = ?", # h_amount, w_id - "getDistrict": "SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", # w_id, d_id - "updateDistrictBalance": "UPDATE DISTRICT SET D_YTD = D_YTD + ? WHERE D_W_ID = ? AND D_ID = ?", # h_amount, d_w_id, d_id - "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id - "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last - "updateBCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ?, C_DATA = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id - "updateGCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id + "getWarehouse": "SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP FROM WAREHOUSE WHERE W_ID = ?", # w_id + "updateWarehouseBalance": "UPDATE WAREHOUSE SET W_YTD = W_YTD + ? WHERE W_ID = ?", # h_amount, w_id + "getDistrict": "SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", # w_id, d_id + "updateDistrictBalance": "UPDATE DISTRICT SET D_YTD = D_YTD + ? WHERE D_W_ID = ? AND D_ID = ?", # h_amount, d_w_id, d_id + "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id + "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last + "updateBCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ?, C_DATA = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id + "updateGCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id "insertHistory": "INSERT INTO HISTORY VALUES (?, ?, ?, ?, ?, ?, ?, ?)", }, - "STOCK_LEVEL": { - "getOId": "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", + "getOId": "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", "getStockCount": """ SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK WHERE OL_W_ID = ? @@ -103,34 +100,37 @@ ## ============================================== class SqliteDriver(AbstractDriver): DEFAULT_CONFIG = { - "database": ("The path to the SQLite database", "/tmp/tpcc.db" ), + "database": ("The path to the SQLite database", "/tmp/tpcc.db"), } - + def __init__(self, ddl): super(SqliteDriver, self).__init__("sqlite", ddl) self.database = None self.conn = None self.cursor = None - + ## ---------------------------------------------- ## makeDefaultConfig ## ---------------------------------------------- def makeDefaultConfig(self): return SqliteDriver.DEFAULT_CONFIG - + ## ---------------------------------------------- ## loadConfig ## ---------------------------------------------- def loadConfig(self, config): for key in SqliteDriver.DEFAULT_CONFIG.keys(): - assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) - + assert key in config, "Missing parameter '%s' in %s configuration" % ( + key, + self.name, + ) + self.database = str(config["database"]) - + if config["reset"] and os.path.exists(self.database): logging.debug("Deleting database '%s'" % self.database) os.unlink(self.database) - + if os.path.exists(self.database) == False: logging.debug("Loading DDL file '%s'" % (self.ddl)) ## HACK @@ -138,20 +138,21 @@ def loadConfig(self, config): (result, output) = commands.getstatusoutput(cmd) assert result == 0, cmd + "\n" + output ## IF - + self.conn = sqlite3.connect(self.database) self.cursor = self.conn.cursor() - + ## ---------------------------------------------- ## loadTuples ## ---------------------------------------------- def loadTuples(self, tableName, tuples): - if len(tuples) == 0: return - - p = ["?"]*len(tuples[0]) + if len(tuples) == 0: + return + + p = ["?"] * len(tuples[0]) sql = "INSERT INTO %s VALUES (%s)" % (tableName, ",".join(p)) self.cursor.executemany(sql, tuples) - + logging.debug("Loaded %d tuples for tableName %s" % (len(tuples), tableName)) return @@ -167,13 +168,13 @@ def loadFinish(self): ## ---------------------------------------------- def doDelivery(self, params): q = TXN_QUERIES["DELIVERY"] - + w_id = params["w_id"] o_carrier_id = params["o_carrier_id"] ol_delivery_d = params["ol_delivery_d"] - result = [ ] - for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): + result = [] + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1): self.cursor.execute(q["getNewOrder"], [d_id, w_id]) newOrder = self.cursor.fetchone() if newOrder == None: @@ -181,22 +182,26 @@ def doDelivery(self, params): continue assert len(newOrder) > 0 no_o_id = newOrder[0] - + self.cursor.execute(q["getCId"], [no_o_id, d_id, w_id]) c_id = self.cursor.fetchone()[0] - + self.cursor.execute(q["sumOLAmount"], [no_o_id, d_id, w_id]) ol_total = self.cursor.fetchone()[0] self.cursor.execute(q["deleteNewOrder"], [d_id, w_id, no_o_id]) self.cursor.execute(q["updateOrders"], [o_carrier_id, no_o_id, d_id, w_id]) - self.cursor.execute(q["updateOrderLine"], [ol_delivery_d, no_o_id, d_id, w_id]) + self.cursor.execute( + q["updateOrderLine"], [ol_delivery_d, no_o_id, d_id, w_id] + ) # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure # them out # If there are no order lines, SUM returns null. There should always be order lines. - assert ol_total != None, "ol_total is NULL: there are no order lines. This should not happen" + assert ( + ol_total != None + ), "ol_total is NULL: there are no order lines. This should not happen" assert ol_total > 0.0 self.cursor.execute(q["updateCustomer"], [ol_total, c_id, d_id, w_id]) @@ -212,7 +217,7 @@ def doDelivery(self, params): ## ---------------------------------------------- def doNewOrder(self, params): q = TXN_QUERIES["NEW_ORDER"] - + w_id = params["w_id"] d_id = params["d_id"] c_id = params["c_id"] @@ -220,20 +225,20 @@ def doNewOrder(self, params): i_ids = params["i_ids"] i_w_ids = params["i_w_ids"] i_qtys = params["i_qtys"] - + assert len(i_ids) > 0 assert len(i_ids) == len(i_w_ids) assert len(i_ids) == len(i_qtys) all_local = True - items = [ ] + items = [] for i in range(len(i_ids)): ## Determine if this is an all local order or not all_local = all_local and i_w_ids[i] == w_id self.cursor.execute(q["getItemInfo"], [i_ids[i]]) items.append(self.cursor.fetchone()) assert len(items) == len(i_ids) - + ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. ## Note that this will happen with 1% of transactions on purpose. for item in items: @@ -241,18 +246,18 @@ def doNewOrder(self, params): ## TODO Abort here! return ## FOR - + ## ---------------- ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER ## ---------------- self.cursor.execute(q["getWarehouseTaxRate"], [w_id]) w_tax = self.cursor.fetchone()[0] - + self.cursor.execute(q["getDistrict"], [d_id, w_id]) district_info = self.cursor.fetchone() d_tax = district_info[0] d_next_o_id = district_info[1] - + self.cursor.execute(q["getCustomer"], [w_id, d_id, c_id]) customer_info = self.cursor.fetchone() c_discount = customer_info[0] @@ -262,15 +267,18 @@ def doNewOrder(self, params): ## ---------------- ol_cnt = len(i_ids) o_carrier_id = constants.NULL_CARRIER_ID - + self.cursor.execute(q["incrementNextOrderId"], [d_next_o_id + 1, d_id, w_id]) - self.cursor.execute(q["createOrder"], [d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, ol_cnt, all_local]) + self.cursor.execute( + q["createOrder"], + [d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, ol_cnt, all_local], + ) self.cursor.execute(q["createNewOrder"], [d_next_o_id, d_id, w_id]) ## ---------------- ## Insert Order Item Information ## ---------------- - item_data = [ ] + item_data = [] total = 0 for i in range(len(i_ids)): ol_number = i + 1 @@ -286,14 +294,17 @@ def doNewOrder(self, params): self.cursor.execute(q["getStockInfo"] % (d_id), [ol_i_id, ol_supply_w_id]) stockInfo = self.cursor.fetchone() if len(stockInfo) == 0: - logging.warn("No STOCK record for (ol_i_id=%d, ol_supply_w_id=%d)" % (ol_i_id, ol_supply_w_id)) + logging.warn( + "No STOCK record for (ol_i_id=%d, ol_supply_w_id=%d)" + % (ol_i_id, ol_supply_w_id) + ) continue s_quantity = stockInfo[0] s_ytd = stockInfo[2] s_order_cnt = stockInfo[3] s_remote_cnt = stockInfo[4] s_data = stockInfo[1] - s_dist_xx = stockInfo[5] # Fetches data from the s_dist_[d_id] column + s_dist_xx = stockInfo[5] # Fetches data from the s_dist_[d_id] column ## Update stock s_ytd += ol_quantity @@ -302,51 +313,72 @@ def doNewOrder(self, params): else: s_quantity = s_quantity + 91 - ol_quantity s_order_cnt += 1 - - if ol_supply_w_id != w_id: s_remote_cnt += 1 - self.cursor.execute(q["updateStock"], [s_quantity, s_ytd, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id]) + if ol_supply_w_id != w_id: + s_remote_cnt += 1 + + self.cursor.execute( + q["updateStock"], + [s_quantity, s_ytd, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id], + ) - if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: - brand_generic = 'B' + if ( + i_data.find(constants.ORIGINAL_STRING) != -1 + and s_data.find(constants.ORIGINAL_STRING) != -1 + ): + brand_generic = "B" else: - brand_generic = 'G' + brand_generic = "G" ## Transaction profile states to use "ol_quantity * i_price" ol_amount = ol_quantity * i_price total += ol_amount - self.cursor.execute(q["createOrderLine"], [d_next_o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, o_entry_d, ol_quantity, ol_amount, s_dist_xx]) + self.cursor.execute( + q["createOrderLine"], + [ + d_next_o_id, + d_id, + w_id, + ol_number, + ol_i_id, + ol_supply_w_id, + o_entry_d, + ol_quantity, + ol_amount, + s_dist_xx, + ], + ) ## Add the info to be returned - item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) + item_data.append((i_name, s_quantity, brand_generic, i_price, ol_amount)) ## FOR - + ## Commit! self.conn.commit() ## Adjust the total for the discount - #print "c_discount:", c_discount, type(c_discount) - #print "w_tax:", w_tax, type(w_tax) - #print "d_tax:", d_tax, type(d_tax) + # print "c_discount:", c_discount, type(c_discount) + # print "w_tax:", w_tax, type(w_tax) + # print "d_tax:", d_tax, type(d_tax) total *= (1 - c_discount) * (1 + w_tax + d_tax) ## Pack up values the client is missing (see TPC-C 2.4.3.5) - misc = [ (w_tax, d_tax, d_next_o_id, total) ] - - return [ customer_info, misc, item_data ] + misc = [(w_tax, d_tax, d_next_o_id, total)] + + return [customer_info, misc, item_data] ## ---------------------------------------------- ## doOrderStatus ## ---------------------------------------------- def doOrderStatus(self, params): q = TXN_QUERIES["ORDER_STATUS"] - + w_id = params["w_id"] d_id = params["d_id"] c_id = params["c_id"] c_last = params["c_last"] - + assert w_id, pformat(params) assert d_id, pformat(params) @@ -359,7 +391,7 @@ def doOrderStatus(self, params): all_customers = self.cursor.fetchall() assert len(all_customers) > 0 namecnt = len(all_customers) - index = (namecnt-1)/2 + index = (namecnt - 1) / 2 customer = all_customers[index] c_id = customer[0] assert len(customer) > 0 @@ -371,14 +403,14 @@ def doOrderStatus(self, params): self.cursor.execute(q["getOrderLines"], [w_id, d_id, order[0]]) orderLines = self.cursor.fetchall() else: - orderLines = [ ] + orderLines = [] self.conn.commit() - return [ customer, order, orderLines ] + return [customer, order, orderLines] ## ---------------------------------------------- ## doPayment - ## ---------------------------------------------- + ## ---------------------------------------------- def doPayment(self, params): q = TXN_QUERIES["PAYMENT"] @@ -400,7 +432,7 @@ def doPayment(self, params): all_customers = self.cursor.fetchall() assert len(all_customers) > 0 namecnt = len(all_customers) - index = (namecnt-1)/2 + index = (namecnt - 1) / 2 customer = all_customers[index] c_id = customer[0] assert len(customer) > 0 @@ -411,27 +443,37 @@ def doPayment(self, params): self.cursor.execute(q["getWarehouse"], [w_id]) warehouse = self.cursor.fetchone() - + self.cursor.execute(q["getDistrict"], [w_id, d_id]) district = self.cursor.fetchone() - + self.cursor.execute(q["updateWarehouseBalance"], [h_amount, w_id]) self.cursor.execute(q["updateDistrictBalance"], [h_amount, w_id, d_id]) # Customer Credit Information if customer[11] == constants.BAD_CREDIT: newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) - c_data = (newData + "|" + c_data) - if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] - self.cursor.execute(q["updateBCCustomer"], [c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id]) + c_data = newData + "|" + c_data + if len(c_data) > constants.MAX_C_DATA: + c_data = c_data[: constants.MAX_C_DATA] + self.cursor.execute( + q["updateBCCustomer"], + [c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id], + ) else: c_data = "" - self.cursor.execute(q["updateGCCustomer"], [c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id]) + self.cursor.execute( + q["updateGCCustomer"], + [c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id], + ) # Concatenate w_name, four spaces, d_name h_data = "%s %s" % (warehouse[0], district[0]) # Create the history record - self.cursor.execute(q["insertHistory"], [c_id, c_d_id, c_w_id, d_id, w_id, h_date, h_amount, h_data]) + self.cursor.execute( + q["insertHistory"], + [c_id, c_d_id, c_w_id, d_id, w_id, h_date, h_amount, h_data], + ) self.conn.commit() @@ -443,28 +485,31 @@ def doPayment(self, params): # H_AMOUNT, and H_DATE. # Hand back all the warehouse, district, and customer data - return [ warehouse, district, customer ] - + return [warehouse, district, customer] + ## ---------------------------------------------- ## doStockLevel - ## ---------------------------------------------- + ## ---------------------------------------------- def doStockLevel(self, params): q = TXN_QUERIES["STOCK_LEVEL"] w_id = params["w_id"] d_id = params["d_id"] threshold = params["threshold"] - + self.cursor.execute(q["getOId"], [w_id, d_id]) result = self.cursor.fetchone() assert result o_id = result[0] - - self.cursor.execute(q["getStockCount"], [w_id, d_id, o_id, (o_id - 20), w_id, threshold]) + + self.cursor.execute( + q["getStockCount"], [w_id, d_id, o_id, (o_id - 20), w_id, threshold] + ) result = self.cursor.fetchone() - + self.conn.commit() - + return int(result[0]) - -## CLASS \ No newline at end of file + + +## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/message.py b/workloads/chbenchmark/py-tpcc/pytpcc/message.py index 8ef84ccc..336c64e2 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/message.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/message.py @@ -33,8 +33,8 @@ import re import argparse import glob -import time -from pprint import pprint,pformat +import time +from pprint import pprint, pformat from util import * from runtime import * @@ -47,6 +47,7 @@ LOAD_COMPLETED = 4 EXECUTE_COMPLETED = 5 + class Message: def __init__(self, header=EMPTY, data=None): self.header = header diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py index 8606b43e..af25ba8d 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/executor.py @@ -36,20 +36,21 @@ import traceback import logging from datetime import datetime -from pprint import pprint,pformat +from pprint import pprint, pformat import constants from util import * class Executor: - - def __init__(self, driver, scaleParameters, stop_on_error = False): + + def __init__(self, driver, scaleParameters, stop_on_error=False): self.driver = driver self.scaleParameters = scaleParameters self.stop_on_error = stop_on_error + ## DEF - + def execute(self, duration): r = results.Results() assert r @@ -60,50 +61,70 @@ def execute(self, duration): while (time.time() - start) <= duration: txn, params = self.doOne() txn_id = r.startTransaction(txn) - - if debug: logging.debug("Executing '%s' transaction" % txn) + + if debug: + logging.debug("Executing '%s' transaction" % txn) try: val = self.driver.executeTransaction(txn, params) except KeyboardInterrupt: return -1 except (Exception, AssertionError) as ex: logging.warn("Failed to execute Transaction '%s': %s" % (txn, ex)) - if debug: traceback.print_exc(file=sys.stdout) - if self.stop_on_error: raise + if debug: + traceback.print_exc(file=sys.stdout) + if self.stop_on_error: + raise r.abortTransaction(txn_id) continue - #if debug: logging.debug("%s\nParameters:\n%s\nResult:\n%s" % (txn, pformat(params), pformat(val))) - + # if debug: logging.debug("%s\nParameters:\n%s\nResult:\n%s" % (txn, pformat(params), pformat(val))) + r.stopTransaction(txn_id) ## WHILE - + r.stopBenchmark() - return (r) + return r + ## DEF - + def doOne(self): """Selects and executes a transaction at random. The number of new order transactions executed per minute is the official "tpmC" metric. See TPC-C 5.4.2 (page 71).""" - + ## This is not strictly accurate: The requirement is for certain ## *minimum* percentages to be maintained. This is close to the right ## thing, but not precisely correct. See TPC-C 5.2.4 (page 68). x = rand.number(1, 100) params = None txn = None - if x <= 4: ## 4% - txn, params = (constants.TransactionTypes.STOCK_LEVEL, self.generateStockLevelParams()) - elif x <= 4 + 4: ## 4% - txn, params = (constants.TransactionTypes.DELIVERY, self.generateDeliveryParams()) - elif x <= 4 + 4 + 4: ## 4% - txn, params = (constants.TransactionTypes.ORDER_STATUS, self.generateOrderStatusParams()) - elif x <= 43 + 4 + 4 + 4: ## 43% - txn, params = (constants.TransactionTypes.PAYMENT, self.generatePaymentParams()) - else: ## 45% + if x <= 4: ## 4% + txn, params = ( + constants.TransactionTypes.STOCK_LEVEL, + self.generateStockLevelParams(), + ) + elif x <= 4 + 4: ## 4% + txn, params = ( + constants.TransactionTypes.DELIVERY, + self.generateDeliveryParams(), + ) + elif x <= 4 + 4 + 4: ## 4% + txn, params = ( + constants.TransactionTypes.ORDER_STATUS, + self.generateOrderStatusParams(), + ) + elif x <= 43 + 4 + 4 + 4: ## 43% + txn, params = ( + constants.TransactionTypes.PAYMENT, + self.generatePaymentParams(), + ) + else: ## 45% assert x > 100 - 45 - txn, params = (constants.TransactionTypes.NEW_ORDER, self.generateNewOrderParams()) - + txn, params = ( + constants.TransactionTypes.NEW_ORDER, + self.generateNewOrderParams(), + ) + return (txn, params) + ## DEF ## ---------------------------------------------- @@ -115,6 +136,7 @@ def generateDeliveryParams(self): o_carrier_id = rand.number(constants.MIN_CARRIER_ID, constants.MAX_CARRIER_ID) ol_delivery_d = datetime.now() return makeParameterDict(locals(), "w_id", "o_carrier_id", "ol_delivery_d") + ## DEF ## ---------------------------------------------- @@ -129,11 +151,11 @@ def generateNewOrderParams(self): o_entry_d = datetime.now() ## 1% of transactions roll back - rollback = False # FIXME rand.number(1, 100) == 1 + rollback = False # FIXME rand.number(1, 100) == 1 - i_ids = [ ] - i_w_ids = [ ] - i_qtys = [ ] + i_ids = [] + i_w_ids = [] + i_qtys = [] for i in range(0, ol_cnt): if rollback and i + 1 == ol_cnt: i_ids.append(self.scaleParameters.items + 1) @@ -144,16 +166,25 @@ def generateNewOrderParams(self): i_ids.append(i_id) ## 1% of items are from a remote warehouse - remote = (rand.number(1, 100) == 1) + remote = rand.number(1, 100) == 1 if self.scaleParameters.warehouses > 1 and remote: - i_w_ids.append(rand.numberExcluding(self.scaleParameters.starting_warehouse, self.scaleParameters.ending_warehouse, w_id)) + i_w_ids.append( + rand.numberExcluding( + self.scaleParameters.starting_warehouse, + self.scaleParameters.ending_warehouse, + w_id, + ) + ) else: i_w_ids.append(w_id) i_qtys.append(rand.number(1, constants.MAX_OL_QUANTITY)) ## FOR - return makeParameterDict(locals(), "w_id", "d_id", "c_id", "o_entry_d", "i_ids", "i_w_ids", "i_qtys") + return makeParameterDict( + locals(), "w_id", "d_id", "c_id", "o_entry_d", "i_ids", "i_w_ids", "i_qtys" + ) + ## DEF ## ---------------------------------------------- @@ -165,7 +196,7 @@ def generateOrderStatusParams(self): d_id = self.makeDistrictId() c_last = None c_id = None - + ## 60%: order status by last name if rand.number(1, 100) <= 60: c_last = rand.makeRandomLastName(self.scaleParameters.customersPerDistrict) @@ -173,8 +204,9 @@ def generateOrderStatusParams(self): ## 40%: order status by id else: c_id = self.makeCustomerId() - + return makeParameterDict(locals(), "w_id", "d_id", "c_id", "c_last") + ## DEF ## ---------------------------------------------- @@ -201,7 +233,11 @@ def generatePaymentParams(self): ## 15%: paying through another warehouse: else: ## select in range [1, num_warehouses] excluding w_id - c_w_id = rand.numberExcluding(self.scaleParameters.starting_warehouse, self.scaleParameters.ending_warehouse, w_id) + c_w_id = rand.numberExcluding( + self.scaleParameters.starting_warehouse, + self.scaleParameters.ending_warehouse, + w_id, + ) assert c_w_id != w_id c_d_id = self.makeDistrictId() @@ -213,7 +249,18 @@ def generatePaymentParams(self): assert y > 60 c_id = self.makeCustomerId() - return makeParameterDict(locals(), "w_id", "d_id", "h_amount", "c_w_id", "c_d_id", "c_id", "c_last", "h_date") + return makeParameterDict( + locals(), + "w_id", + "d_id", + "h_amount", + "c_w_id", + "c_d_id", + "c_id", + "c_last", + "h_date", + ) + ## DEF ## ---------------------------------------------- @@ -223,30 +270,47 @@ def generateStockLevelParams(self): """Returns parameters for STOCK_LEVEL""" w_id = self.makeWarehouseId() d_id = self.makeDistrictId() - threshold = rand.number(constants.MIN_STOCK_LEVEL_THRESHOLD, constants.MAX_STOCK_LEVEL_THRESHOLD) + threshold = rand.number( + constants.MIN_STOCK_LEVEL_THRESHOLD, constants.MAX_STOCK_LEVEL_THRESHOLD + ) return makeParameterDict(locals(), "w_id", "d_id", "threshold") + ## DEF def makeWarehouseId(self): - w_id = rand.number(self.scaleParameters.starting_warehouse, self.scaleParameters.ending_warehouse) - assert(w_id >= self.scaleParameters.starting_warehouse), "Invalid W_ID: %d" % w_id - assert(w_id <= self.scaleParameters.ending_warehouse), "Invalid W_ID: %d" % w_id + w_id = rand.number( + self.scaleParameters.starting_warehouse, + self.scaleParameters.ending_warehouse, + ) + assert w_id >= self.scaleParameters.starting_warehouse, ( + "Invalid W_ID: %d" % w_id + ) + assert w_id <= self.scaleParameters.ending_warehouse, "Invalid W_ID: %d" % w_id return w_id + ## DEF def makeDistrictId(self): return rand.number(1, self.scaleParameters.districtsPerWarehouse) + ## DEF def makeCustomerId(self): return rand.NURand(1023, 1, self.scaleParameters.customersPerDistrict) + ## DEF def makeItemId(self): return rand.NURand(8191, 1, self.scaleParameters.items) + ## DEF + + ## CLASS + def makeParameterDict(values, *args): return dict(map(lambda x: (x, values[x]), args)) + + ## DEF diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/runtime/loader.py b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/loader.py index 1c9225e1..1a196f58 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/runtime/loader.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/runtime/loader.py @@ -35,120 +35,163 @@ import logging from datetime import datetime from random import shuffle -from pprint import pprint,pformat +from pprint import pprint, pformat import constants from util import * + class Loader: - + def __init__(self, handle, scaleParameters, w_ids, needLoadItems): self.handle = handle self.scaleParameters = scaleParameters self.w_ids = w_ids self.needLoadItems = needLoadItems self.batch_size = 2500 - + ## ============================================== ## execute ## ============================================== def execute(self): - + ## Item Table if self.needLoadItems: logging.debug("Loading ITEM table") self.loadItems() self.handle.loadFinishItem() - + ## Then create the warehouse-specific tuples for w_id in self.w_ids: self.loadWarehouse(w_id) self.handle.loadFinishWarehouse(w_id) ## FOR - - return (None) + + return None ## ============================================== ## loadItems ## ============================================== def loadItems(self): ## Select 10% of the rows to be marked "original" - originalRows = rand.selectUniqueIds(self.scaleParameters.items / 10, 1, self.scaleParameters.items) - + originalRows = rand.selectUniqueIds( + self.scaleParameters.items / 10, 1, self.scaleParameters.items + ) + ## Load all of the items - tuples = [ ] + tuples = [] total_tuples = 0 - for i in range(1, self.scaleParameters.items+1): - original = (i in originalRows) + for i in range(1, self.scaleParameters.items + 1): + original = i in originalRows tuples.append(self.generateItem(i, original)) total_tuples += 1 if len(tuples) == self.batch_size: - logging.debug("LOAD - %s: %5d / %d" % (constants.TABLENAME_ITEM, total_tuples, self.scaleParameters.items)) + logging.debug( + "LOAD - %s: %5d / %d" + % ( + constants.TABLENAME_ITEM, + total_tuples, + self.scaleParameters.items, + ) + ) self.handle.loadTuples(constants.TABLENAME_ITEM, tuples) - tuples = [ ] + tuples = [] ## FOR if len(tuples) > 0: - logging.debug("LOAD - %s: %5d / %d" % (constants.TABLENAME_ITEM, total_tuples, self.scaleParameters.items)) + logging.debug( + "LOAD - %s: %5d / %d" + % (constants.TABLENAME_ITEM, total_tuples, self.scaleParameters.items) + ) self.handle.loadTuples(constants.TABLENAME_ITEM, tuples) + ## DEF ## ============================================== ## loadWarehouse ## ============================================== def loadWarehouse(self, w_id): - logging.debug("LOAD - %s: %d / %d" % (constants.TABLENAME_WAREHOUSE, w_id, len(self.w_ids))) - + logging.debug( + "LOAD - %s: %d / %d" + % (constants.TABLENAME_WAREHOUSE, w_id, len(self.w_ids)) + ) + ## WAREHOUSE - w_tuples = [ self.generateWarehouse(w_id) ] + w_tuples = [self.generateWarehouse(w_id)] self.handle.loadTuples(constants.TABLENAME_WAREHOUSE, w_tuples) ## DISTRICT - d_tuples = [ ] - for d_id in range(1, self.scaleParameters.districtsPerWarehouse+1): + d_tuples = [] + for d_id in range(1, self.scaleParameters.districtsPerWarehouse + 1): d_next_o_id = self.scaleParameters.customersPerDistrict + 1 - d_tuples = [ self.generateDistrict(w_id, d_id, d_next_o_id) ] - - c_tuples = [ ] - h_tuples = [ ] - + d_tuples = [self.generateDistrict(w_id, d_id, d_next_o_id)] + + c_tuples = [] + h_tuples = [] + ## Select 10% of the customers to have bad credit - selectedRows = rand.selectUniqueIds(self.scaleParameters.customersPerDistrict / 10, 1, self.scaleParameters.customersPerDistrict) - + selectedRows = rand.selectUniqueIds( + self.scaleParameters.customersPerDistrict / 10, + 1, + self.scaleParameters.customersPerDistrict, + ) + ## TPC-C 4.3.3.1. says that o_c_id should be a permutation of [1, 3000]. But since it ## is a c_id field, it seems to make sense to have it be a permutation of the ## customers. For the "real" thing this will be equivalent - cIdPermutation = [ ] + cIdPermutation = [] - for c_id in range(1, self.scaleParameters.customersPerDistrict+1): - badCredit = (c_id in selectedRows) - c_tuples.append(self.generateCustomer(w_id, d_id, c_id, badCredit, True)) + for c_id in range(1, self.scaleParameters.customersPerDistrict + 1): + badCredit = c_id in selectedRows + c_tuples.append( + self.generateCustomer(w_id, d_id, c_id, badCredit, True) + ) h_tuples.append(self.generateHistory(w_id, d_id, c_id)) cIdPermutation.append(c_id) ## FOR assert cIdPermutation[0] == 1 - assert cIdPermutation[self.scaleParameters.customersPerDistrict - 1] == self.scaleParameters.customersPerDistrict + assert ( + cIdPermutation[self.scaleParameters.customersPerDistrict - 1] + == self.scaleParameters.customersPerDistrict + ) shuffle(cIdPermutation) - - o_tuples = [ ] - ol_tuples = [ ] - no_tuples = [ ] - - for o_id in range(1, self.scaleParameters.customersPerDistrict+1): + + o_tuples = [] + ol_tuples = [] + no_tuples = [] + + for o_id in range(1, self.scaleParameters.customersPerDistrict + 1): o_ol_cnt = rand.number(constants.MIN_OL_CNT, constants.MAX_OL_CNT) - + ## The last newOrdersPerDistrict are new orders - newOrder = ((self.scaleParameters.customersPerDistrict - self.scaleParameters.newOrdersPerDistrict) < o_id) - o_tuples.append(self.generateOrder(w_id, d_id, o_id, cIdPermutation[o_id - 1], o_ol_cnt, newOrder)) + newOrder = ( + self.scaleParameters.customersPerDistrict + - self.scaleParameters.newOrdersPerDistrict + ) < o_id + o_tuples.append( + self.generateOrder( + w_id, d_id, o_id, cIdPermutation[o_id - 1], o_ol_cnt, newOrder + ) + ) ## Generate each OrderLine for the order for ol_number in range(0, o_ol_cnt): - ol_tuples.append(self.generateOrderLine(w_id, d_id, o_id, ol_number, self.scaleParameters.items, newOrder)) + ol_tuples.append( + self.generateOrderLine( + w_id, + d_id, + o_id, + ol_number, + self.scaleParameters.items, + newOrder, + ) + ) ## FOR ## This is a new order: make one for it - if newOrder: no_tuples.append([o_id, d_id, w_id]) + if newOrder: + no_tuples.append([o_id, d_id, w_id]) ## FOR - + self.handle.loadTuples(constants.TABLENAME_DISTRICT, d_tuples) self.handle.loadTuples(constants.TABLENAME_CUSTOMER, c_tuples) self.handle.loadTuples(constants.TABLENAME_ORDERS, o_tuples) @@ -157,23 +200,42 @@ def loadWarehouse(self, w_id): self.handle.loadTuples(constants.TABLENAME_HISTORY, h_tuples) self.handle.loadFinishDistrict(w_id, d_id) ## FOR - + ## Select 10% of the stock to be marked "original" - s_tuples = [ ] - selectedRows = rand.selectUniqueIds(self.scaleParameters.items / 10, 1, self.scaleParameters.items) + s_tuples = [] + selectedRows = rand.selectUniqueIds( + self.scaleParameters.items / 10, 1, self.scaleParameters.items + ) total_tuples = 0 - for i_id in range(1, self.scaleParameters.items+1): - original = (i_id in selectedRows) + for i_id in range(1, self.scaleParameters.items + 1): + original = i_id in selectedRows s_tuples.append(self.generateStock(w_id, i_id, original)) if len(s_tuples) >= self.batch_size: - logging.debug("LOAD - %s [W_ID=%d]: %5d / %d" % (constants.TABLENAME_STOCK, w_id, total_tuples, self.scaleParameters.items)) + logging.debug( + "LOAD - %s [W_ID=%d]: %5d / %d" + % ( + constants.TABLENAME_STOCK, + w_id, + total_tuples, + self.scaleParameters.items, + ) + ) self.handle.loadTuples(constants.TABLENAME_STOCK, s_tuples) - s_tuples = [ ] + s_tuples = [] total_tuples += 1 ## FOR if len(s_tuples) > 0: - logging.debug("LOAD - %s [W_ID=%d]: %5d / %d" % (constants.TABLENAME_STOCK, w_id, total_tuples, self.scaleParameters.items)) + logging.debug( + "LOAD - %s [W_ID=%d]: %5d / %d" + % ( + constants.TABLENAME_STOCK, + w_id, + total_tuples, + self.scaleParameters.items, + ) + ) self.handle.loadTuples(constants.TABLENAME_STOCK, s_tuples) + ## DEF ## ============================================== @@ -183,11 +245,15 @@ def generateItem(self, id, original): i_id = id i_im_id = rand.number(constants.MIN_IM, constants.MAX_IM) i_name = rand.astring(constants.MIN_I_NAME, constants.MAX_I_NAME) - i_price = rand.fixedPoint(constants.MONEY_DECIMALS, constants.MIN_PRICE, constants.MAX_PRICE) + i_price = rand.fixedPoint( + constants.MONEY_DECIMALS, constants.MIN_PRICE, constants.MAX_PRICE + ) i_data = rand.astring(constants.MIN_I_DATA, constants.MAX_I_DATA) - if original: i_data = self.fillOriginal(i_data) + if original: + i_data = self.fillOriginal(i_data) return [i_id, i_im_id, i_name, i_price, i_data] + ## DEF ## ============================================== @@ -198,6 +264,7 @@ def generateWarehouse(self, w_id): w_ytd = constants.INITIAL_W_YTD w_address = self.generateAddress() return [w_id] + w_address + [w_tax, w_ytd] + ## DEF ## ============================================== @@ -208,6 +275,7 @@ def generateDistrict(self, d_w_id, d_id, d_next_o_id): d_ytd = constants.INITIAL_D_YTD d_address = self.generateAddress() return [d_id, d_w_id] + d_address + [d_tax, d_ytd, d_next_o_id] + ## DEF ## ============================================== @@ -227,7 +295,9 @@ def generateCustomer(self, c_w_id, c_d_id, c_id, badCredit, doesReplicateName): c_since = datetime.now() c_credit = constants.BAD_CREDIT if badCredit else constants.GOOD_CREDIT c_credit_lim = constants.INITIAL_CREDIT_LIM - c_discount = rand.fixedPoint(constants.DISCOUNT_DECIMALS, constants.MIN_DISCOUNT, constants.MAX_DISCOUNT) + c_discount = rand.fixedPoint( + constants.DISCOUNT_DECIMALS, constants.MIN_DISCOUNT, constants.MAX_DISCOUNT + ) c_balance = constants.INITIAL_BALANCE c_ytd_payment = constants.INITIAL_YTD_PAYMENT c_payment_cnt = constants.INITIAL_PAYMENT_CNT @@ -240,10 +310,30 @@ def generateCustomer(self, c_w_id, c_d_id, c_id, badCredit, doesReplicateName): c_state = rand.astring(constants.STATE, constants.STATE) c_zip = self.generateZip() - return [ c_id, c_d_id, c_w_id, c_first, c_middle, c_last, \ - c_street1, c_street2, c_city, c_state, c_zip, \ - c_phone, c_since, c_credit, c_credit_lim, c_discount, c_balance, \ - c_ytd_payment, c_payment_cnt, c_delivery_cnt, c_data ] + return [ + c_id, + c_d_id, + c_w_id, + c_first, + c_middle, + c_last, + c_street1, + c_street2, + c_city, + c_state, + c_zip, + c_phone, + c_since, + c_credit, + c_credit_lim, + c_discount, + c_balance, + c_ytd_payment, + c_payment_cnt, + c_delivery_cnt, + c_data, + ] + ## DEF ## ============================================== @@ -252,56 +342,94 @@ def generateCustomer(self, c_w_id, c_d_id, c_id, badCredit, doesReplicateName): def generateOrder(self, o_w_id, o_d_id, o_id, o_c_id, o_ol_cnt, newOrder): """Returns the generated o_ol_cnt value.""" o_entry_d = datetime.now() - o_carrier_id = constants.NULL_CARRIER_ID if newOrder else rand.number(constants.MIN_CARRIER_ID, constants.MAX_CARRIER_ID) + o_carrier_id = ( + constants.NULL_CARRIER_ID + if newOrder + else rand.number(constants.MIN_CARRIER_ID, constants.MAX_CARRIER_ID) + ) o_all_local = constants.INITIAL_ALL_LOCAL - return [ o_id, o_c_id, o_d_id, o_w_id, o_entry_d, o_carrier_id, o_ol_cnt, o_all_local ] + return [ + o_id, + o_c_id, + o_d_id, + o_w_id, + o_entry_d, + o_carrier_id, + o_ol_cnt, + o_all_local, + ] + ## DEF ## ============================================== ## generateOrderLine ## ============================================== - def generateOrderLine(self, ol_w_id, ol_d_id, ol_o_id, ol_number, max_items, newOrder): + def generateOrderLine( + self, ol_w_id, ol_d_id, ol_o_id, ol_number, max_items, newOrder + ): ol_i_id = rand.number(1, max_items) ol_supply_w_id = ol_w_id ol_delivery_d = datetime.now() ol_quantity = constants.INITIAL_QUANTITY ## 1% of items are from a remote warehouse - remote = (rand.number(1, 100) == 1) + remote = rand.number(1, 100) == 1 if self.scaleParameters.warehouses > 1 and remote: - ol_supply_w_id = rand.numberExcluding(self.scaleParameters.starting_warehouse, - self.scaleParameters.ending_warehouse, - ol_w_id) + ol_supply_w_id = rand.numberExcluding( + self.scaleParameters.starting_warehouse, + self.scaleParameters.ending_warehouse, + ol_w_id, + ) if newOrder == False: ol_amount = 0.00 else: - ol_amount = rand.fixedPoint(constants.MONEY_DECIMALS, constants.MIN_AMOUNT, constants.MAX_PRICE * constants.MAX_OL_QUANTITY) + ol_amount = rand.fixedPoint( + constants.MONEY_DECIMALS, + constants.MIN_AMOUNT, + constants.MAX_PRICE * constants.MAX_OL_QUANTITY, + ) ol_delivery_d = None ol_dist_info = rand.astring(constants.DIST, constants.DIST) - return [ ol_o_id, ol_d_id, ol_w_id, ol_number, ol_i_id, ol_supply_w_id, ol_delivery_d, ol_quantity, ol_amount, ol_dist_info ] + return [ + ol_o_id, + ol_d_id, + ol_w_id, + ol_number, + ol_i_id, + ol_supply_w_id, + ol_delivery_d, + ol_quantity, + ol_amount, + ol_dist_info, + ] + ## DEF ## ============================================== ## generateStock ## ============================================== def generateStock(self, s_w_id, s_i_id, original): - s_quantity = rand.number(constants.MIN_QUANTITY, constants.MAX_QUANTITY); - s_ytd = 0; - s_order_cnt = 0; - s_remote_cnt = 0; + s_quantity = rand.number(constants.MIN_QUANTITY, constants.MAX_QUANTITY) + s_ytd = 0 + s_order_cnt = 0 + s_remote_cnt = 0 - s_data = rand.astring(constants.MIN_I_DATA, constants.MAX_I_DATA); - if original: self.fillOriginal(s_data) + s_data = rand.astring(constants.MIN_I_DATA, constants.MAX_I_DATA) + if original: + self.fillOriginal(s_data) - s_dists = [ ] + s_dists = [] for i in range(0, constants.DISTRICTS_PER_WAREHOUSE): s_dists.append(rand.astring(constants.DIST, constants.DIST)) - - return [ s_i_id, s_w_id, s_quantity ] + \ - s_dists + \ - [ s_ytd, s_order_cnt, s_remote_cnt, s_data ] + + return ( + [s_i_id, s_w_id, s_quantity] + + s_dists + + [s_ytd, s_order_cnt, s_remote_cnt, s_data] + ) + ## DEF ## ============================================== @@ -313,7 +441,8 @@ def generateHistory(self, h_c_w_id, h_c_d_id, h_c_id): h_date = datetime.now() h_amount = constants.INITIAL_AMOUNT h_data = rand.astring(constants.MIN_DATA, constants.MAX_DATA) - return [ h_c_id, h_c_d_id, h_c_w_id, h_d_id, h_w_id, h_date, h_amount, h_data ] + return [h_c_id, h_c_d_id, h_c_w_id, h_d_id, h_w_id, h_date, h_amount, h_data] + ## DEF ## ============================================== @@ -321,11 +450,12 @@ def generateHistory(self, h_c_w_id, h_c_d_id, h_c_id): ## ============================================== def generateAddress(self): """ - Returns a name and a street address - Used by both generateWarehouse and generateDistrict. + Returns a name and a street address + Used by both generateWarehouse and generateDistrict. """ name = rand.astring(constants.MIN_NAME, constants.MAX_NAME) - return [ name ] + self.generateStreetAddress() + return [name] + self.generateStreetAddress() + ## DEF ## ============================================== @@ -333,8 +463,8 @@ def generateAddress(self): ## ============================================== def generateStreetAddress(self): """ - Returns a list for a street address - Used for warehouses, districts and customers. + Returns a list for a street address + Used for warehouses, districts and customers. """ street1 = rand.astring(constants.MIN_STREET, constants.MAX_STREET) street2 = rand.astring(constants.MIN_STREET, constants.MAX_STREET) @@ -342,14 +472,18 @@ def generateStreetAddress(self): state = rand.astring(constants.STATE, constants.STATE) zip = self.generateZip() - return [ street1, street2, city, state, zip ] + return [street1, street2, city, state, zip] + ## DEF ## ============================================== ## generateTax ## ============================================== def generateTax(self): - return rand.fixedPoint(constants.TAX_DECIMALS, constants.MIN_TAX, constants.MAX_TAX) + return rand.fixedPoint( + constants.TAX_DECIMALS, constants.MIN_TAX, constants.MAX_TAX + ) + ## DEF ## ============================================== @@ -358,6 +492,7 @@ def generateTax(self): def generateZip(self): length = constants.ZIP_LENGTH - len(constants.ZIP_SUFFIX) return rand.nstring(length, length) + constants.ZIP_SUFFIX + ## DEF ## ============================================== @@ -365,12 +500,19 @@ def generateZip(self): ## ============================================== def fillOriginal(self, data): """ - a string with ORIGINAL_STRING at a random position + a string with ORIGINAL_STRING at a random position """ originalLength = len(constants.ORIGINAL_STRING) position = rand.number(0, len(data) - originalLength) - out = data[:position] + constants.ORIGINAL_STRING + data[position + originalLength:] + out = ( + data[:position] + + constants.ORIGINAL_STRING + + data[position + originalLength :] + ) assert len(out) == len(data) return out + ## DEF + + ## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py b/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py index dfde3e29..923bf6fe 100755 --- a/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/tpcc.py @@ -33,222 +33,308 @@ import re import argparse import glob -import time +import time import multiprocessing from ConfigParser import SafeConfigParser -from pprint import pprint,pformat +from pprint import pprint, pformat from util import * from runtime import * import drivers -logging.basicConfig(level = logging.INFO, - format="%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s", - datefmt="%m-%d-%Y %H:%M:%S", - stream = sys.stdout) - +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s", + datefmt="%m-%d-%Y %H:%M:%S", + stream=sys.stdout, +) + + ## ============================================== ## createDriverClass ## ============================================== def createDriverClass(name): full_name = "%sDriver" % name.title() - mod = __import__('drivers.%s' % full_name.lower(), globals(), locals(), [full_name]) + mod = __import__("drivers.%s" % full_name.lower(), globals(), locals(), [full_name]) klass = getattr(mod, full_name) return klass + + ## DEF + ## ============================================== ## getDrivers ## ============================================== def getDrivers(): - drivers = [ ] - for f in map(lambda x: os.path.basename(x).replace("driver.py", ""), glob.glob("./drivers/*driver.py")): - if f != "abstract": drivers.append(f) - return (drivers) + drivers = [] + for f in map( + lambda x: os.path.basename(x).replace("driver.py", ""), + glob.glob("./drivers/*driver.py"), + ): + if f != "abstract": + drivers.append(f) + return drivers + + ## DEF + ## ============================================== ## startLoading ## ============================================== def startLoading(driverClass, scaleParameters, args, config): - logging.debug("Creating client pool with %d processes" % args['clients']) - pool = multiprocessing.Pool(args['clients']) + logging.debug("Creating client pool with %d processes" % args["clients"]) + pool = multiprocessing.Pool(args["clients"]) debug = logging.getLogger().isEnabledFor(logging.DEBUG) - + # Split the warehouses into chunks - w_ids = map(lambda x: [ ], range(args['clients'])) - for w_id in range(scaleParameters.starting_warehouse, scaleParameters.ending_warehouse+1): - idx = w_id % args['clients'] + w_ids = map(lambda x: [], range(args["clients"])) + for w_id in range( + scaleParameters.starting_warehouse, scaleParameters.ending_warehouse + 1 + ): + idx = w_id % args["clients"] w_ids[idx].append(w_id) ## FOR - - loader_results = [ ] - for i in range(args['clients']): - r = pool.apply_async(loaderFunc, (driverClass, scaleParameters, args, config, w_ids[i], True)) + + loader_results = [] + for i in range(args["clients"]): + r = pool.apply_async( + loaderFunc, (driverClass, scaleParameters, args, config, w_ids[i], True) + ) loader_results.append(r) ## FOR - + pool.close() - logging.debug("Waiting for %d loaders to finish" % args['clients']) + logging.debug("Waiting for %d loaders to finish" % args["clients"]) pool.join() + + ## DEF + ## ============================================== ## loaderFunc ## ============================================== def loaderFunc(driverClass, scaleParameters, args, config, w_ids, debug): - driver = driverClass(args['ddl']) + driver = driverClass(args["ddl"]) assert driver != None - logging.debug("Starting client execution: %s [warehouses=%d]" % (driver, len(w_ids))) - - config['load'] = True - config['execute'] = False - config['reset'] = False + logging.debug( + "Starting client execution: %s [warehouses=%d]" % (driver, len(w_ids)) + ) + + config["load"] = True + config["execute"] = False + config["reset"] = False driver.loadConfig(config) - + try: - loadItems = (1 in w_ids) + loadItems = 1 in w_ids l = loader.Loader(driver, scaleParameters, w_ids, loadItems) driver.loadStart() l.execute() - driver.loadFinish() + driver.loadFinish() except KeyboardInterrupt: - return -1 + return -1 except (Exception, AssertionError) as ex: logging.warn("Failed to load data: %s" % (ex)) - #if debug: + # if debug: traceback.print_exc(file=sys.stdout) raise - + + ## DEF + ## ============================================== ## startExecution ## ============================================== def startExecution(driverClass, scaleParameters, args, config): - logging.debug("Creating client pool with %d processes" % args['clients']) - pool = multiprocessing.Pool(args['clients']) + logging.debug("Creating client pool with %d processes" % args["clients"]) + pool = multiprocessing.Pool(args["clients"]) debug = logging.getLogger().isEnabledFor(logging.DEBUG) - - worker_results = [ ] - for i in range(args['clients']): - r = pool.apply_async(executorFunc, (driverClass, scaleParameters, args, config, debug,)) + + worker_results = [] + for i in range(args["clients"]): + r = pool.apply_async( + executorFunc, + ( + driverClass, + scaleParameters, + args, + config, + debug, + ), + ) worker_results.append(r) ## FOR pool.close() pool.join() - + total_results = results.Results() for asyncr in worker_results: asyncr.wait() r = asyncr.get() assert r != None, "No results object returned!" - if type(r) == int and r == -1: sys.exit(1) + if type(r) == int and r == -1: + sys.exit(1) total_results.append(r) ## FOR - - return (total_results) + + return total_results + + ## DEF + ## ============================================== ## executorFunc ## ============================================== def executorFunc(driverClass, scaleParameters, args, config, debug): - driver = driverClass(args['ddl']) + driver = driverClass(args["ddl"]) assert driver != None logging.debug("Starting client execution: %s" % driver) - - config['execute'] = True - config['reset'] = False + + config["execute"] = True + config["reset"] = False driver.loadConfig(config) - e = executor.Executor(driver, scaleParameters, stop_on_error=args['stop_on_error']) + e = executor.Executor(driver, scaleParameters, stop_on_error=args["stop_on_error"]) driver.executeStart() - results = e.execute(args['duration']) + results = e.execute(args["duration"]) driver.executeFinish() - + return results + + ## DEF ## ============================================== ## main ## ============================================== -if __name__ == '__main__': - aparser = argparse.ArgumentParser(description='Python implementation of the TPC-C Benchmark') - aparser.add_argument('system', choices=getDrivers(), - help='Target system driver') - aparser.add_argument('--config', type=file, - help='Path to driver configuration file') - aparser.add_argument('--reset', action='store_true', - help='Instruct the driver to reset the contents of the database') - aparser.add_argument('--scalefactor', default=1, type=float, metavar='SF', - help='Benchmark scale factor') - aparser.add_argument('--warehouses', default=4, type=int, metavar='W', - help='Number of Warehouses') - aparser.add_argument('--duration', default=60, type=int, metavar='D', - help='How long to run the benchmark in seconds') - aparser.add_argument('--ddl', default=os.path.realpath(os.path.join(os.path.dirname(__file__), "tpcc.sql")), - help='Path to the TPC-C DDL SQL file') - aparser.add_argument('--clients', default=1, type=int, metavar='N', - help='The number of blocking clients to fork') - aparser.add_argument('--stop-on-error', action='store_true', - help='Stop the transaction execution when the driver throws an exception.') - aparser.add_argument('--no-load', action='store_true', - help='Disable loading the data') - aparser.add_argument('--no-execute', action='store_true', - help='Disable executing the workload') - aparser.add_argument('--print-config', action='store_true', - help='Print out the default configuration file for the system and exit') - aparser.add_argument('--debug', action='store_true', - help='Enable debug log messages') +if __name__ == "__main__": + aparser = argparse.ArgumentParser( + description="Python implementation of the TPC-C Benchmark" + ) + aparser.add_argument("system", choices=getDrivers(), help="Target system driver") + aparser.add_argument( + "--config", type=file, help="Path to driver configuration file" + ) + aparser.add_argument( + "--reset", + action="store_true", + help="Instruct the driver to reset the contents of the database", + ) + aparser.add_argument( + "--scalefactor", + default=1, + type=float, + metavar="SF", + help="Benchmark scale factor", + ) + aparser.add_argument( + "--warehouses", default=4, type=int, metavar="W", help="Number of Warehouses" + ) + aparser.add_argument( + "--duration", + default=60, + type=int, + metavar="D", + help="How long to run the benchmark in seconds", + ) + aparser.add_argument( + "--ddl", + default=os.path.realpath(os.path.join(os.path.dirname(__file__), "tpcc.sql")), + help="Path to the TPC-C DDL SQL file", + ) + aparser.add_argument( + "--clients", + default=1, + type=int, + metavar="N", + help="The number of blocking clients to fork", + ) + aparser.add_argument( + "--stop-on-error", + action="store_true", + help="Stop the transaction execution when the driver throws an exception.", + ) + aparser.add_argument( + "--no-load", action="store_true", help="Disable loading the data" + ) + aparser.add_argument( + "--no-execute", action="store_true", help="Disable executing the workload" + ) + aparser.add_argument( + "--print-config", + action="store_true", + help="Print out the default configuration file for the system and exit", + ) + aparser.add_argument( + "--debug", action="store_true", help="Enable debug log messages" + ) args = vars(aparser.parse_args()) - if args['debug']: logging.getLogger().setLevel(logging.DEBUG) + if args["debug"]: + logging.getLogger().setLevel(logging.DEBUG) ## Arguments validation - assert args['reset'] == False or args['no_load'] == False, \ - "'--reset' and '--no-load' are incompatible with each other" + assert ( + args["reset"] == False or args["no_load"] == False + ), "'--reset' and '--no-load' are incompatible with each other" ## Create a handle to the target client driver - driverClass = createDriverClass(args['system']) - assert driverClass != None, "Failed to find '%s' class" % args['system'] - driver = driverClass(args['ddl']) - assert driver != None, "Failed to create '%s' driver" % args['system'] - if args['print_config']: + driverClass = createDriverClass(args["system"]) + assert driverClass != None, "Failed to find '%s' class" % args["system"] + driver = driverClass(args["ddl"]) + assert driver != None, "Failed to create '%s' driver" % args["system"] + if args["print_config"]: config = driver.makeDefaultConfig() print(driver.formatConfig(config)) print sys.exit(0) ## Load Configuration file - if args['config']: - logging.debug("Loading configuration file '%s'" % args['config']) + if args["config"]: + logging.debug("Loading configuration file '%s'" % args["config"]) cparser = SafeConfigParser() - cparser.read(os.path.realpath(args['config'].name)) - config = dict(cparser.items(args['system'])) + cparser.read(os.path.realpath(args["config"].name)) + config = dict(cparser.items(args["system"])) else: - logging.debug("Using default configuration for %s" % args['system']) + logging.debug("Using default configuration for %s" % args["system"]) defaultConfig = driver.makeDefaultConfig() config = dict(map(lambda x: (x, defaultConfig[x][1]), defaultConfig.keys())) - config['reset'] = args['reset'] - config['load'] = False - config['execute'] = False - if config['reset']: logging.info("Reseting database") + config["reset"] = args["reset"] + config["load"] = False + config["execute"] = False + if config["reset"]: + logging.info("Reseting database") driver.loadConfig(config) logging.info("Initializing TPC-C benchmark using %s" % driver) ## Create ScaleParameters - scaleParameters = scaleparameters.makeWithScaleFactor(args['warehouses'], args['scalefactor']) + scaleParameters = scaleparameters.makeWithScaleFactor( + args["warehouses"], args["scalefactor"] + ) nurand = rand.setNURand(nurand.makeForLoad()) - if args['debug']: logging.debug("Scale Parameters:\n%s" % scaleParameters) - + if args["debug"]: + logging.debug("Scale Parameters:\n%s" % scaleParameters) + ## DATA LOADER!!! load_time = None - if not args['no_load']: + if not args["no_load"]: logging.info("Loading TPC-C benchmark data using %s" % (driver)) load_start = time.time() - if args['clients'] == 1: - l = loader.Loader(driver, scaleParameters, range(scaleParameters.starting_warehouse, scaleParameters.ending_warehouse+1), True) + if args["clients"] == 1: + l = loader.Loader( + driver, + scaleParameters, + range( + scaleParameters.starting_warehouse, + scaleParameters.ending_warehouse + 1, + ), + True, + ) driver.loadStart() l.execute() driver.loadFinish() @@ -256,18 +342,20 @@ def executorFunc(driverClass, scaleParameters, args, config, debug): startLoading(driverClass, scaleParameters, args, config) load_time = time.time() - load_start ## IF - + ## WORKLOAD DRIVER!!! - if not args['no_execute']: - if args['clients'] == 1: - e = executor.Executor(driver, scaleParameters, stop_on_error=args['stop_on_error']) + if not args["no_execute"]: + if args["clients"] == 1: + e = executor.Executor( + driver, scaleParameters, stop_on_error=args["stop_on_error"] + ) driver.executeStart() - results = e.execute(args['duration']) + results = e.execute(args["duration"]) driver.executeFinish() else: results = startExecution(driverClass, scaleParameters, args, config) assert results print(results.show(load_time)) ## IF - + ## MAIN diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/util/__init__.py b/workloads/chbenchmark/py-tpcc/pytpcc/util/__init__.py index 2d2d802f..242a6b44 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/util/__init__.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/util/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__all__ = ["scaleparameters", "rand", "nurand", "results"] \ No newline at end of file +__all__ = ["scaleparameters", "rand", "nurand", "results"] diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/util/nurand.py b/workloads/chbenchmark/py-tpcc/pytpcc/util/nurand.py index c4854228..7127427a 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/util/nurand.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/util/nurand.py @@ -31,6 +31,7 @@ import rand + def makeForLoad(): """Create random NURand constants, appropriate for loading the database.""" cLast = rand.number(0, 255) @@ -38,22 +39,25 @@ def makeForLoad(): orderLineItemId = rand.number(0, 8191) return NURandC(cLast, cId, orderLineItemId) + def validCRun(cRun, cLoad): """Returns true if the cRun value is valid for running. See TPC-C 2.1.6.1 (page 20)""" cDelta = abs(cRun - cLoad) return 65 <= cDelta and cDelta <= 119 and cDelta != 96 and cDelta != 112 + def makeForRun(loadC): """Create random NURand constants for running TPC-C. TPC-C 2.1.6.1. (page 20) specifies the valid range for these constants.""" cRun = rand.number(0, 255) while validCRun(cRun, loadC.cLast) == False: cRun = rand.number(0, 255) assert validCRun(cRun, loadC.cLast) - + cId = rand.number(0, 1023) orderLineItemId = rand.number(0, 8191) return NURandC(cRun, cId, orderLineItemId) + class NURandC: def __init__(self, cLast, cId, orderLineItemId): self.cLast = cLast diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/util/results.py b/workloads/chbenchmark/py-tpcc/pytpcc/util/results.py index 33bf9dd9..99e32bab 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/util/results.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/util/results.py @@ -27,56 +27,57 @@ import logging import time + class Results: - + def __init__(self): self.start = None self.stop = None self.txn_id = 0 - - self.txn_counters = { } - self.txn_times = { } - self.running = { } - + + self.txn_counters = {} + self.txn_times = {} + self.running = {} + def startBenchmark(self): """Mark the benchmark as having been started""" assert self.start == None logging.debug("Starting benchmark statistics collection") self.start = time.time() return self.start - + def stopBenchmark(self): """Mark the benchmark as having been stopped""" assert self.start != None assert self.stop == None logging.debug("Stopping benchmark statistics collection") self.stop = time.time() - + def startTransaction(self, txn): self.txn_id += 1 id = self.txn_id self.running[id] = (txn, time.time()) return id - + def abortTransaction(self, id): """Abort a transaction and discard its times""" assert id in self.running txn_name, txn_start = self.running[id] del self.running[id] - + def stopTransaction(self, id): """Record that the benchmark completed an invocation of the given transaction""" assert id in self.running txn_name, txn_start = self.running[id] del self.running[id] - + duration = time.time() - txn_start total_time = self.txn_times.get(txn_name, 0) self.txn_times[txn_name] = total_time + duration - + total_cnt = self.txn_counters.get(txn_name, 0) self.txn_counters[txn_name] = total_cnt + 1 - + def append(self, r): for txn_name in r.txn_counters.keys(): orig_cnt = self.txn_counters.get(txn_name, 0) @@ -84,47 +85,49 @@ def append(self, r): self.txn_counters[txn_name] = orig_cnt + r.txn_counters[txn_name] self.txn_times[txn_name] = orig_time + r.txn_times[txn_name] - #logging.debug("%s [cnt=%d, time=%d]" % (txn_name, self.txn_counters[txn_name], self.txn_times[txn_name])) + # logging.debug("%s [cnt=%d, time=%d]" % (txn_name, self.txn_counters[txn_name], self.txn_times[txn_name])) ## HACK self.start = r.start self.stop = r.stop - + def __str__(self): return self.show() - - def show(self, load_time = None): + + def show(self, load_time=None): if self.start == None: return "Benchmark not started" if self.stop == None: duration = time.time() - self.start else: duration = self.stop - self.start - + col_width = 16 - total_width = (col_width*4)+2 - f = "\n " + (("%-" + str(col_width) + "s")*4) - line = "-"*total_width + total_width = (col_width * 4) + 2 + f = "\n " + (("%-" + str(col_width) + "s") * 4) + line = "-" * total_width - ret = u"" + "="*total_width + "\n" + ret = "" + "=" * total_width + "\n" if load_time != None: ret += "Data Loading Time: %d seconds\n\n" % (load_time) - + ret += "Execution Results after %d seconds\n%s" % (duration, line) - ret += f % ("", "Executed", u"Time (µs)", "Rate") - + ret += f % ("", "Executed", "Time (µs)", "Rate") + total_time = 0 total_cnt = 0 for txn in sorted(self.txn_counters.keys()): txn_time = self.txn_times[txn] txn_cnt = self.txn_counters[txn] - rate = u"%.02f txn/s" % ((txn_cnt / txn_time)) + rate = "%.02f txn/s" % ((txn_cnt / txn_time)) ret += f % (txn, str(txn_cnt), str(txn_time * 1000000), rate) - + total_time += txn_time total_cnt += txn_cnt - ret += "\n" + ("-"*total_width) + ret += "\n" + ("-" * total_width) total_rate = "%.02f txn/s" % ((total_cnt / total_time)) ret += f % ("TOTAL", str(total_cnt), str(total_time * 1000000), total_rate) - return (ret.encode('utf-8')) -## CLASS \ No newline at end of file + return ret.encode("utf-8") + + +## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/util/scaleparameters.py b/workloads/chbenchmark/py-tpcc/pytpcc/util/scaleparameters.py index 00417968..76a53028 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/util/scaleparameters.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/util/scaleparameters.py @@ -32,51 +32,80 @@ import constants + def makeDefault(warehouses): - return ScaleParameters(constants.NUM_ITEMS, \ - warehouses, \ - constants.DISTRICTS_PER_WAREHOUSE, \ - constants.CUSTOMERS_PER_DISTRICT, \ - constants.INITIAL_NEW_ORDERS_PER_DISTRICT) + return ScaleParameters( + constants.NUM_ITEMS, + warehouses, + constants.DISTRICTS_PER_WAREHOUSE, + constants.CUSTOMERS_PER_DISTRICT, + constants.INITIAL_NEW_ORDERS_PER_DISTRICT, + ) + + ## DEF + def makeWithScaleFactor(warehouses, scaleFactor): assert scaleFactor >= 1.0 - items = int(constants.NUM_ITEMS/scaleFactor) - if items <= 0: items = 1 + items = int(constants.NUM_ITEMS / scaleFactor) + if items <= 0: + items = 1 districts = int(max(constants.DISTRICTS_PER_WAREHOUSE, 1)) - customers = int(max(constants.CUSTOMERS_PER_DISTRICT/scaleFactor, 1)) - newOrders = int(max(constants.INITIAL_NEW_ORDERS_PER_DISTRICT/scaleFactor, 0)) + customers = int(max(constants.CUSTOMERS_PER_DISTRICT / scaleFactor, 1)) + newOrders = int(max(constants.INITIAL_NEW_ORDERS_PER_DISTRICT / scaleFactor, 0)) return ScaleParameters(items, warehouses, districts, customers, newOrders) + + ## DEF + class ScaleParameters: - - def __init__(self, items, warehouses, districtsPerWarehouse, customersPerDistrict, newOrdersPerDistrict): + + def __init__( + self, + items, + warehouses, + districtsPerWarehouse, + customersPerDistrict, + newOrdersPerDistrict, + ): assert 1 <= items and items <= constants.NUM_ITEMS self.items = items assert warehouses > 0 self.warehouses = warehouses self.starting_warehouse = 1 - assert 1 <= districtsPerWarehouse and districtsPerWarehouse <= constants.DISTRICTS_PER_WAREHOUSE + assert ( + 1 <= districtsPerWarehouse + and districtsPerWarehouse <= constants.DISTRICTS_PER_WAREHOUSE + ) self.districtsPerWarehouse = districtsPerWarehouse - assert 1 <= customersPerDistrict and customersPerDistrict <= constants.CUSTOMERS_PER_DISTRICT + assert ( + 1 <= customersPerDistrict + and customersPerDistrict <= constants.CUSTOMERS_PER_DISTRICT + ) self.customersPerDistrict = customersPerDistrict - assert 0 <= newOrdersPerDistrict and newOrdersPerDistrict <= constants.CUSTOMERS_PER_DISTRICT + assert ( + 0 <= newOrdersPerDistrict + and newOrdersPerDistrict <= constants.CUSTOMERS_PER_DISTRICT + ) assert newOrdersPerDistrict <= constants.INITIAL_NEW_ORDERS_PER_DISTRICT self.newOrdersPerDistrict = newOrdersPerDistrict - self.ending_warehouse = (self.warehouses + self.starting_warehouse - 1) + self.ending_warehouse = self.warehouses + self.starting_warehouse - 1 + ## DEF def __str__(self): - out = "%d items\n" % self.items + out = "%d items\n" % self.items out += "%d warehouses\n" % self.warehouses out += "%d districts/warehouse\n" % self.districtsPerWarehouse out += "%d customers/district\n" % self.customersPerDistrict out += "%d initial new orders/district" % self.newOrdersPerDistrict return out + ## DEF -## CLASS \ No newline at end of file + +## CLASS diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/worker.py b/workloads/chbenchmark/py-tpcc/pytpcc/worker.py index 02941973..ee20516c 100755 --- a/workloads/chbenchmark/py-tpcc/pytpcc/worker.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/worker.py @@ -33,24 +33,27 @@ import re import argparse import glob -import time +import time import message import pickle import traceback -from pprint import pprint,pformat +from pprint import pprint, pformat from util import * from runtime import * import drivers + ## ============================================== ## createDriverClass ## ============================================== def createDriverClass(name): full_name = "%sDriver" % name.title() - mod = __import__('drivers.%s' % full_name.lower(), globals(), locals(), [full_name]) + mod = __import__("drivers.%s" % full_name.lower(), globals(), locals(), [full_name]) klass = getattr(mod, full_name) return klass + + ## DEF @@ -58,29 +61,32 @@ def createDriverClass(name): ## loaderFunc ## ============================================== def loaderFunc(driverClass, scaleParameters, args, config, w_ids, debug): - driver = driverClass(args['ddl']) + driver = driverClass(args["ddl"]) assert driver != None - logging.debug("Starting client execution: %s [warehouses=%d]" % (driver, len(w_ids))) + logging.debug( + "Starting client execution: %s [warehouses=%d]" % (driver, len(w_ids)) + ) - config['load'] = True - config['execute'] = False - config['reset'] = False + config["load"] = True + config["execute"] = False + config["reset"] = False driver.loadConfig(config) try: - loadItems = (1 in w_ids) + loadItems = 1 in w_ids l = loader.Loader(driver, scaleParameters, w_ids, loadItems) driver.loadStart() l.execute() - driver.loadFinish() + driver.loadFinish() except KeyboardInterrupt: - return -1 + return -1 except (Exception, AssertionError) as ex: logging.warn("Failed to load data: %s" % (ex)) - #if debug: + # if debug: traceback.print_exc(file=sys.stdout) raise - + + ## DEF @@ -88,24 +94,26 @@ def loaderFunc(driverClass, scaleParameters, args, config, w_ids, debug): ## executorFunc ## ============================================== def executorFunc(driverClass, scaleParameters, args, config, debug): - driver = driverClass(args['ddl']) + driver = driverClass(args["ddl"]) assert driver != None logging.debug("Starting client execution: %s" % driver) - config['execute'] = True - config['reset'] = False + config["execute"] = True + config["reset"] = False driver.loadConfig(config) - e = executor.Executor(driver, scaleParameters, stop_on_error=args['stop_on_error']) + e = executor.Executor(driver, scaleParameters, stop_on_error=args["stop_on_error"]) driver.executeStart() - results = e.execute(args['duration']) + results = e.execute(args["duration"]) driver.executeFinish() return results + + ## DEF ## MAIN -if __name__=='__channelexec__': +if __name__ == "__channelexec__": driverClass = None for item in channel: @@ -116,31 +124,31 @@ def executorFunc(driverClass, scaleParameters, args, config, debug): config = command.data[2] w_ids = command.data[3] - ## Create a handle to the target client driver at the client side - driverClass = createDriverClass(args['system']) - assert driverClass != None, "Failed to find '%s' class" % args['system'] - driver = driverClass(args['ddl']) - assert driver != None, "Failed to create '%s' driver" % args['system'] + ## Create a handle to the target client driver at the client side + driverClass = createDriverClass(args["system"]) + assert driverClass != None, "Failed to find '%s' class" % args["system"] + driver = driverClass(args["ddl"]) + assert driver != None, "Failed to create '%s' driver" % args["system"] - loaderFunc(driverClass,scaleParameters,args,config,w_ids,True) + loaderFunc(driverClass, scaleParameters, args, config, w_ids, True) m = message.Message(header=message.LOAD_COMPLETED) - channel.send(pickle.dumps(m,-1)) + channel.send(pickle.dumps(m, -1)) elif command.header == message.CMD_EXECUTE: scaleParameters = command.data[0] args = command.data[1] config = command.data[2] - ## Create a handle to the target client driver at the client side + ## Create a handle to the target client driver at the client side if driverClass == None: - driverClass = createDriverClass(args['system']) - assert driverClass != None, "Failed to find '%s' class" % args['system'] - driver = driverClass(args['ddl']) - assert driver != None, "Failed to create '%s' driver" % args['system'] + driverClass = createDriverClass(args["system"]) + assert driverClass != None, "Failed to find '%s' class" % args["system"] + driver = driverClass(args["ddl"]) + assert driver != None, "Failed to create '%s' driver" % args["system"] - results=executorFunc(driverClass,scaleParameters,args,config,True) - m=message.Message(header=message.EXECUTE_COMPLETED,data=results) - channel.send(pickle.dumps(m,-1)) + results = executorFunc(driverClass, scaleParameters, args, config, True) + m = message.Message(header=message.EXECUTE_COMPLETED, data=results) + channel.send(pickle.dumps(m, -1)) - elif command.header==message.CMD_STOP: - pass + elif command.header == message.CMD_STOP: + pass diff --git a/workloads/chbenchmark/py-tpcc/setup.py b/workloads/chbenchmark/py-tpcc/setup.py index ba005d4a..8a1779e5 100644 --- a/workloads/chbenchmark/py-tpcc/setup.py +++ b/workloads/chbenchmark/py-tpcc/setup.py @@ -1,26 +1,27 @@ from setuptools import setup, find_packages import sys, os -version = '0.0' +version = "0.0" -setup(name='py-tpcc', - version=version, - description="Python implementation of the TPC-C benchmark", - long_description="""\ +setup( + name="py-tpcc", + version=version, + description="Python implementation of the TPC-C benchmark", + long_description="""\ """, - classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers - keywords='', - author='Andy Pavlo', - author_email='pavlo@cs.brown.edu', - url='http://www.cs.brown.edu/~pavlo/', - license='BSD', - packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), - include_package_data=True, - zip_safe=False, - install_requires=[ - # -*- Extra requirements: -*- - ], - entry_points=""" + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords="", + author="Andy Pavlo", + author_email="pavlo@cs.brown.edu", + url="http://www.cs.brown.edu/~pavlo/", + license="BSD", + packages=find_packages(exclude=["ez_setup", "examples", "tests"]), + include_package_data=True, + zip_safe=False, + install_requires=[ + # -*- Extra requirements: -*- + ], + entry_points=""" # -*- Entry points: -*- """, - ) +) From e388826be4a598625d52feca936e9b7b0f65ac1a Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Fri, 12 Apr 2024 18:07:14 -0400 Subject: [PATCH 4/5] Implement BradDriver with delivery and new_order --- .../py-tpcc/pytpcc/drivers/braddriver.py | 312 ++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 workloads/chbenchmark/py-tpcc/pytpcc/drivers/braddriver.py diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/braddriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/braddriver.py new file mode 100644 index 00000000..73f203af --- /dev/null +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/braddriver.py @@ -0,0 +1,312 @@ +import logging +from typing import Dict, Tuple, Any, Optional, List + +from abstractdriver import * +import constants + +from brad.grpc_client import BradGrpcClient + +Config = Dict[str, Tuple[str, Any]] + +logger = logging.getLogger(__name__) + + +TXN_QUERIES = { + "DELIVERY": { + "getNewOrder": "SELECT no_o_id FROM new_order WHERE no_d_id = {} AND no_w_id = {} AND no_o_id > -1 LIMIT 1", # + "deleteNewOrder": "DELETE FROM new_order WHERE no_d_id = {} AND no_w_id = {} AND no_o_id = {}", # d_id, w_id, no_o_id + "getCId": "SELECT o_c_id FROM orders WHERE o_id = {} AND o_d_id = {} AND o_w_id = {}", # no_o_id, d_id, w_id + "updateOrders": "UPDATE orders SET o_carrier_id = {} WHERE o_id = {} AND o_d_id = {} AND o_w_id = {}", # o_carrier_id, no_o_id, d_id, w_id + "updateOrderLine": "UPDATE order_line SET ol_delivery_d = {} WHERE ol_o_id = {} AND ol_d_id = {} AND ol_w_id = {}", # o_entry_d, no_o_id, d_id, w_id + "sumOLAmount": "SELECT SUM(ol_amount) FROM order_line WHERE ol_o_id = {} AND ol_d_id = {} AND ol_w_id = {}", # no_o_id, d_id, w_id + "updateCustomer": "UPDATE customer SET c_balance = c_balance + {} WHERE c_id = {} AND c_d_id = {} AND c_w_id = {}", # ol_total, c_id, d_id, w_id + }, + "NEW_ORDER": { + "getWarehouseTaxRate": "SELECT w_tax FROM warehouse WHERE w_id = {}", # w_id + "getDistrict": "SELECT d_tax, d_next_o_id FROM district WHERE d_id = {} AND d_w_id = {}", # d_id, w_id + "incrementNextOrderId": "UPDATE district SET d_next_o_id = {} WHERE d_id = {} AND d_w_id = {}", # d_next_o_id, d_id, w_id + "getCustomer": "SELECT c_discount, c_last, c_credit FROM customer WHERE c_w_id = {} AND c_d_id = {} AND c_id = {}", # w_id, d_id, c_id + "createOrder": "INSERT INTO orders (o_id, o_d_id, o_w_id, o_c_id, o_entry_d, o_carrier_id, o_ol_cnt, o_all_local) VALUES ({}, {}, {}, {}, {}, {}, {}, {})", # d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, o_ol_cnt, o_all_local + "createNewOrder": "INSERT INTO new_order (no_o_id, no_d_id, no_w_id) VALUES ({}, {}, {})", # o_id, d_id, w_id + "getItemInfo": "SELECT i_price, i_name, i_data FROM item WHERE i_id = {}", # ol_i_id + "getStockInfo": "SELECT s_quantity, s_data, s_ytd, s_order_cnt, s_remote_cnt, s_dist_{:02d} FROM stock WHERE s_i_id = {} AND s_w_id = {}", # d_id, ol_i_id, ol_supply_w_id + "updateStock": "UPDATE stock SET s_quantity = {}, s_ytd = {}, s_order_cnt = {}, s_remote_cnt = {} WHERE s_i_id = {} AND s_w_id = {}", # s_quantity, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id + "createOrderLine": "INSERT INTO order_line (ol_o_id, ol_d_id, ol_w_id, ol_number, ol_i_id, ol_supply_w_id, ol_delivery_d, ol_quantity, ol_amount, ol_dist_info) VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, {})", # o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info + }, + "ORDER_STATUS": { + "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id + "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last + "getLastOrder": "SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS WHERE O_W_ID = ? AND O_D_ID = ? AND O_C_ID = ? ORDER BY O_ID DESC LIMIT 1", # w_id, d_id, c_id + "getOrderLines": "SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID = ?", # w_id, d_id, o_id + }, + "PAYMENT": { + "getWarehouse": "SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP FROM WAREHOUSE WHERE W_ID = ?", # w_id + "updateWarehouseBalance": "UPDATE WAREHOUSE SET W_YTD = W_YTD + ? WHERE W_ID = ?", # h_amount, w_id + "getDistrict": "SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", # w_id, d_id + "updateDistrictBalance": "UPDATE DISTRICT SET D_YTD = D_YTD + ? WHERE D_W_ID = ? AND D_ID = ?", # h_amount, d_w_id, d_id + "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id + "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last + "updateBCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ?, C_DATA = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id + "updateGCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id + "insertHistory": "INSERT INTO HISTORY VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + }, + "STOCK_LEVEL": { + "getOId": "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", + "getStockCount": """ + SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK + WHERE OL_W_ID = ? + AND OL_D_ID = ? + AND OL_O_ID < ? + AND OL_O_ID >= ? + AND S_W_ID = ? + AND S_I_ID = OL_I_ID + AND S_QUANTITY < ? + """, + }, +} + + +class BradDriver(AbstractDriver): + DEFAULT_CONFIG = { + "host": ("Host running the BRAD front end.", "localhost"), + "port": ("Port on which the BRAD front end is listening.", 6583), + } + + def __init__(self, ddl: str) -> None: + super().__init__("BradDriver", ddl) + self._client: Optional[BradGrpcClient] = None + + def makeDefaultConfig(self) -> Config: + return BradDriver.DEFAULT_CONFIG + + def loadConfig(self, config: Config) -> None: + self._client = BradGrpcClient(host=config["host"], port=config["port"]) + self._client.connect() + + def loadTuples(self, tableName: str, tuples) -> None: + # We don't support data loading directly here. + pass + + def doDelivery(self, params: Dict[str, Any]) -> List[Tuple[Any, ...]]: + assert self._client is not None + + q = TXN_QUERIES["DELIVERY"] + w_id = params["w_id"] + o_carrier_id = params["o_carrier_id"] + ol_delivery_d = params["ol_delivery_d"] + + result = [] + self._client.run_query_json("BEGIN") + for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE + 1): + r, _ = self._client.run_query_json(q["getNewOrder"].format(d_id, w_id)) + if len(r) == 0: + ## No orders for this district: skip it. Note: This must be reported if > 1% + continue + no_o_id = r[0][0] + + r, _ = self._client.run_query_json(q["getCId"].format(no_o_id, d_id, w_id)) + c_id = r[0][0] + + r, _ = self._client.run_query_json( + q["sumOLAmount"].format(no_o_id, d_id, w_id) + ) + ol_total = r[0][0] + + self._client.run_query_json(q["deleteNewOrder"].format(d_id, w_id, no_o_id)) + self._client.run_query_json( + q["updateOrders"].format(o_carrier_id, no_o_id, d_id, w_id) + ) + self._client.run_query_json( + q["updateOrderLine"].format(ol_delivery_d, no_o_id, d_id, w_id) + ) + + # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) + # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure + # them out + # If there are no order lines, SUM returns null. There should always be order lines. + assert ( + ol_total != None + ), "ol_total is NULL: there are no order lines. This should not happen" + assert ol_total > 0.0 + + self._client.run_query_json( + q["updateCustomer"].format(ol_total, c_id, d_id, w_id) + ) + + result.append((d_id, no_o_id)) + + self._client.run_query_json("COMMIT") + return result + + def doNewOrder(self, params: Dict[str, Any]) -> List[Tuple[Any, ...]]: + assert self._client is not None + + q = TXN_QUERIES["NEW_ORDER"] + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + o_entry_d = params["o_entry_d"] + i_ids = params["i_ids"] + i_w_ids = params["i_w_ids"] + i_qtys = params["i_qtys"] + + assert len(i_ids) > 0 + assert len(i_ids) == len(i_w_ids) + assert len(i_ids) == len(i_qtys) + + self._client.run_query_json("BEGIN") + all_local = True + items = [] + for i in range(len(i_ids)): + ## Determine if this is an all local order or not + all_local = all_local and i_w_ids[i] == w_id + r, _ = self._client.run_query_json(q["getItemInfo"].format(i_ids[i])) + items.append(r[0]) + assert len(items) == len(i_ids) + + ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. + ## Note that this will happen with 1% of transactions on purpose. + for item in items: + if len(item) == 0: + self._client.run_query_json("ROLLBACK") + return + ## FOR + + ## ---------------- + ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER + ## ---------------- + r, _ = self._client.run_query_json(q["getWarehouseTaxRate"].format(w_id)) + w_tax = r[0][0] + + r, _ = self._client.run_query_json(q["getDistrict"].format(d_id, w_id)) + district_info = r[0] + d_tax = district_info[0] + d_next_o_id = district_info[1] + + r, _ = self._client.run_query_json(q["getCustomer"].format(w_id, d_id, c_id)) + customer_info = r[0] + c_discount = customer_info[0] + + ## ---------------- + ## Insert Order Information + ## ---------------- + ol_cnt = len(i_ids) + o_carrier_id = constants.NULL_CARRIER_ID + + self._client.run_query_json( + q["incrementNextOrderId"].format(d_next_o_id + 1, d_id, w_id) + ) + self._client.run_query_json( + q["createOrder"].format( + d_next_o_id, + d_id, + w_id, + c_id, + o_entry_d, + o_carrier_id, + ol_cnt, + all_local, + ), + ) + self._client.run_query_json(q["createNewOrder"].format(d_next_o_id, d_id, w_id)) + + ## ---------------- + ## Insert Order Item Information + ## ---------------- + item_data = [] + total = 0 + for i in range(len(i_ids)): + ol_number = i + 1 + ol_supply_w_id = i_w_ids[i] + ol_i_id = i_ids[i] + ol_quantity = i_qtys[i] + + itemInfo = items[i] + i_name = itemInfo[1] + i_data = itemInfo[2] + i_price = itemInfo[0] + + self._client.run_query_json( + q["getStockInfo"].format(d_id, ol_i_id, ol_supply_w_id) + ) + stockInfo = self.cursor.fetchone() + if len(stockInfo) == 0: + logger.warning( + "No STOCK record for (ol_i_id=%d, ol_supply_w_id=%d)", + ol_i_id, + ol_supply_w_id, + ) + continue + s_quantity = stockInfo[0] + s_ytd = stockInfo[2] + s_order_cnt = stockInfo[3] + s_remote_cnt = stockInfo[4] + s_data = stockInfo[1] + s_dist_xx = stockInfo[5] # Fetches data from the s_dist_[d_id] column + + ## Update stock + s_ytd += ol_quantity + if s_quantity >= ol_quantity + 10: + s_quantity = s_quantity - ol_quantity + else: + s_quantity = s_quantity + 91 - ol_quantity + s_order_cnt += 1 + + if ol_supply_w_id != w_id: + s_remote_cnt += 1 + + self._client.run_query_json( + q["updateStock"].format( + s_quantity, + s_ytd, + s_order_cnt, + s_remote_cnt, + ol_i_id, + ol_supply_w_id, + ), + ) + + if ( + i_data.find(constants.ORIGINAL_STRING) != -1 + and s_data.find(constants.ORIGINAL_STRING) != -1 + ): + brand_generic = "B" + else: + brand_generic = "G" + + ## Transaction profile states to use "ol_quantity * i_price" + ol_amount = ol_quantity * i_price + total += ol_amount + + self._client.run_query_json( + q["createOrderLine"].format( + d_next_o_id, + d_id, + w_id, + ol_number, + ol_i_id, + ol_supply_w_id, + o_entry_d, + ol_quantity, + ol_amount, + s_dist_xx, + ), + ) + + ## Add the info to be returned + item_data.append((i_name, s_quantity, brand_generic, i_price, ol_amount)) + ## FOR + + ## Commit! + self._client.run_query_json("COMMIT") + + ## Adjust the total for the discount + # print "c_discount:", c_discount, type(c_discount) + # print "w_tax:", w_tax, type(w_tax) + # print "d_tax:", d_tax, type(d_tax) + total *= (1 - c_discount) * (1 + w_tax + d_tax) + + ## Pack up values the client is missing (see TPC-C 2.4.3.5) + misc = [(w_tax, d_tax, d_next_o_id, total)] + + return [customer_info, misc, item_data] From 5cc324fe0328760595cde6283bfcab7166087dbf Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Fri, 12 Apr 2024 18:27:04 -0400 Subject: [PATCH 5/5] Implement order_status, payment, and stock_level --- .../py-tpcc/pytpcc/drivers/braddriver.py | 201 ++++++++++++++++-- 1 file changed, 179 insertions(+), 22 deletions(-) diff --git a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/braddriver.py b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/braddriver.py index 73f203af..a95c495a 100644 --- a/workloads/chbenchmark/py-tpcc/pytpcc/drivers/braddriver.py +++ b/workloads/chbenchmark/py-tpcc/pytpcc/drivers/braddriver.py @@ -34,33 +34,33 @@ "createOrderLine": "INSERT INTO order_line (ol_o_id, ol_d_id, ol_w_id, ol_number, ol_i_id, ol_supply_w_id, ol_delivery_d, ol_quantity, ol_amount, ol_dist_info) VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, {})", # o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info }, "ORDER_STATUS": { - "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id - "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last - "getLastOrder": "SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS WHERE O_W_ID = ? AND O_D_ID = ? AND O_C_ID = ? ORDER BY O_ID DESC LIMIT 1", # w_id, d_id, c_id - "getOrderLines": "SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID = ?", # w_id, d_id, o_id + "getCustomerByCustomerId": "SELECT c_id, c_first, c_middle, c_last, c_balance FROM customer WHERE c_w_id = {} AND c_d_id = {} AND c_id = {}", # w_id, d_id, c_id + "getCustomersByLastName": "SELECT c_id, c_first, c_middle, c_last, c_balance FROM customer WHERE c_w_id = {} AND c_d_id = {} AND c_last = {} ORDER BY c_first", # w_id, d_id, c_last + "getLastOrder": "SELECT o_id, o_carrier_id, o_entry_d FROM orders WHERE o_w_id = ? AND o_d_id = ? AND o_c_id = ? ORDER BY o_id DESC LIMIT 1", # w_id, d_id, c_id + "getOrderLines": "SELECT ol_supply_w_id, ol_i_id, ol_quantity, ol_amount, ol_delivery_d FROM order_line WHERE ol_w_id = ? AND ol_d_id = ? AND ol_o_id = ?", # w_id, d_id, o_id }, "PAYMENT": { - "getWarehouse": "SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP FROM WAREHOUSE WHERE W_ID = ?", # w_id - "updateWarehouseBalance": "UPDATE WAREHOUSE SET W_YTD = W_YTD + ? WHERE W_ID = ?", # h_amount, w_id - "getDistrict": "SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", # w_id, d_id - "updateDistrictBalance": "UPDATE DISTRICT SET D_YTD = D_YTD + ? WHERE D_W_ID = ? AND D_ID = ?", # h_amount, d_w_id, d_id - "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id - "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last - "updateBCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ?, C_DATA = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id - "updateGCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id - "insertHistory": "INSERT INTO HISTORY VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + "getWarehouse": "SELECT w_name, w_street_1, w_street_2, w_city, w_state, w_zip FROM warehouse WHERE w_id = {}", # w_id + "updateWarehouseBalance": "UPDATE warehouse SET w_ytd = w_ytd + {} WHERE w_id = {}", # h_amount, w_id + "getDistrict": "SELECT d_name, d_street_1, d_street_2, d_city, d_state, d_zip FROM district WHERE d_w_id = {} AND d_id = {}", # w_id, d_id + "updateDistrictBalance": "UPDATE district SET d_ytd = d_ytd + {} WHERE d_w_id = {} AND d_id = {}", # h_amount, d_w_id, d_id + "getCustomerByCustomerId": "SELECT c_id, c_first, c_middle, c_last, c_street_1, c_street_2, c_city, c_state, c_zip, c_phone, c_since, c_credit, c_credit_lim, c_discount, c_balance, c_ytd_payment, c_payment_cnt, c_data FROM customer WHERE c_w_id = {} AND c_d_id = {} AND c_id = {}", # w_id, d_id, c_id + "getCustomersByLastName": "SELECT c_id, c_first, c_middle, c_last, c_street_1, c_street_2, c_city, c_state, c_zip, c_phone, c_since, c_credit, c_credit_lim, c_discount, c_balance, c_ytd_payment, c_payment_cnt, c_data FROM customer WHERE c_w_id = {} AND c_d_id = {} AND c_last = {} ORDER BY c_first", # w_id, d_id, c_last + "updateBCCustomer": "UPDATE customer SET c_balance = {}, c_ytd_payment = {}, c_payment_cnt = {}, c_data = {} WHERE c_w_id = {} AND c_d_id = {} AND c_id = {}", # c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id + "updateGCCustomer": "UPDATE customer SET c_balance = {}, c_ytd_payment = {}, c_payment_cnt = {} WHERE c_w_id = {} AND c_d_id = {} AND c_id = {}", # c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id + "insertHistory": "INSERT INTO history VALUES ({}, {}, {}, {}, {}, {}, {}, {})", }, "STOCK_LEVEL": { - "getOId": "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", + "getOId": "SELECT d_next_o_id FROM district WHERE d_w_id = {} AND d_id = {}", "getStockCount": """ - SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK - WHERE OL_W_ID = ? - AND OL_D_ID = ? - AND OL_O_ID < ? - AND OL_O_ID >= ? - AND S_W_ID = ? - AND S_I_ID = OL_I_ID - AND S_QUANTITY < ? + SELECT COUNT(DISTINCT(ol_i_id)) FROM order_line, stock + WHERE ol_w_id = {} + AND ol_d_id = {} + AND ol_o_id < {} + AND ol_o_id >= {} + AND s_w_id = {} + AND s_i_id = ol_i_id + AND s_quantity < {} """, }, } @@ -310,3 +310,160 @@ def doNewOrder(self, params: Dict[str, Any]) -> List[Tuple[Any, ...]]: misc = [(w_tax, d_tax, d_next_o_id, total)] return [customer_info, misc, item_data] + + def doOrderStatus(self, params: Dict[str, Any]) -> List[Tuple[Any, ...]]: + assert self._client is not None + + q = TXN_QUERIES["ORDER_STATUS"] + w_id = params["w_id"] + d_id = params["d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + + self._client.run_query_json("BEGIN") + if c_id != None: + r, _ = self._client.run_query_json( + q["getCustomerByCustomerId"].format(w_id, d_id, c_id) + ) + customer = r[0] + else: + # Get the midpoint customer's id + r, _ = self._client.run_query_json( + q["getCustomersByLastName"].format(w_id, d_id, c_last) + ) + all_customers = r + assert len(all_customers) > 0 + namecnt = len(all_customers) + index = (namecnt - 1) / 2 + customer = all_customers[index] + c_id = customer[0] + assert len(customer) > 0 + assert c_id != None + + r, _ = self._client.run_query_json(q["getLastOrder"].format(w_id, d_id, c_id)) + order = r[0] + if order: + r, _ = self._client.run_query_json( + q["getOrderLines"].format(w_id, d_id, order[0]) + ) + orderLines = r + else: + orderLines = [] + + self._client.run_query_json("COMMIT") + return [customer, order, orderLines] + + def doPayment(self, params: Dict[str, Any]) -> List[Tuple[Any, ...]]: + assert self._client is not None + + q = TXN_QUERIES["PAYMENT"] + w_id = params["w_id"] + d_id = params["d_id"] + h_amount = params["h_amount"] + c_w_id = params["c_w_id"] + c_d_id = params["c_d_id"] + c_id = params["c_id"] + c_last = params["c_last"] + h_date = params["h_date"] + + self._client.run_query_json("BEGIN") + if c_id != None: + r, _ = self._client.run_query_json( + q["getCustomerByCustomerId"].format(w_id, d_id, c_id) + ) + customer = r[0] + else: + # Get the midpoint customer's id + r, _ = self._client.run_query_json( + q["getCustomersByLastName"].format(w_id, d_id, c_last) + ) + all_customers = r + assert len(all_customers) > 0 + namecnt = len(all_customers) + index = (namecnt - 1) / 2 + customer = all_customers[index] + c_id = customer[0] + assert len(customer) > 0 + c_balance = customer[14] - h_amount + c_ytd_payment = customer[15] + h_amount + c_payment_cnt = customer[16] + 1 + c_data = customer[17] + + r, _ = self._client.run_query_json(q["getWarehouse"].format(w_id)) + warehouse = r[0] + + r, _ = self._client.run_query_json(q["getDistrict"].format(w_id, d_id)) + district = r[0] + + self._client.run_query_json(q["updateWarehouseBalance"].format(h_amount, w_id)) + self._client.run_query_json( + q["updateDistrictBalance"].format(h_amount, w_id, d_id) + ) + + # Customer Credit Information + if customer[11] == constants.BAD_CREDIT: + newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) + c_data = newData + "|" + c_data + if len(c_data) > constants.MAX_C_DATA: + c_data = c_data[: constants.MAX_C_DATA] + self._client.run_query_json( + q["updateBCCustomer"].format( + c_balance, + c_ytd_payment, + c_payment_cnt, + c_data, + c_w_id, + c_d_id, + c_id, + ), + ) + else: + c_data = "" + self._client.run_query_json( + q["updateGCCustomer"].format( + c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id + ), + ) + + # Concatenate w_name, four spaces, d_name + h_data = "%s %s" % (warehouse[0], district[0]) + # Create the history record + self._client.run_query_json( + q["insertHistory"].format( + c_id, c_d_id, c_w_id, d_id, w_id, h_date, h_amount, h_data + ), + ) + + self._client.run_query_json("COMMIT") + + # TPC-C 2.5.3.3: Must display the following fields: + # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, + # D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, + # C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, + # C_DISCOUNT, C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = "BC"), + # H_AMOUNT, and H_DATE. + + # Hand back all the warehouse, district, and customer data + return [warehouse, district, customer] + + def doStockLevel(self, params: Dict[str, Any]) -> int: + assert self._client is not None + + q = TXN_QUERIES["STOCK_LEVEL"] + w_id = params["w_id"] + d_id = params["d_id"] + threshold = params["threshold"] + + self._client.run_query_json("BEGIN") + r, _ = self._client.run_query_json(["getOId"].format(w_id, d_id)) + result = r[0] + assert result + o_id = result[0] + + r, _ = self._client.run_query_json( + q["getStockCount"].format(w_id, d_id, o_id, (o_id - 20), w_id, threshold) + ) + result = r[0] + + self._client.run_query_json("COMMIT") + return int(result[0])