From 110531268ef02a05ead4974c37c364fa93205a2f Mon Sep 17 00:00:00 2001 From: Doug Baldwin Date: Sun, 7 Jul 2019 02:17:48 -0400 Subject: [PATCH 01/17] Add logging module --- Browser.py | 40 +++++++++++++++++++++------------------- client.py | 24 ++++++++++++++---------- config.json | 35 ----------------------------------- db_funcs.py | 30 ++++++++++++++++-------------- logging.json | 19 +++++++++++++++++++ rpc_client.py | 13 ++++++++----- stats.py | 24 +++++++++++++++--------- 7 files changed, 93 insertions(+), 92 deletions(-) delete mode 100644 config.json create mode 100644 logging.json diff --git a/Browser.py b/Browser.py index 202d21f..9a06798 100644 --- a/Browser.py +++ b/Browser.py @@ -2,14 +2,21 @@ # -*- coding: utf-8 -*- # execute in production with: nohup python3 Browser.py &> browser.log < /dev/null & +################ +# Logging init # +################ +import json +from logging.config import dictConfig + +with open('logging.json', 'r') as f: + dictConfig( json.load(f) ) + ########### # Imports # ########### import re import sys import os -import traceback -import json from time import sleep from multiprocessing import Process @@ -71,13 +78,13 @@ def update_counters(): global ctr ctr += 1 - print('counter:', ctr) + app.logger.info('counter: {}'.format(ctr)) sys.stdout.flush() def is_valid_account(acct): if (not re.match("^[A-Za-z0-9]*$", acct)) or (len(acct) != 64): - print("invalid Account:", acct) + app.logger.info("invalid Account: {}".format(acct)) return False return True @@ -154,7 +161,7 @@ def version(ver): @app.route('/account/') def acct_details(acct): - print(acct) + app.logger.info('Account: {}'.format(acct)) update_counters() try: page = int(request.args.get('page')) @@ -169,7 +176,7 @@ def acct_details(acct): acct_state_raw = get_acct_raw(acct) acct_info = get_acct_info(acct_state_raw) - print('acct_info', acct_info) + app.logger.info('acct_info: {}'.format(acct_info)) try: tx_list = get_all_account_tx(c2, acct, page) @@ -177,9 +184,7 @@ def acct_details(acct): for tx in tx_list: tx_tbl += gen_tx_table_row(tx) except: - print(sys.exc_info()) - traceback.print_exception(*sys.exc_info()) - print('error in building table') + app.logger.exception('error in building table') next_page = "/account/" + acct + "?page=" + str(page + 1) @@ -191,10 +196,10 @@ def acct_details(acct): def search_redir(): tgt = request.args.get('acct') if len(tgt) == 64: - print('redir to account', tgt) + app.logger.info('redir to account: {}'.format(tgt)) return redirect('/account/'+tgt) else: - print('redir to tx', tgt) + app.logger.info('redir to tx: {}'.format(tgt)) return redirect('/version/'+tgt) @@ -211,9 +216,7 @@ def stats(): ret = stats_template.format(*stats_all_time, *stats_24_hours, *stats_one_hour) except: - print(sys.exc_info()) - traceback.print_exception(*sys.exc_info()) - print('error in stats') + app.logger.exception('error in stats') conn.close() @@ -231,9 +234,9 @@ def faucet(): if request.method == 'POST': try: acct = request.form.get('acct') - print(acct) + app.logger.info('acct: {}'.format(acct)) amount = request.form.get('amount') - print(amount) + app.logger.info('amount: {}'.format(amount)) if float(amount) < 0: message = 'Amount must be >= 0' elif not is_valid_account(acct): @@ -244,8 +247,8 @@ def faucet(): acct_link = '{0}'.format(acct) message = 'Sent ' + amount + ' Libra to ' + acct_link except: - traceback.print_exception(*sys.exc_info()) message = 'Invalid request logged!' + app.logger.exception(message) if message: message = faucet_alert_template.format(message) @@ -271,8 +274,7 @@ def send_asset(path): except: config = config["PRODUCTION"] - print("system configuration:") - print(json.dumps(config, indent=4)) + app.logger.info("system configuration: {}".format(json.dumps(config, indent=4))) tx_p = Process(target=tx_db_worker, args=(config['DB_PATH'], config['RPC_SERVER'], config['MINT_ACCOUNT'])) tx_p.start() diff --git a/client.py b/client.py index 0514f71..2655759 100644 --- a/client.py +++ b/client.py @@ -1,12 +1,17 @@ # Library to automate Libra client +########## +# Logger # +########## +import logging +logger = logging.getLogger(__name__) + ########### # Imports # ########### import os import re import sys -import traceback from datetime import datetime from subprocess import Popen, PIPE from time import sleep @@ -17,15 +22,15 @@ ######### def start_client_instance(client_path = '', account_file = ''): c_path = os.path.expanduser(client_path + "target/debug/client") - p = Popen([c_path, "--host", "ac.testnet.libra.org", "--port", "80", - "-s", "./scripts/cli/trusted_peers.config.toml"], cwd=os.path.expanduser(client_path), + args = [c_path, "--host", "ac.testnet.libra.org", "--port", "80", + "-s", "./scripts/cli/trusted_peers.config.toml"] + logger.info(' '.join(args)) + p = Popen(args, cwd=os.path.expanduser(client_path), shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True, bufsize=0, universal_newlines=True) sleep(5) p.stdout.flush() - print(os.read(p.stdout.fileno(), 10000)) - - print('loading account') - print(do_cmd("a r " + account_file, p = p)) + logger.info(os.read(p.stdout.fileno(), 10000).decode('unicode_escape')) + logger.info('loading account {}: {}'.format(account_file, do_cmd("a r " + account_file, p = p))) sys.stdout.flush() return p @@ -54,8 +59,7 @@ def get_acct_info(raw_account_status): sent_events = next(re.finditer(r'sent_events_count: (\d+),', raw_account_status)).group(1) recv_events = next(re.finditer(r'received_events_count: (\d+),', raw_account_status)).group(1) except: - print(sys.exc_info()) - traceback.print_exception(*sys.exc_info()) + logger.exception('Error in getting account info') return account, balance, sq_num, sent_events, recv_events @@ -74,4 +78,4 @@ def parse_raw_tx(raw): sq_num = next(re.finditer(r'sequence_number: (\d+),', raw)).group(1) pubkey = next(re.finditer(r'public_key: ([a-z0-9]+),', raw)).group(1) - return ver, expiration, sender, target, t_type, amount, gas_price, gas_max, sq_num, pubkey, expiration_num \ No newline at end of file + return ver, expiration, sender, target, t_type, amount, gas_price, gas_max, sq_num, pubkey, expiration_num diff --git a/config.json b/config.json deleted file mode 100644 index 794d35b..0000000 --- a/config.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "PRODUCTION" : { - "DB_PATH" : "./tx_cache.db", - "CLIENT_PATH" : "~/libra/", - "ACCOUNT_FILE" : "./test_acct", - "RPC_SERVER" : "ac.testnet.libra.org:8000", - "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", - "FLASK_HOST" : "0.0.0.0", - "FLASK_PORT" : 5000, - "FLASK_DEBUG" : false, - "FLASK_THREADED" : false - }, - "DEVELOPMENT" : { - "DB_PATH" : "./tx_cache.db", - "CLIENT_PATH" : "~/Source/libra/", - "ACCOUNT_FILE" : "./test_acct", - "RPC_SERVER" : "ac.testnet.libra.org:8000", - "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", - "FLASK_HOST" : "127.0.0.1", - "FLASK_PORT" : 5000, - "FLASK_DEBUG" : false, - "FLASK_THREADED" : false - }, - "STAGING" : { - "DB_PATH" : "./tx_cache.db", - "CLIENT_PATH" : "~/libra/", - "ACCOUNT_FILE" : "./test_acct", - "RPC_SERVER" : "ac.testnet.libra.org:8000", - "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", - "FLASK_HOST" : "0.0.0.0", - "FLASK_PORT" : 5001, - "FLASK_DEBUG" : false, - "FLASK_THREADED" : false - } -} \ No newline at end of file diff --git a/db_funcs.py b/db_funcs.py index 6dbad8b..7fed81b 100644 --- a/db_funcs.py +++ b/db_funcs.py @@ -1,11 +1,16 @@ # All DB manipulation functions + DB worker code +########## +# Logger # +########## +import logging +logger = logging.getLogger(__name__) + ########### # Imports # ########### import sqlite3 import sys -import traceback from time import sleep import struct @@ -27,9 +32,7 @@ def get_latest_version(c): c.execute("SELECT MAX(version) FROM transactions") cur_ver = int(c.fetchall()[0][0]) except: - print(sys.exc_info()) - traceback.print_exception(*sys.exc_info()) - print("couldn't find any records setting current version to 0") + logger.exception("couldn't find any records; setting current version to 0") cur_ver = 0 if not type(cur_ver) is int: cur_ver = 0 @@ -50,7 +53,7 @@ def get_tx_from_db_by_version(ver, c): try: ver = int(ver) # safety except: - print('potential attempt to inject:', ver) + logger.info('potential attempt to inject: {}'.format(ver)) ver = 1 c.execute("SELECT * FROM transactions WHERE version = " + str(ver)) @@ -58,7 +61,7 @@ def get_tx_from_db_by_version(ver, c): res = [parse_db_row(row) for row in res] if len(res) > 1: - print('possible duplicates detected in db, record version:', ver) + logger.info('possible duplicates detected in db, record version: {}'.format(ver)) return res[0] @@ -82,7 +85,7 @@ def init_db(c): expiration_unixtime INTEGER, gas_used text, sender_sig text, signed_tx_hash text, state_root_hash text, event_root_hash text, code_hex text, program text)''') except: - print('reusing existing db') + logger.info('reusing existing db') # Test DB version c.execute("SELECT * FROM transactions where version = 1") @@ -90,14 +93,14 @@ def init_db(c): if tmp is None: pass elif len(tmp) != 18: - print("DB version mismatch! please delete the old db and allow the system to repopulate") + logger.critical("DB version mismatch! please delete the old db and allow the system to repopulate") sys.exit() def tx_db_worker(db_path, rpc_server, mint_addr): while True: try: - print('transactions db worker starting') + logger.info('transactions db worker starting') # create rpc connection try: @@ -113,7 +116,7 @@ def tx_db_worker(db_path, rpc_server, mint_addr): # get latest version in the db cur_ver = get_latest_version(c) cur_ver += 1 # TODO: later handle genesis - print('starting update at version', cur_ver) + logger.info('starting update at version {}'.format(cur_ver)) # start the main loop while True: @@ -142,7 +145,7 @@ def tx_db_worker(db_path, rpc_server, mint_addr): # update counter to the latest version we inserted cur_ver = res[-1]['version'] - print('update to version:', cur_ver, 'success') + logger.debug('update to version: {} - success'.format(cur_ver)) # Save (commit) the changes conn.commit() @@ -154,7 +157,6 @@ def tx_db_worker(db_path, rpc_server, mint_addr): sleep(0.001 * num) except: - print('Major error in tx_db_worker, details:', sys.exc_info()) - traceback.print_exception(*sys.exc_info()) + logger.exception('Major error in tx_db_worker') sleep(2) - print('restarting tx_db_worker') + logger.info('restarting tx_db_worker') diff --git a/logging.json b/logging.json new file mode 100644 index 0000000..ff2c2b7 --- /dev/null +++ b/logging.json @@ -0,0 +1,19 @@ +{ + "version": 1, + "formatters": { + "default": { + "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s" + } + }, + "handlers": { + "wsgi": { + "class": "logging.StreamHandler", + "stream": "ext://flask.logging.wsgi_errors_stream", + "formatter": "default" + } + }, + "root": { + "level": "DEBUG", + "handlers": ["wsgi"] + } +} diff --git a/rpc_client.py b/rpc_client.py index d4a5971..8df9524 100644 --- a/rpc_client.py +++ b/rpc_client.py @@ -1,6 +1,12 @@ # Library to fetch information from Libra via rpc # based on https://github.com/egorsmkv/libra-grpc-py/ +########## +# Logger # +########## +import logging +logger = logging.getLogger(__name__) + ########### # Imports # ########### @@ -15,7 +21,6 @@ from hexdump import hexdump import sys -import traceback from datetime import datetime @@ -57,7 +62,7 @@ def get_latest_version_from_ledger(): ledger_info = response.ledger_info_with_sigs.ledger_info last_version_seen = ledger_info.version - print('last version seen:', last_version_seen) + logger.debug('last version seen: {}'.format(last_version_seen)) return last_version_seen @@ -85,8 +90,7 @@ def get_acct_info(state): sent_events = acct_state[3] sq_num = acct_state[4] except: - print(sys.exc_info()) - traceback.print_exception(*sys.exc_info()) + logger.exception() return account, balance, sq_num, sent_events, recv_events @@ -98,7 +102,6 @@ def get_raw_tx_lst(version, limit): tx_req = GetTransactionsRequest(start_version=version, limit=limit, fetch_events=True) item = RequestItem(get_transactions_request=tx_req) request = UpdateToLatestLedgerRequest(client_known_version=last_version_seen, requested_items=[item]) - # print(request) response = stub.UpdateToLatestLedger(request) infos = response.response_items[0].get_transactions_response.txn_list_with_proof.infos diff --git a/stats.py b/stats.py index b6f5995..5c66dd2 100644 --- a/stats.py +++ b/stats.py @@ -1,13 +1,17 @@ # All functions for stat generation +########## +# Logger # +########## +import logging +logger = logging.getLogger(__name__) + ########### # Imports # ########### from datetime import datetime, timedelta - from db_funcs import get_tx_from_db_by_version, get_latest_version - ######### # Funcs # ######### @@ -33,32 +37,34 @@ def calc_stats(c, limit = None): # first block c.execute("SELECT MIN(version) FROM transactions WHERE version > 0" + t_str) first_version = c.fetchall()[0][0] + if not first_version: + first_version = 1 first_block_time = datetime.fromtimestamp(get_tx_from_db_by_version(first_version, c)[10]) - print('first ver = ', first_version) + logger.info('first ver = {}'.format(first_version)) # get max block last_block = get_latest_version(c) - print('last block = ', last_block) + logger.info('last block = {}'.format(last_block)) # deltas td = timedelta(0, limit) if limit else (cur_time - first_block_time) dhms = days_hours_minutes_seconds(td) blocks_delta = last_block - first_version + 1 - print('deltas:', td, blocks_delta) + logger.info('deltas: {} {}'.format(td, blocks_delta)) # mints c.execute("SELECT count(DISTINCT version) FROM transactions WHERE type = 'mint_transaction'" + t_str) mint_count = c.fetchall()[0][0] c.execute("SELECT DISTINCT version FROM transactions WHERE type = 'mint_transaction'" + t_str) mint_sum = sum([x[0] for x in c.fetchall()]) - print('mint cnt', mint_count, mint_sum) + logger.info('mint cnt {} {}'.format(mint_count, mint_sum)) # p2p txs c.execute("SELECT count(DISTINCT version) FROM transactions WHERE type = 'peer_to_peer_transaction'" + t_str) p2p_count = c.fetchall()[0][0] c.execute("SELECT DISTINCT version FROM transactions WHERE type = 'peer_to_peer_transaction'" + t_str) p2p_sum = sum([x[0] for x in c.fetchall()]) - print('p2p',p2p_count, p2p_sum) + logger.info('p2p {} {}'.format(p2p_count, p2p_sum)) # add 1 to account for the genesis block until it is added to DB c.execute("SELECT count(DISTINCT version) FROM transactions " + @@ -69,9 +75,9 @@ def calc_stats(c, limit = None): c.execute("SELECT DISTINCT version FROM transactions " + "WHERE (type != 'peer_to_peer_transaction') and (type != 'mint_transaction')" + t_str) other_sum = sum([x[0] for x in c.fetchall()]) - print('others', other_tx_count, other_sum) + logger.info('others {} {}'.format(other_tx_count, other_sum)) - print('p2p + mint =', mint_count + p2p_count) + logger.info('p2p + mint = {} {}'.format(mint_count, p2p_count)) # unique accounts c.execute("SELECT COUNT(DISTINCT dest) FROM transactions WHERE version > 0" + t_str) From bca010b0001cbfcb0580b8c92827ec0393bf2659 Mon Sep 17 00:00:00 2001 From: Doug Baldwin Date: Sun, 7 Jul 2019 02:43:53 -0400 Subject: [PATCH 02/17] restore config.json --- config.json | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 config.json diff --git a/config.json b/config.json new file mode 100644 index 0000000..6741509 --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "PRODUCTION" : { + "DB_PATH" : "./tx_cache.db", + "CLIENT_PATH" : "~/libra/", + "ACCOUNT_FILE" : "./test_acct", + "RPC_SERVER" : "ac.testnet.libra.org:8000", + "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", + "FLASK_HOST" : "0.0.0.0", + "FLASK_PORT" : 5000, + "FLASK_DEBUG" : false, + "FLASK_THREADED" : false + }, + "DEVELOPMENT" : { + "DB_PATH" : "./tx_cache.db", + "CLIENT_PATH" : "~/Source/libra/", + "ACCOUNT_FILE" : "./test_acct", + "RPC_SERVER" : "ac.testnet.libra.org:8000", + "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", + "FLASK_HOST" : "127.0.0.1", + "FLASK_PORT" : 5000, + "FLASK_DEBUG" : false, + "FLASK_THREADED" : false + }, + "STAGING" : { + "DB_PATH" : "./tx_cache.db", + "CLIENT_PATH" : "~/libra/", + "ACCOUNT_FILE" : "./test_acct", + "RPC_SERVER" : "ac.testnet.libra.org:8000", + "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", + "FLASK_HOST" : "0.0.0.0", + "FLASK_PORT" : 5001, + "FLASK_DEBUG" : false, + "FLASK_THREADED" : false + } +} From 766fdbd9b360dd9a84bf1fde4939adbbbe774c7c Mon Sep 17 00:00:00 2001 From: Doug Baldwin Date: Sun, 14 Jul 2019 11:37:50 -0400 Subject: [PATCH 03/17] switch to SQLAlchemy --- Browser.py | 42 +++------ config.json | 35 -------- db_funcs.py | 236 ++++++++++++++++++++++++-------------------------- rpc_client.py | 21 ++--- stats.py | 105 +++++++++++----------- 5 files changed, 190 insertions(+), 249 deletions(-) delete mode 100644 config.json diff --git a/Browser.py b/Browser.py index 9a06798..b91525e 100644 --- a/Browser.py +++ b/Browser.py @@ -19,11 +19,10 @@ import os from time import sleep -from multiprocessing import Process from rpc_client import get_acct_raw, get_acct_info, start_rpc_client_instance from client import start_client_instance, do_cmd -from db_funcs import connect_to_db, get_latest_version, get_tx_from_db_by_version, get_all_account_tx, tx_db_worker +from db_funcs import get_latest_version, get_tx_from_db_by_version, get_all_account_tx, TxDBWorker from stats import calc_stats @@ -42,7 +41,6 @@ # Definitions # ############### ctr = 0 # counter of requests since last init -c2 = None # placeholder for connection object header = '''Libra Testnet Experimental Browser

Experimental Libra testnet explorer by @gal_diskin @@ -121,11 +119,7 @@ def add_br_every64(s): @app.route('/') def index(): update_counters() - c2, conn = connect_to_db(config['DB_PATH']) - - bver = str(get_latest_version(c2)) - - conn.close() + bver = str(get_latest_version()) return index_template.format(bver) @@ -133,15 +127,13 @@ def index(): @cache.cached(timeout=3600) # versions don't change so we can cache long-term def version(ver): update_counters() - c2, conn = connect_to_db(config['DB_PATH']) - bver = str(get_latest_version(c2)) + bver = str(get_latest_version()) try: ver = int(ver) - tx = get_tx_from_db_by_version(ver, c2) + tx = get_tx_from_db_by_version(ver) except: - conn.close() return version_error_template # for toggle raw view @@ -155,7 +147,6 @@ def version(ver): extra = '' not_raw = '1' - conn.close() return version_template.format(bver, *tx, add_br_every64(tx[12]), extra, not_raw, tx[-2].replace('<', '<')) @@ -171,15 +162,14 @@ def acct_details(acct): if not is_valid_account(acct): return invalid_account_template - c2, conn = connect_to_db(config['DB_PATH']) - bver = str(get_latest_version(c2)) + bver = str(get_latest_version()) acct_state_raw = get_acct_raw(acct) acct_info = get_acct_info(acct_state_raw) app.logger.info('acct_info: {}'.format(acct_info)) try: - tx_list = get_all_account_tx(c2, acct, page) + tx_list = get_all_account_tx(acct, page) tx_tbl = '' for tx in tx_list: tx_tbl += gen_tx_table_row(tx) @@ -188,7 +178,6 @@ def acct_details(acct): next_page = "/account/" + acct + "?page=" + str(page + 1) - conn.close() return account_template.format(bver, *acct_info, tx_tbl, next_page) @@ -207,28 +196,22 @@ def search_redir(): @cache.cached(timeout=60) # no point updating states more than once per minute def stats(): update_counters() - c2, conn = connect_to_db(config['DB_PATH']) try: # get stats - stats_all_time = calc_stats(c2) - stats_24_hours = calc_stats(c2, limit = 3600 * 24)[5:] - stats_one_hour = calc_stats(c2, limit = 3600)[5:] + stats_all_time = calc_stats() + stats_24_hours = calc_stats(limit = 3600 * 24)[5:] + stats_one_hour = calc_stats(limit = 3600)[5:] ret = stats_template.format(*stats_all_time, *stats_24_hours, *stats_one_hour) except: app.logger.exception('error in stats') - - conn.close() - return ret - @app.route('/faucet', methods=['GET', 'POST']) def faucet(): update_counters() - c2, conn = connect_to_db(config['DB_PATH']) - bver = str(get_latest_version(c2)) + bver = str(get_latest_version()) message = '' if request.method == 'POST': @@ -276,12 +259,11 @@ def send_asset(path): app.logger.info("system configuration: {}".format(json.dumps(config, indent=4))) - tx_p = Process(target=tx_db_worker, args=(config['DB_PATH'], config['RPC_SERVER'], config['MINT_ACCOUNT'])) - tx_p.start() + TxDBWorker(config['DB_PATH'], config['RPC_SERVER'], config['MINT_ACCOUNT']).start() start_rpc_client_instance(config['RPC_SERVER'], config['MINT_ACCOUNT']) - p = start_client_instance(config['CLIENT_PATH'], config['ACCOUNT_FILE']) + start_client_instance(config['CLIENT_PATH'], config['ACCOUNT_FILE']) sleep(1) diff --git a/config.json b/config.json deleted file mode 100644 index 6741509..0000000 --- a/config.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "PRODUCTION" : { - "DB_PATH" : "./tx_cache.db", - "CLIENT_PATH" : "~/libra/", - "ACCOUNT_FILE" : "./test_acct", - "RPC_SERVER" : "ac.testnet.libra.org:8000", - "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", - "FLASK_HOST" : "0.0.0.0", - "FLASK_PORT" : 5000, - "FLASK_DEBUG" : false, - "FLASK_THREADED" : false - }, - "DEVELOPMENT" : { - "DB_PATH" : "./tx_cache.db", - "CLIENT_PATH" : "~/Source/libra/", - "ACCOUNT_FILE" : "./test_acct", - "RPC_SERVER" : "ac.testnet.libra.org:8000", - "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", - "FLASK_HOST" : "127.0.0.1", - "FLASK_PORT" : 5000, - "FLASK_DEBUG" : false, - "FLASK_THREADED" : false - }, - "STAGING" : { - "DB_PATH" : "./tx_cache.db", - "CLIENT_PATH" : "~/libra/", - "ACCOUNT_FILE" : "./test_acct", - "RPC_SERVER" : "ac.testnet.libra.org:8000", - "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", - "FLASK_HOST" : "0.0.0.0", - "FLASK_PORT" : 5001, - "FLASK_DEBUG" : false, - "FLASK_THREADED" : false - } -} diff --git a/db_funcs.py b/db_funcs.py index 7fed81b..7f9ded3 100644 --- a/db_funcs.py +++ b/db_funcs.py @@ -9,154 +9,144 @@ ########### # Imports # ########### -import sqlite3 +from sqlalchemy import create_engine, Table, Column, Integer, String, MetaData, select, desc, func +from sqlalchemy.pool import StaticPool +from threading import Thread import sys from time import sleep import struct -# from client import start_client_instance, do_cmd, parse_raw_tx from rpc_client import get_latest_version_from_ledger, get_raw_tx_lst, parse_raw_tx_lst, start_rpc_client_instance +############ +# Database # +############ + +metadata = MetaData() +txs = Table('transactions', metadata, + Column('version', Integer, primary_key=True), + Column('expiration_date', String), + Column('src', String), + Column('dest', String), + Column('type', String), + Column('amount', String), + Column('gas_price', String), + Column('max_gas', String), + Column('sq_num', Integer), + Column('pub_key', String), + Column('expiration_unixtime', Integer), + Column('gas_used', String), + Column('sender_sig', String), + Column('signed_tx_hash', String), + Column('state_root_hash', String), + Column('event_root_hash', String), + Column('code_hex', String), + Column('program', String), +) +engine = create_engine( + 'sqlite://', + connect_args={'check_same_thread': False}, + poolclass = StaticPool, + echo=False +) +metadata.create_all(engine) ######### # Funcs # ######### -def connect_to_db(path): - conn = sqlite3.connect(path) - return conn.cursor(), conn - -def get_latest_version(c): - try: - c.execute("SELECT MAX(version) FROM transactions") - cur_ver = int(c.fetchall()[0][0]) - except: - logger.exception("couldn't find any records; setting current version to 0") - cur_ver = 0 - if not type(cur_ver) is int: +def get_latest_version(): + cur_ver = engine.execute(select([func.max(txs.c.version)])).scalar() + if cur_ver is None: + logger.info("couldn't find any records; setting current version to 0") cur_ver = 0 return cur_ver - def parse_db_row(row): - r = list(row) - r[5] = struct.unpack(' bver: - sleep(1) - continue - - # batch update - num = min(1000, bver - cur_ver) # at most 5000 records at once - tx_data = get_raw_tx_lst(cur_ver, num) - - # read records - res = parse_raw_tx_lst(*tx_data) - if len(res) == 0: - sleep(5) - continue - - # do the insertion - db_data = [tuple(x.values()) for x in res] - c.executemany("INSERT INTO transactions VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);", db_data) - - # update counter to the latest version we inserted - cur_ver = res[-1]['version'] - logger.debug('update to version: {} - success'.format(cur_ver)) - - # Save (commit) the changes - conn.commit() - - # update latest version to next - cur_ver += 1 - - # sleep relative to amount of rows fetched so we don't get a 429 error - sleep(0.001 * num) - - except: - logger.exception('Major error in tx_db_worker') - sleep(2) - logger.info('restarting tx_db_worker') + def run(self): + while True: + try: + # get latest version in the db + cur_ver = get_latest_version() + cur_ver += 1 # TODO: later handle genesis + logger.info('starting update at version {}'.format(cur_ver)) + # start the main loop + while True: + try: + bver = get_latest_version_from_ledger() + except: + sleep(1) + continue + if cur_ver > bver: + sleep(1) + continue + + # batch update + num = min(1000, bver - cur_ver) # at most 5000 records at once + tx_data = get_raw_tx_lst(cur_ver, num) + + # read records + res = parse_raw_tx_lst(*tx_data) + if len(res) == 0: + sleep(5) + continue + + # do the insertion + engine.execute(txs.insert(), res) + + # update counter to the latest version we inserted + cur_ver = res[-1]['version'] + logger.debug('update to version: {} - success'.format(cur_ver)) + + # update latest version to next + cur_ver += 1 + + # sleep relative to amount of rows fetched so we don't get a 429 error + sleep(0.001 * num) + + except: + logger.exception('Major error in tx_db_worker') + sleep(2) + logger.info('restarting tx_db_worker') diff --git a/rpc_client.py b/rpc_client.py index 8df9524..fbb1bf7 100644 --- a/rpc_client.py +++ b/rpc_client.py @@ -121,25 +121,22 @@ def parse_raw_tx_lst(struct_lst, infos, raw, events): cur_ver = raw.first_transaction_version.value res = [] - for x in zip(infos, struct_lst, raw.transactions): #, events): - info = x[0] - tx = x[1] - r = x[2] + for (info, tx, r) in zip(infos, struct_lst, raw.transactions): #, events): tmp = dict() tmp['version'] = cur_ver cur_ver += 1 - tmp['expiration'] = str(datetime.fromtimestamp(min(tx.expiration_time, 2147485547))) - tmp['sender'] = bytes.hex(tx.sender_account) - tmp['target'] = bytes.hex(tx.program.arguments[0].data) - tmp['t_type'] = 'peer_to_peer_transaction' if tmp['sender'] != MINT_ACCOUNT else 'mint_transaction' + tmp['expiration_date'] = str(datetime.fromtimestamp(min(tx.expiration_time, 2147485547))) + tmp['src'] = bytes.hex(tx.sender_account) + tmp['dest'] = bytes.hex(tx.program.arguments[0].data) + tmp['type'] = 'peer_to_peer_transaction' if tmp['src'] != MINT_ACCOUNT else 'mint_transaction' tmp['amount'] = tx.program.arguments[1].data tmp['gas_price'] = struct.pack("= int_ts - limit + 100) + .where(txs.c.expiration_unixtime < int_ts + 600) + ) if limit else s # first block - c.execute("SELECT MIN(version) FROM transactions WHERE version > 0" + t_str) - first_version = c.fetchall()[0][0] + first_version = engine.execute( + s_limit( + select( + [func.min(txs.c.version)] + ).where( + txs.c.version > 0 + ) + ) + ).scalar() if not first_version: first_version = 1 - first_block_time = datetime.fromtimestamp(get_tx_from_db_by_version(first_version, c)[10]) logger.info('first ver = {}'.format(first_version)) # get max block - last_block = get_latest_version(c) + last_block = get_latest_version() logger.info('last block = {}'.format(last_block)) # deltas + first_block_time = datetime.fromtimestamp(get_tx_from_db_by_version(first_version)[10]) td = timedelta(0, limit) if limit else (cur_time - first_block_time) dhms = days_hours_minutes_seconds(td) blocks_delta = last_block - first_version + 1 logger.info('deltas: {} {}'.format(td, blocks_delta)) - # mints - c.execute("SELECT count(DISTINCT version) FROM transactions WHERE type = 'mint_transaction'" + t_str) - mint_count = c.fetchall()[0][0] - c.execute("SELECT DISTINCT version FROM transactions WHERE type = 'mint_transaction'" + t_str) - mint_sum = sum([x[0] for x in c.fetchall()]) - logger.info('mint cnt {} {}'.format(mint_count, mint_sum)) - - # p2p txs - c.execute("SELECT count(DISTINCT version) FROM transactions WHERE type = 'peer_to_peer_transaction'" + t_str) - p2p_count = c.fetchall()[0][0] - c.execute("SELECT DISTINCT version FROM transactions WHERE type = 'peer_to_peer_transaction'" + t_str) - p2p_sum = sum([x[0] for x in c.fetchall()]) + # mint p2p other + def get_tx_cnt_sum(whereclause): + selected = engine.execute( + s_limit( + select( + [txs.c.amount] + ).where( + whereclause + ).distinct(txs.c.version) + ) + ).fetchall() + return len(selected), sum(map(lambda r: unpack(r['amount']), selected)) + mint_count, mint_sum = get_tx_cnt_sum(txs.c.type == 'mint_transaction') + logger.info('mint {} {}'.format(mint_count, mint_sum)) + p2p_count, p2p_sum = get_tx_cnt_sum(txs.c.type == 'peer_to_peer_transaction') logger.info('p2p {} {}'.format(p2p_count, p2p_sum)) - + other_count, other_sum = get_tx_cnt_sum((txs.c.type != 'mint_transaction') & (txs.c.type != 'peer_to_peer_transaction')) # add 1 to account for the genesis block until it is added to DB - c.execute("SELECT count(DISTINCT version) FROM transactions " + - "WHERE (type != 'peer_to_peer_transaction') and (type != 'mint_transaction')" + t_str) - other_tx_count = c.fetchall()[0][0] - if limit is None: - other_tx_count += 1 # TODO: this is for genesis block - remove later - c.execute("SELECT DISTINCT version FROM transactions " + - "WHERE (type != 'peer_to_peer_transaction') and (type != 'mint_transaction')" + t_str) - other_sum = sum([x[0] for x in c.fetchall()]) - logger.info('others {} {}'.format(other_tx_count, other_sum)) - - logger.info('p2p + mint = {} {}'.format(mint_count, p2p_count)) + other_count += 1 + logger.info('others {} {}'.format(other_count, other_sum)) # unique accounts - c.execute("SELECT COUNT(DISTINCT dest) FROM transactions WHERE version > 0" + t_str) - count_dest = c.fetchone()[0] - c.execute("SELECT COUNT(DISTINCT src) FROM transactions WHERE version > 0" + t_str) - count_src = c.fetchone()[0] + def get_acct_cnt(acct): + return engine.execute( + s_limit( + select( + [acct] + ).where( + txs.c.version > 0 + ).distinct() + ).count() + ).fetchone()[0] + count_dest = get_acct_cnt(txs.c.dest) + count_src = get_acct_cnt(txs.c.src) - r = (blocks_delta, *dhms, blocks_delta/td.total_seconds(), 100*mint_count/blocks_delta, - 100*p2p_count/blocks_delta, 100*other_tx_count/blocks_delta, mint_sum, p2p_sum, other_sum, - count_dest, count_src) - return r + return (blocks_delta, *dhms, blocks_delta/td.total_seconds(), 100*mint_count/blocks_delta, + 100*p2p_count/blocks_delta, 100*other_count/blocks_delta, mint_sum, p2p_sum, other_sum, + count_dest, count_src) From 80048aaf8852935fe50d4c69ec5501cc9ab2a205 Mon Sep 17 00:00:00 2001 From: Doug Baldwin Date: Sun, 14 Jul 2019 11:59:19 -0400 Subject: [PATCH 04/17] restore config.json --- config.json | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 config.json diff --git a/config.json b/config.json new file mode 100644 index 0000000..6741509 --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "PRODUCTION" : { + "DB_PATH" : "./tx_cache.db", + "CLIENT_PATH" : "~/libra/", + "ACCOUNT_FILE" : "./test_acct", + "RPC_SERVER" : "ac.testnet.libra.org:8000", + "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", + "FLASK_HOST" : "0.0.0.0", + "FLASK_PORT" : 5000, + "FLASK_DEBUG" : false, + "FLASK_THREADED" : false + }, + "DEVELOPMENT" : { + "DB_PATH" : "./tx_cache.db", + "CLIENT_PATH" : "~/Source/libra/", + "ACCOUNT_FILE" : "./test_acct", + "RPC_SERVER" : "ac.testnet.libra.org:8000", + "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", + "FLASK_HOST" : "127.0.0.1", + "FLASK_PORT" : 5000, + "FLASK_DEBUG" : false, + "FLASK_THREADED" : false + }, + "STAGING" : { + "DB_PATH" : "./tx_cache.db", + "CLIENT_PATH" : "~/libra/", + "ACCOUNT_FILE" : "./test_acct", + "RPC_SERVER" : "ac.testnet.libra.org:8000", + "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", + "FLASK_HOST" : "0.0.0.0", + "FLASK_PORT" : 5001, + "FLASK_DEBUG" : false, + "FLASK_THREADED" : false + } +} From 299252dfea1ccf7bd842959642b50959329f076f Mon Sep 17 00:00:00 2001 From: Doug Baldwin Date: Tue, 16 Jul 2019 03:22:59 -0400 Subject: [PATCH 05/17] Add DB backend to config.json --- config.json | 6 ++++-- db_funcs.py | 37 ++++++++++++++++++++++++------------- stats.py | 8 ++++---- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/config.json b/config.json index 6741509..827141e 100644 --- a/config.json +++ b/config.json @@ -8,7 +8,8 @@ "FLASK_HOST" : "0.0.0.0", "FLASK_PORT" : 5000, "FLASK_DEBUG" : false, - "FLASK_THREADED" : false + "FLASK_THREADED" : false, + "sqlalchemy.url": "postgresql://postgres:postgres@localhost:5432/libra_browser" }, "DEVELOPMENT" : { "DB_PATH" : "./tx_cache.db", @@ -30,6 +31,7 @@ "FLASK_HOST" : "0.0.0.0", "FLASK_PORT" : 5001, "FLASK_DEBUG" : false, - "FLASK_THREADED" : false + "FLASK_THREADED" : false, + "sqlalchemy.url": "postgresql://postgres:postgres@localhost:5432/libra_browser" } } diff --git a/db_funcs.py b/db_funcs.py index 7f9ded3..a215f6a 100644 --- a/db_funcs.py +++ b/db_funcs.py @@ -9,12 +9,12 @@ ########### # Imports # ########### -from sqlalchemy import create_engine, Table, Column, Integer, String, MetaData, select, desc, func +from sqlalchemy import create_engine, engine_from_config, Table, Column, Integer, BigInteger, LargeBinary, String, MetaData, select, desc, func from sqlalchemy.pool import StaticPool from threading import Thread import sys from time import sleep - +import json import struct from rpc_client import get_latest_version_from_ledger, get_raw_tx_lst, parse_raw_tx_lst, start_rpc_client_instance @@ -30,13 +30,13 @@ Column('src', String), Column('dest', String), Column('type', String), - Column('amount', String), - Column('gas_price', String), - Column('max_gas', String), + Column('amount', LargeBinary), + Column('gas_price', LargeBinary), + Column('max_gas', LargeBinary), Column('sq_num', Integer), Column('pub_key', String), - Column('expiration_unixtime', Integer), - Column('gas_used', String), + Column('expiration_unixtime', BigInteger), + Column('gas_used', LargeBinary), Column('sender_sig', String), Column('signed_tx_hash', String), Column('state_root_hash', String), @@ -44,12 +44,23 @@ Column('code_hex', String), Column('program', String), ) -engine = create_engine( - 'sqlite://', - connect_args={'check_same_thread': False}, - poolclass = StaticPool, - echo=False -) + +with open('config.json', 'r') as f: + config = json.load(f) +try: + config = config[os.getenv("BROWSER")] +except: + config = config["PRODUCTION"] + +if 'sqlalchemy.url' in config: + engine = create_engine(config['sqlalchemy.url']) +else: + engine = create_engine( + 'sqlite://', + connect_args={'check_same_thread': False}, + poolclass = StaticPool, + ) + metadata.create_all(engine) ######### diff --git a/stats.py b/stats.py index 45fb526..85e75f6 100644 --- a/stats.py +++ b/stats.py @@ -84,12 +84,12 @@ def get_acct_cnt(acct): return engine.execute( s_limit( select( - [acct] + [func.count(acct.distinct())] ).where( txs.c.version > 0 - ).distinct() - ).count() - ).fetchone()[0] + ) + ) + ).scalar() count_dest = get_acct_cnt(txs.c.dest) count_src = get_acct_cnt(txs.c.src) From ecc1150373ebb340f4557ce0b75ef10b439a6b73 Mon Sep 17 00:00:00 2001 From: Doug Baldwin Date: Tue, 16 Jul 2019 22:22:50 -0400 Subject: [PATCH 06/17] Add DB instructions to README --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9b829b5..4eddce3 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,17 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io 1. Install Libra per official instructions 2. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching 3. Open the official client, create an account and save the account to disk (should be set in ACCOUNT_FILE setting) -4. Edit config.json and make sure that settings match your environment (in particular CLIENT_PATH) +4. Edit config.json and make sure that settings match your environment (in particular CLIENT_PATH and sqlalchemy.url) + +## Database +* Default config assumes a newly created [postgresql](https://wiki.postgresql.org/wiki/Main_Page) database with: + username: postgres + password: postgres + host: localhost + port: 5432 + database name: libra_browser + [Please see SQLAlchemy Docs for configuration options](https://docs.sqlalchemy.org/en/13/core/engines.html) +* Alternatively, delete from config.json the line containing "sqlalchemy.url", for a simple default sqlite database in memory ## Running the project At the root project folder execute the command: From ba7028d67cc911869ccc19606578dcec511e457a Mon Sep 17 00:00:00 2001 From: Doug Baldwin Date: Tue, 16 Jul 2019 22:30:40 -0400 Subject: [PATCH 07/17] Update DB instructions to README --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4eddce3..c7aefdc 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,12 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io ## Database * Default config assumes a newly created [postgresql](https://wiki.postgresql.org/wiki/Main_Page) database with: - username: postgres - password: postgres - host: localhost - port: 5432 - database name: libra_browser - [Please see SQLAlchemy Docs for configuration options](https://docs.sqlalchemy.org/en/13/core/engines.html) + > username: postgres + > password: postgres + > host: localhost + > port: 5432 + > database name: libra_browser + > [Please see SQLAlchemy Docs for configuration options](https://docs.sqlalchemy.org/en/13/core/engines.html) * Alternatively, delete from config.json the line containing "sqlalchemy.url", for a simple default sqlite database in memory ## Running the project From 8f44e9b0b635a2255fc53904637e499f72787028 Mon Sep 17 00:00:00 2001 From: gdbaldw Date: Tue, 16 Jul 2019 22:37:09 -0400 Subject: [PATCH 08/17] Update README.md --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c7aefdc..9777f68 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,13 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io ## Database * Default config assumes a newly created [postgresql](https://wiki.postgresql.org/wiki/Main_Page) database with: - > username: postgres - > password: postgres - > host: localhost - > port: 5432 - > database name: libra_browser - > [Please see SQLAlchemy Docs for configuration options](https://docs.sqlalchemy.org/en/13/core/engines.html) -* Alternatively, delete from config.json the line containing "sqlalchemy.url", for a simple default sqlite database in memory + * username = postgres + * password = postgres + * host = localhost + * port = 5432 + * database name = libra_browser + * Please see [SQLAlchemy Docs](https://docs.sqlalchemy.org/en/13/core/engines.html) for configuration options +* Alternatively, delete from config.json the lines containing "sqlalchemy.url", for a simple default sqlite database in memory ## Running the project At the root project folder execute the command: From 637e40a21c5331f8af1064819ba6fb794108106d Mon Sep 17 00:00:00 2001 From: gdbaldw Date: Tue, 16 Jul 2019 22:39:28 -0400 Subject: [PATCH 09/17] Update config.json --- config.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/config.json b/config.json index 827141e..2a9c206 100644 --- a/config.json +++ b/config.json @@ -1,6 +1,5 @@ { "PRODUCTION" : { - "DB_PATH" : "./tx_cache.db", "CLIENT_PATH" : "~/libra/", "ACCOUNT_FILE" : "./test_acct", "RPC_SERVER" : "ac.testnet.libra.org:8000", @@ -12,7 +11,6 @@ "sqlalchemy.url": "postgresql://postgres:postgres@localhost:5432/libra_browser" }, "DEVELOPMENT" : { - "DB_PATH" : "./tx_cache.db", "CLIENT_PATH" : "~/Source/libra/", "ACCOUNT_FILE" : "./test_acct", "RPC_SERVER" : "ac.testnet.libra.org:8000", @@ -23,7 +21,6 @@ "FLASK_THREADED" : false }, "STAGING" : { - "DB_PATH" : "./tx_cache.db", "CLIENT_PATH" : "~/libra/", "ACCOUNT_FILE" : "./test_acct", "RPC_SERVER" : "ac.testnet.libra.org:8000", From 702693857c5eb42a873c6d84fa208b84964ae75c Mon Sep 17 00:00:00 2001 From: gdbaldw Date: Fri, 19 Jul 2019 00:43:27 -0400 Subject: [PATCH 10/17] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9777f68..ea2dd1d 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io ## Installation 1. Install Libra per official instructions -2. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching +2. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching sqlalchemy psycopg2 3. Open the official client, create an account and save the account to disk (should be set in ACCOUNT_FILE setting) 4. Edit config.json and make sure that settings match your environment (in particular CLIENT_PATH and sqlalchemy.url) @@ -25,6 +25,9 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io * port = 5432 * database name = libra_browser * Please see [SQLAlchemy Docs](https://docs.sqlalchemy.org/en/13/core/engines.html) for configuration options +* make sure that /etc/postgresql//main/pg_hba.conf has the configuration of password, i.e. auth method md5 and not peer: +> local all postgres md5 +* To create the DB after installing postgresql you can run: sudo -u postgres createdb libra_browser * Alternatively, delete from config.json the lines containing "sqlalchemy.url", for a simple default sqlite database in memory ## Running the project @@ -43,3 +46,4 @@ To use "DEVELOPMENT" mode settings set the environment variable "BROWSER=DEVELOP ## Credits rpc support is based on: https://github.com/egorsmkv/libra-grpc-py Contributors: [@gdbaldw](https://github.com/gdbaldw) [@lucasverra](https://github.com/lucasverra) + From 4a1c95a41be1bfab26de8e49b9e1092c73b4529c Mon Sep 17 00:00:00 2001 From: Doug Baldwin Date: Fri, 19 Jul 2019 00:59:12 -0400 Subject: [PATCH 11/17] add automatic handling for testnet reset --- Browser.py | 5 +- README.md | 10 ++-- config.json | 28 ++++++++++-- db_funcs.py | 129 ++++++++++++++++++++++++++++++++++++++-------------- stats.py | 45 +++--------------- 5 files changed, 131 insertions(+), 86 deletions(-) diff --git a/Browser.py b/Browser.py index 6fa5f2c..8f0c7f1 100644 --- a/Browser.py +++ b/Browser.py @@ -207,7 +207,6 @@ def stats(): app.logger.exception('error in stats') return ret - @app.route('/faucet', methods=['GET', 'POST']) def faucet(): update_counters() @@ -260,9 +259,7 @@ def send_asset(path): app.logger.info("system configuration: {}".format(json.dumps(config, indent=4))) - if not 'DB_PATH' in config.keys(): - config['DB_PATH'] = './fake.db' - TxDBWorker(config['DB_PATH'], config['RPC_SERVER'], config['MINT_ACCOUNT']).start() + TxDBWorker(config).start() start_rpc_client_instance(config['RPC_SERVER'], config['MINT_ACCOUNT']) diff --git a/README.md b/README.md index ea2dd1d..a85ad17 100644 --- a/README.md +++ b/README.md @@ -13,22 +13,18 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io ## Installation 1. Install Libra per official instructions -2. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching sqlalchemy psycopg2 +2. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching 3. Open the official client, create an account and save the account to disk (should be set in ACCOUNT_FILE setting) -4. Edit config.json and make sure that settings match your environment (in particular CLIENT_PATH and sqlalchemy.url) +4. Edit config.json and make sure that settings match your environment (in particular CLIENT_PATH) ## Database -* Default config assumes a newly created [postgresql](https://wiki.postgresql.org/wiki/Main_Page) database with: +* Default config assumes a [postgresql](https://wiki.postgresql.org/wiki/Main_Page) database with: * username = postgres * password = postgres * host = localhost * port = 5432 * database name = libra_browser * Please see [SQLAlchemy Docs](https://docs.sqlalchemy.org/en/13/core/engines.html) for configuration options -* make sure that /etc/postgresql//main/pg_hba.conf has the configuration of password, i.e. auth method md5 and not peer: -> local all postgres md5 -* To create the DB after installing postgresql you can run: sudo -u postgres createdb libra_browser -* Alternatively, delete from config.json the lines containing "sqlalchemy.url", for a simple default sqlite database in memory ## Running the project At the root project folder execute the command: diff --git a/config.json b/config.json index 2a9c206..64bc6d6 100644 --- a/config.json +++ b/config.json @@ -8,7 +8,14 @@ "FLASK_PORT" : 5000, "FLASK_DEBUG" : false, "FLASK_THREADED" : false, - "sqlalchemy.url": "postgresql://postgres:postgres@localhost:5432/libra_browser" + "DB_DIALECT" : "postgres", + "DB_DRIVER" : "psycopg2", + "DB_USERNAME" : "postgres", + "DB_PASSWORD" : "postgres", + "DB_HOST" : "localhost", + "DB_PORT" : 5432, + "DB_NAME" : "libra_browser", + "DB_BACKUP_PATH" : "./db_backup" }, "DEVELOPMENT" : { "CLIENT_PATH" : "~/Source/libra/", @@ -18,7 +25,15 @@ "FLASK_HOST" : "127.0.0.1", "FLASK_PORT" : 5000, "FLASK_DEBUG" : false, - "FLASK_THREADED" : false + "FLASK_THREADED" : false, + "DB_DIALECT" : "postgres", + "DB_DRIVER" : "psycopg2", + "DB_USERNAME" : "postgres", + "DB_PASSWORD" : "postgres", + "DB_HOST" : "localhost", + "DB_PORT" : 5432, + "DB_NAME" : "libra_browser", + "DB_BACKUP_PATH" : "./db_backup" }, "STAGING" : { "CLIENT_PATH" : "~/libra/", @@ -29,6 +44,13 @@ "FLASK_PORT" : 5001, "FLASK_DEBUG" : false, "FLASK_THREADED" : false, - "sqlalchemy.url": "postgresql://postgres:postgres@localhost:5432/libra_browser" + "DB_DIALECT" : "postgres", + "DB_DRIVER" : "psycopg2", + "DB_USERNAME" : "postgres", + "DB_PASSWORD" : "postgres", + "DB_HOST" : "localhost", + "DB_PORT" : 5432, + "DB_NAME" : "libra_browser", + "DB_BACKUP_PATH" : "./db_backup" } } diff --git a/db_funcs.py b/db_funcs.py index a215f6a..e1c0464 100644 --- a/db_funcs.py +++ b/db_funcs.py @@ -11,20 +11,21 @@ ########### from sqlalchemy import create_engine, engine_from_config, Table, Column, Integer, BigInteger, LargeBinary, String, MetaData, select, desc, func from sqlalchemy.pool import StaticPool +from sqlalchemy.ext.serializer import dumps from threading import Thread import sys -from time import sleep +from time import sleep, gmtime, strftime import json import struct +import gzip from rpc_client import get_latest_version_from_ledger, get_raw_tx_lst, parse_raw_tx_lst, start_rpc_client_instance -############ -# Database # -############ +############# +# Constants # +############# -metadata = MetaData() -txs = Table('transactions', metadata, +columns = ( Column('version', Integer, primary_key=True), Column('expiration_date', String), Column('src', String), @@ -44,30 +45,41 @@ Column('code_hex', String), Column('program', String), ) +metadata = MetaData() +txs = Table('transactions', metadata, *columns) -with open('config.json', 'r') as f: - config = json.load(f) -try: - config = config[os.getenv("BROWSER")] -except: - config = config["PRODUCTION"] - -if 'sqlalchemy.url' in config: - engine = create_engine(config['sqlalchemy.url']) -else: - engine = create_engine( - 'sqlite://', - connect_args={'check_same_thread': False}, - poolclass = StaticPool, - ) +########### +# Globals # +########### -metadata.create_all(engine) +engine = None + +#with open('config.json', 'r') as f: +# config = json.load(f) +#try: +# config = config[os.getenv("BROWSER")] +#except: +# config = config["PRODUCTION"] +# +#if 'sqlalchemy.url' in config: +# engine = create_engine(config['sqlalchemy.url']) +#else: +# engine = create_engine( +# 'sqlite://', +# connect_args={'check_same_thread': False}, +# poolclass = StaticPool, +# ) +# +#metadata.create_all(engine) ######### # Funcs # ######### +unpack = lambda x: struct.unpack(' 0 + ) + ) + ).scalar() + +def get_tx_cnt_sum(whereclause, s_limit): + global engine + selected = engine.execute( + s_limit( + select( + [txs.c.amount] + ).where( + whereclause + ).distinct(txs.c.version) + ) + ).fetchall() + return len(selected), sum(map(lambda r: unpack(r['amount']), selected)) + +def get_acct_cnt(acct, s_limit): + global engine + return engine.execute( + s_limit( + select( + [func.count(acct.distinct())] + ).where( + txs.c.version > 0 + ) + ) + ).scalar() + + ############# # DB Worker # ############# class TxDBWorker(Thread): - def __init__(self, db_path, rpc_server, mint_addr): + def __init__(self, config): Thread.__init__(self) - self.db_path = db_path - started = False - logger.info('transactions db worker starting') - while not started: + self.url = "{DB_DIALECT}+{DB_DRIVER}://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}".format(**config) + logger.info('sqlalchemy.url: {}'.format(self.url)) + self.db_backup_path = config['DB_BACKUP_PATH'] + running = False + while not running: try: - start_rpc_client_instance(rpc_server, mint_addr) - started = True + start_rpc_client_instance(config['RPC_SERVER'], config['MINT_ACCOUNT']) + running = True except: sleep(10) def run(self): + global engine while True: + logger.info('transactions db worker starting') + engine = create_engine(self.url) + metadata.create_all(engine) try: # get latest version in the db cur_ver = get_latest_version() @@ -131,8 +188,13 @@ def run(self): sleep(1) continue if cur_ver > bver: - sleep(1) - continue + file_path = self.db_backup_path + '_' + strftime('%Y%m%d%H%M%S') + '.gz' + logger.info('saving database to {}'.format(file_path)) + with gzip.open(file_path, 'wb') as f: + f.write(dumps(engine.execute(select([txs])).fetchall())) + metadata.drop_all(engine) + metadata.create_all(engine) + break # batch update num = min(1000, bver - cur_ver) # at most 5000 records at once @@ -140,7 +202,7 @@ def run(self): # read records res = parse_raw_tx_lst(*tx_data) - if len(res) == 0: + if not res: sleep(5) continue @@ -160,4 +222,3 @@ def run(self): except: logger.exception('Major error in tx_db_worker') sleep(2) - logger.info('restarting tx_db_worker') diff --git a/stats.py b/stats.py index 85e75f6..075c96f 100644 --- a/stats.py +++ b/stats.py @@ -10,7 +10,7 @@ # Imports # ########### from datetime import datetime, timedelta -from db_funcs import get_tx_from_db_by_version, get_latest_version, engine, txs +from db_funcs import get_tx_from_db_by_version, get_latest_version, get_first_version, get_tx_cnt_sum, get_acct_cnt, txs from sqlalchemy import select, desc, func import struct @@ -18,8 +18,6 @@ # Funcs # ######### -unpack = lambda x: struct.unpack(' 0 - ) - ) - ).scalar() + first_version = get_first_version(s_limit) if not first_version: first_version = 1 logger.info('first ver = {}'.format(first_version)) @@ -59,39 +49,18 @@ def calc_stats(limit = None): logger.info('deltas: {} {}'.format(td, blocks_delta)) # mint p2p other - def get_tx_cnt_sum(whereclause): - selected = engine.execute( - s_limit( - select( - [txs.c.amount] - ).where( - whereclause - ).distinct(txs.c.version) - ) - ).fetchall() - return len(selected), sum(map(lambda r: unpack(r['amount']), selected)) - mint_count, mint_sum = get_tx_cnt_sum(txs.c.type == 'mint_transaction') + mint_count, mint_sum = get_tx_cnt_sum(txs.c.type == 'mint_transaction', s_limit) logger.info('mint {} {}'.format(mint_count, mint_sum)) - p2p_count, p2p_sum = get_tx_cnt_sum(txs.c.type == 'peer_to_peer_transaction') + p2p_count, p2p_sum = get_tx_cnt_sum(txs.c.type == 'peer_to_peer_transaction', s_limit) logger.info('p2p {} {}'.format(p2p_count, p2p_sum)) - other_count, other_sum = get_tx_cnt_sum((txs.c.type != 'mint_transaction') & (txs.c.type != 'peer_to_peer_transaction')) + other_count, other_sum = get_tx_cnt_sum((txs.c.type != 'mint_transaction') & (txs.c.type != 'peer_to_peer_transaction'), s_limit) # add 1 to account for the genesis block until it is added to DB other_count += 1 logger.info('others {} {}'.format(other_count, other_sum)) # unique accounts - def get_acct_cnt(acct): - return engine.execute( - s_limit( - select( - [func.count(acct.distinct())] - ).where( - txs.c.version > 0 - ) - ) - ).scalar() - count_dest = get_acct_cnt(txs.c.dest) - count_src = get_acct_cnt(txs.c.src) + count_dest = get_acct_cnt(txs.c.dest, s_limit) + count_src = get_acct_cnt(txs.c.src, s_limit) return (blocks_delta, *dhms, blocks_delta/td.total_seconds(), 100*mint_count/blocks_delta, 100*p2p_count/blocks_delta, 100*other_count/blocks_delta, mint_sum, p2p_sum, other_sum, From 988c830cd3a96f44efb847e6e42bbde9b99ec50f Mon Sep 17 00:00:00 2001 From: gdbaldw Date: Fri, 19 Jul 2019 01:05:54 -0400 Subject: [PATCH 12/17] remove commented out code --- db_funcs.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/db_funcs.py b/db_funcs.py index e1c0464..0f851a9 100644 --- a/db_funcs.py +++ b/db_funcs.py @@ -54,24 +54,6 @@ engine = None -#with open('config.json', 'r') as f: -# config = json.load(f) -#try: -# config = config[os.getenv("BROWSER")] -#except: -# config = config["PRODUCTION"] -# -#if 'sqlalchemy.url' in config: -# engine = create_engine(config['sqlalchemy.url']) -#else: -# engine = create_engine( -# 'sqlite://', -# connect_args={'check_same_thread': False}, -# poolclass = StaticPool, -# ) -# -#metadata.create_all(engine) - ######### # Funcs # ######### From 69f4643986bbe768b7a58e195dc5cfa931abad0e Mon Sep 17 00:00:00 2001 From: gdbaldw Date: Fri, 19 Jul 2019 01:08:13 -0400 Subject: [PATCH 13/17] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index a85ad17..2b0f214 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io * port = 5432 * database name = libra_browser * Please see [SQLAlchemy Docs](https://docs.sqlalchemy.org/en/13/core/engines.html) for configuration options +* make sure that /etc/postgresql//main/pg_hba.conf has the configuration of password, i.e. auth method md5 and not peer: +> local all postgres md5 +* To create the DB after installing postgresql you can run: sudo -u postgres createdb libra_browser ## Running the project At the root project folder execute the command: From 95f425c52cccbcd30a5345d1efe2662b31bde3a6 Mon Sep 17 00:00:00 2001 From: gdbaldw Date: Fri, 19 Jul 2019 01:12:05 -0400 Subject: [PATCH 14/17] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b0f214..de623fb 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io ## Installation 1. Install Libra per official instructions -2. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching +2. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching sqlalchemy psycopg2 3. Open the official client, create an account and save the account to disk (should be set in ACCOUNT_FILE setting) 4. Edit config.json and make sure that settings match your environment (in particular CLIENT_PATH) From f4ca341f1ae8ca2b6ff348068d22f563c9682144 Mon Sep 17 00:00:00 2001 From: gdbaldw Date: Fri, 19 Jul 2019 01:50:56 -0400 Subject: [PATCH 15/17] Update db_funcs.py --- db_funcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_funcs.py b/db_funcs.py index 0f851a9..71032bb 100644 --- a/db_funcs.py +++ b/db_funcs.py @@ -170,7 +170,7 @@ def run(self): sleep(1) continue if cur_ver > bver: - file_path = self.db_backup_path + '_' + strftime('%Y%m%d%H%M%S') + '.gz' + file_path = '{}_{}.gz'.format(self.db_backup_path, strftime('%Y%m%d%H%M%S')) logger.info('saving database to {}'.format(file_path)) with gzip.open(file_path, 'wb') as f: f.write(dumps(engine.execute(select([txs])).fetchall())) From 464b90387c4020073a752421a49dc3fd0d2c74de Mon Sep 17 00:00:00 2001 From: gdbaldw Date: Sat, 20 Jul 2019 11:38:46 -0400 Subject: [PATCH 16/17] Update db_funcs.py --- db_funcs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/db_funcs.py b/db_funcs.py index 71032bb..7496638 100644 --- a/db_funcs.py +++ b/db_funcs.py @@ -170,6 +170,9 @@ def run(self): sleep(1) continue if cur_ver > bver: + if cur_ver > bver + 50: # for safety due to typical blockchain behavior + sleep(1) + continue file_path = '{}_{}.gz'.format(self.db_backup_path, strftime('%Y%m%d%H%M%S')) logger.info('saving database to {}'.format(file_path)) with gzip.open(file_path, 'wb') as f: From 9e6e71a1db6b76e90a8b4fc83f4de73a987ca8b1 Mon Sep 17 00:00:00 2001 From: Doug Baldwin Date: Tue, 23 Jul 2019 23:33:49 -0400 Subject: [PATCH 17/17] faucet upgrade --- Browser.py | 18 ++++++------ README.md | 6 ++-- client.py | 81 ----------------------------------------------------- config.json | 9 ++---- 4 files changed, 14 insertions(+), 100 deletions(-) delete mode 100644 client.py diff --git a/Browser.py b/Browser.py index 8f0c7f1..31ab02a 100644 --- a/Browser.py +++ b/Browser.py @@ -17,11 +17,11 @@ import re import sys import os +import requests from time import sleep from rpc_client import get_acct_raw, get_acct_info, start_rpc_client_instance -from client import start_client_instance, do_cmd from db_funcs import get_latest_version, get_tx_from_db_by_version, get_all_account_tx, TxDBWorker from stats import calc_stats @@ -218,17 +218,19 @@ def faucet(): try: acct = request.form.get('acct') app.logger.info('acct: {}'.format(acct)) - amount = request.form.get('amount') + amount = float(request.form.get('amount')) app.logger.info('amount: {}'.format(amount)) - if float(amount) < 0: + if amount < 0: message = 'Amount must be >= 0' elif not is_valid_account(acct): message = 'Invalid account format' else: - do_cmd('a mb 0 ' + str(float(amount)), p = p) - do_cmd('tb 0 ' + acct + ' ' + str(float(amount)), p = p) - acct_link = '{0}'.format(acct) - message = 'Sent ' + amount + ' Libra to ' + acct_link + response = requests.get( + config['FAUCET_HOST'], + params={"address": acct, "amount": str(int(amount * 1e6))} + ) + if response.status_code == 200: + message = 'Sent {0} Libra to {1}'.format(amount, acct) except: message = 'Invalid request logged!' app.logger.exception(message) @@ -263,8 +265,6 @@ def send_asset(path): start_rpc_client_instance(config['RPC_SERVER'], config['MINT_ACCOUNT']) - p = start_client_instance(config['CLIENT_PATH'], config['ACCOUNT_FILE']) - sleep(1) app.run(port=config['FLASK_PORT'], threaded=config['FLASK_THREADED'], diff --git a/README.md b/README.md index de623fb..c02cba7 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,8 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io * Simple Libra client automation (soon to be deprecated) ## Installation -1. Install Libra per official instructions -2. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching sqlalchemy psycopg2 -3. Open the official client, create an account and save the account to disk (should be set in ACCOUNT_FILE setting) -4. Edit config.json and make sure that settings match your environment (in particular CLIENT_PATH) +1. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching sqlalchemy psycopg2 requests +2. Have access to a Postgres Database server ## Database * Default config assumes a [postgresql](https://wiki.postgresql.org/wiki/Main_Page) database with: diff --git a/client.py b/client.py deleted file mode 100644 index 2655759..0000000 --- a/client.py +++ /dev/null @@ -1,81 +0,0 @@ -# Library to automate Libra client - -########## -# Logger # -########## -import logging -logger = logging.getLogger(__name__) - -########### -# Imports # -########### -import os -import re -import sys -from datetime import datetime -from subprocess import Popen, PIPE -from time import sleep - - -######### -# Funcs # -######### -def start_client_instance(client_path = '', account_file = ''): - c_path = os.path.expanduser(client_path + "target/debug/client") - args = [c_path, "--host", "ac.testnet.libra.org", "--port", "80", - "-s", "./scripts/cli/trusted_peers.config.toml"] - logger.info(' '.join(args)) - p = Popen(args, cwd=os.path.expanduser(client_path), - shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True, bufsize=0, universal_newlines=True) - sleep(5) - p.stdout.flush() - logger.info(os.read(p.stdout.fileno(), 10000).decode('unicode_escape')) - logger.info('loading account {}: {}'.format(account_file, do_cmd("a r " + account_file, p = p))) - sys.stdout.flush() - - return p - - -def do_cmd(cmd, delay=0.5, bufsize=50000, decode=True, p=None): - p.stdin.write(cmd+'\n') - p.stdin.flush() - sleep(delay) - p.stdout.flush() - if decode: - return os.read(p.stdout.fileno(), bufsize).decode('utf-8') - else: - return os.read(p.stdout.fileno(), bufsize) - - -def get_version_from_raw(s): - return next(re.finditer(r'(\d+)\s+$', s)).group(1) - - -def get_acct_info(raw_account_status): - try: - account = next(re.finditer(r'Account: ([a-z0-9]+)', raw_account_status)).group(1) - balance = str(int(next(re.finditer(r'balance: (\d+),', raw_account_status)).group(1)) / 1000000) - sq_num = next(re.finditer(r'sequence_number: (\d+),', raw_account_status)).group(1) - sent_events = next(re.finditer(r'sent_events_count: (\d+),', raw_account_status)).group(1) - recv_events = next(re.finditer(r'received_events_count: (\d+),', raw_account_status)).group(1) - except: - logger.exception('Error in getting account info') - - return account, balance, sq_num, sent_events, recv_events - - -def parse_raw_tx(raw): - ver = int(next(re.finditer(r'Transaction at version (\d+):', raw)).group(1)) - expiration_num = int(next(re.finditer(r'expiration_time: (\d+)s', raw)).group(1)) - expiration_num = min(expiration_num, 2147485547) # handle values above max unixtime - expiration = str(datetime.fromtimestamp(expiration_num)) - sender = next(re.finditer(r'sender: ([a-z0-9]+),', raw)).group(1) - target = next(re.finditer(r'ADDRESS: ([a-z0-9]+)', raw)).group(1) - t_type = next(re.finditer(r'transaction: ([a-z_-]+),', raw)).group(1) - amount = str(int(next(re.finditer(r'U64: (\d+)', raw)).group(1)) / 1000000) - gas_price = str(int(next(re.finditer(r'gas_unit_price: (\d+),', raw)).group(1)) / 1000000) - gas_max = str(int(next(re.finditer(r'max_gas_amount: (\d+),', raw)).group(1)) / 1000000) - sq_num = next(re.finditer(r'sequence_number: (\d+),', raw)).group(1) - pubkey = next(re.finditer(r'public_key: ([a-z0-9]+),', raw)).group(1) - - return ver, expiration, sender, target, t_type, amount, gas_price, gas_max, sq_num, pubkey, expiration_num diff --git a/config.json b/config.json index 64bc6d6..abd36c0 100644 --- a/config.json +++ b/config.json @@ -1,8 +1,7 @@ { "PRODUCTION" : { - "CLIENT_PATH" : "~/libra/", - "ACCOUNT_FILE" : "./test_acct", "RPC_SERVER" : "ac.testnet.libra.org:8000", + "FAUCET_HOST" : "http://faucet.testnet.libra.org", "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", "FLASK_HOST" : "0.0.0.0", "FLASK_PORT" : 5000, @@ -18,9 +17,8 @@ "DB_BACKUP_PATH" : "./db_backup" }, "DEVELOPMENT" : { - "CLIENT_PATH" : "~/Source/libra/", - "ACCOUNT_FILE" : "./test_acct", "RPC_SERVER" : "ac.testnet.libra.org:8000", + "FAUCET_HOST" : "http://faucet.testnet.libra.org", "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", "FLASK_HOST" : "127.0.0.1", "FLASK_PORT" : 5000, @@ -36,9 +34,8 @@ "DB_BACKUP_PATH" : "./db_backup" }, "STAGING" : { - "CLIENT_PATH" : "~/libra/", - "ACCOUNT_FILE" : "./test_acct", "RPC_SERVER" : "ac.testnet.libra.org:8000", + "FAUCET_HOST" : "http://faucet.testnet.libra.org", "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", "FLASK_HOST" : "0.0.0.0", "FLASK_PORT" : 5001,