diff --git a/Browser.py b/Browser.py index 6fa5f2c..31ab02a 100644 --- a/Browser.py +++ b/Browser.py @@ -17,11 +17,11 @@ import re import sys import os +import requests from time import sleep from rpc_client import get_acct_raw, get_acct_info, start_rpc_client_instance -from client import start_client_instance, do_cmd from db_funcs import get_latest_version, get_tx_from_db_by_version, get_all_account_tx, TxDBWorker from stats import calc_stats @@ -207,7 +207,6 @@ def stats(): app.logger.exception('error in stats') return ret - @app.route('/faucet', methods=['GET', 'POST']) def faucet(): update_counters() @@ -219,17 +218,19 @@ def faucet(): try: acct = request.form.get('acct') app.logger.info('acct: {}'.format(acct)) - amount = request.form.get('amount') + amount = float(request.form.get('amount')) app.logger.info('amount: {}'.format(amount)) - if float(amount) < 0: + if amount < 0: message = 'Amount must be >= 0' elif not is_valid_account(acct): message = 'Invalid account format' else: - do_cmd('a mb 0 ' + str(float(amount)), p = p) - do_cmd('tb 0 ' + acct + ' ' + str(float(amount)), p = p) - acct_link = '{0}'.format(acct) - message = 'Sent ' + amount + ' Libra to ' + acct_link + response = requests.get( + config['FAUCET_HOST'], + params={"address": acct, "amount": str(int(amount * 1e6))} + ) + if response.status_code == 200: + message = 'Sent {0} Libra to {1}'.format(amount, acct) except: message = 'Invalid request logged!' app.logger.exception(message) @@ -260,14 +261,10 @@ def send_asset(path): app.logger.info("system configuration: {}".format(json.dumps(config, indent=4))) - if not 'DB_PATH' in config.keys(): - config['DB_PATH'] = './fake.db' - TxDBWorker(config['DB_PATH'], config['RPC_SERVER'], config['MINT_ACCOUNT']).start() + TxDBWorker(config).start() start_rpc_client_instance(config['RPC_SERVER'], config['MINT_ACCOUNT']) - p = start_client_instance(config['CLIENT_PATH'], config['ACCOUNT_FILE']) - sleep(1) app.run(port=config['FLASK_PORT'], threaded=config['FLASK_THREADED'], diff --git a/README.md b/README.md index fa4d08e..c02cba7 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,11 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io * Simple Libra client automation (soon to be deprecated) ## Installation -1. Install Libra per official instructions -2. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching sqlalchemy psycopg2 -3. Open the official client, create an account and save the account to disk (should be set in ACCOUNT_FILE setting) -4. Edit config.json and make sure that settings match your environment (in particular CLIENT_PATH and sqlalchemy.url) +1. Run: pip3 install grpcio grpcio-tools hexdump Flask Flask-Caching sqlalchemy psycopg2 requests +2. Have access to a Postgres Database server ## Database -* Default config assumes a newly created [postgresql](https://wiki.postgresql.org/wiki/Main_Page) database with: +* Default config assumes a [postgresql](https://wiki.postgresql.org/wiki/Main_Page) database with: * username = postgres * password = postgres * host = localhost @@ -28,7 +26,6 @@ A Block Explorer for the Libra Blockchain TestNet. See: https://librabrowser.io * make sure that /etc/postgresql//main/pg_hba.conf has the configuration of password, i.e. auth method md5 and not peer: > local all postgres md5 * To create the DB after installing postgresql you can run: sudo -u postgres createdb libra_browser -* Alternatively, delete from config.json the lines containing "sqlalchemy.url", for a simple default sqlite database in memory ## Running the project At the root project folder execute the command: @@ -46,3 +43,4 @@ To use "DEVELOPMENT" mode settings set the environment variable "BROWSER=DEVELOP ## Credits rpc support is based on: https://github.com/egorsmkv/libra-grpc-py Contributors: [@gdbaldw](https://github.com/gdbaldw) [@lucasverra](https://github.com/lucasverra) + diff --git a/client.py b/client.py deleted file mode 100644 index 2655759..0000000 --- a/client.py +++ /dev/null @@ -1,81 +0,0 @@ -# Library to automate Libra client - -########## -# Logger # -########## -import logging -logger = logging.getLogger(__name__) - -########### -# Imports # -########### -import os -import re -import sys -from datetime import datetime -from subprocess import Popen, PIPE -from time import sleep - - -######### -# Funcs # -######### -def start_client_instance(client_path = '', account_file = ''): - c_path = os.path.expanduser(client_path + "target/debug/client") - args = [c_path, "--host", "ac.testnet.libra.org", "--port", "80", - "-s", "./scripts/cli/trusted_peers.config.toml"] - logger.info(' '.join(args)) - p = Popen(args, cwd=os.path.expanduser(client_path), - shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True, bufsize=0, universal_newlines=True) - sleep(5) - p.stdout.flush() - logger.info(os.read(p.stdout.fileno(), 10000).decode('unicode_escape')) - logger.info('loading account {}: {}'.format(account_file, do_cmd("a r " + account_file, p = p))) - sys.stdout.flush() - - return p - - -def do_cmd(cmd, delay=0.5, bufsize=50000, decode=True, p=None): - p.stdin.write(cmd+'\n') - p.stdin.flush() - sleep(delay) - p.stdout.flush() - if decode: - return os.read(p.stdout.fileno(), bufsize).decode('utf-8') - else: - return os.read(p.stdout.fileno(), bufsize) - - -def get_version_from_raw(s): - return next(re.finditer(r'(\d+)\s+$', s)).group(1) - - -def get_acct_info(raw_account_status): - try: - account = next(re.finditer(r'Account: ([a-z0-9]+)', raw_account_status)).group(1) - balance = str(int(next(re.finditer(r'balance: (\d+),', raw_account_status)).group(1)) / 1000000) - sq_num = next(re.finditer(r'sequence_number: (\d+),', raw_account_status)).group(1) - sent_events = next(re.finditer(r'sent_events_count: (\d+),', raw_account_status)).group(1) - recv_events = next(re.finditer(r'received_events_count: (\d+),', raw_account_status)).group(1) - except: - logger.exception('Error in getting account info') - - return account, balance, sq_num, sent_events, recv_events - - -def parse_raw_tx(raw): - ver = int(next(re.finditer(r'Transaction at version (\d+):', raw)).group(1)) - expiration_num = int(next(re.finditer(r'expiration_time: (\d+)s', raw)).group(1)) - expiration_num = min(expiration_num, 2147485547) # handle values above max unixtime - expiration = str(datetime.fromtimestamp(expiration_num)) - sender = next(re.finditer(r'sender: ([a-z0-9]+),', raw)).group(1) - target = next(re.finditer(r'ADDRESS: ([a-z0-9]+)', raw)).group(1) - t_type = next(re.finditer(r'transaction: ([a-z_-]+),', raw)).group(1) - amount = str(int(next(re.finditer(r'U64: (\d+)', raw)).group(1)) / 1000000) - gas_price = str(int(next(re.finditer(r'gas_unit_price: (\d+),', raw)).group(1)) / 1000000) - gas_max = str(int(next(re.finditer(r'max_gas_amount: (\d+),', raw)).group(1)) / 1000000) - sq_num = next(re.finditer(r'sequence_number: (\d+),', raw)).group(1) - pubkey = next(re.finditer(r'public_key: ([a-z0-9]+),', raw)).group(1) - - return ver, expiration, sender, target, t_type, amount, gas_price, gas_max, sq_num, pubkey, expiration_num diff --git a/config.json b/config.json index 2a9c206..abd36c0 100644 --- a/config.json +++ b/config.json @@ -1,34 +1,53 @@ { "PRODUCTION" : { - "CLIENT_PATH" : "~/libra/", - "ACCOUNT_FILE" : "./test_acct", "RPC_SERVER" : "ac.testnet.libra.org:8000", + "FAUCET_HOST" : "http://faucet.testnet.libra.org", "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", "FLASK_HOST" : "0.0.0.0", "FLASK_PORT" : 5000, "FLASK_DEBUG" : false, "FLASK_THREADED" : false, - "sqlalchemy.url": "postgresql://postgres:postgres@localhost:5432/libra_browser" + "DB_DIALECT" : "postgres", + "DB_DRIVER" : "psycopg2", + "DB_USERNAME" : "postgres", + "DB_PASSWORD" : "postgres", + "DB_HOST" : "localhost", + "DB_PORT" : 5432, + "DB_NAME" : "libra_browser", + "DB_BACKUP_PATH" : "./db_backup" }, "DEVELOPMENT" : { - "CLIENT_PATH" : "~/Source/libra/", - "ACCOUNT_FILE" : "./test_acct", "RPC_SERVER" : "ac.testnet.libra.org:8000", + "FAUCET_HOST" : "http://faucet.testnet.libra.org", "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", "FLASK_HOST" : "127.0.0.1", "FLASK_PORT" : 5000, "FLASK_DEBUG" : false, - "FLASK_THREADED" : false + "FLASK_THREADED" : false, + "DB_DIALECT" : "postgres", + "DB_DRIVER" : "psycopg2", + "DB_USERNAME" : "postgres", + "DB_PASSWORD" : "postgres", + "DB_HOST" : "localhost", + "DB_PORT" : 5432, + "DB_NAME" : "libra_browser", + "DB_BACKUP_PATH" : "./db_backup" }, "STAGING" : { - "CLIENT_PATH" : "~/libra/", - "ACCOUNT_FILE" : "./test_acct", "RPC_SERVER" : "ac.testnet.libra.org:8000", + "FAUCET_HOST" : "http://faucet.testnet.libra.org", "MINT_ACCOUNT" : "0000000000000000000000000000000000000000000000000000000000000000", "FLASK_HOST" : "0.0.0.0", "FLASK_PORT" : 5001, "FLASK_DEBUG" : false, "FLASK_THREADED" : false, - "sqlalchemy.url": "postgresql://postgres:postgres@localhost:5432/libra_browser" + "DB_DIALECT" : "postgres", + "DB_DRIVER" : "psycopg2", + "DB_USERNAME" : "postgres", + "DB_PASSWORD" : "postgres", + "DB_HOST" : "localhost", + "DB_PORT" : 5432, + "DB_NAME" : "libra_browser", + "DB_BACKUP_PATH" : "./db_backup" } } diff --git a/db_funcs.py b/db_funcs.py index a215f6a..7496638 100644 --- a/db_funcs.py +++ b/db_funcs.py @@ -11,20 +11,21 @@ ########### from sqlalchemy import create_engine, engine_from_config, Table, Column, Integer, BigInteger, LargeBinary, String, MetaData, select, desc, func from sqlalchemy.pool import StaticPool +from sqlalchemy.ext.serializer import dumps from threading import Thread import sys -from time import sleep +from time import sleep, gmtime, strftime import json import struct +import gzip from rpc_client import get_latest_version_from_ledger, get_raw_tx_lst, parse_raw_tx_lst, start_rpc_client_instance -############ -# Database # -############ +############# +# Constants # +############# -metadata = MetaData() -txs = Table('transactions', metadata, +columns = ( Column('version', Integer, primary_key=True), Column('expiration_date', String), Column('src', String), @@ -44,30 +45,23 @@ Column('code_hex', String), Column('program', String), ) +metadata = MetaData() +txs = Table('transactions', metadata, *columns) -with open('config.json', 'r') as f: - config = json.load(f) -try: - config = config[os.getenv("BROWSER")] -except: - config = config["PRODUCTION"] - -if 'sqlalchemy.url' in config: - engine = create_engine(config['sqlalchemy.url']) -else: - engine = create_engine( - 'sqlite://', - connect_args={'check_same_thread': False}, - poolclass = StaticPool, - ) +########### +# Globals # +########### -metadata.create_all(engine) +engine = None ######### # Funcs # ######### +unpack = lambda x: struct.unpack(' 0 + ) + ) + ).scalar() + +def get_tx_cnt_sum(whereclause, s_limit): + global engine + selected = engine.execute( + s_limit( + select( + [txs.c.amount] + ).where( + whereclause + ).distinct(txs.c.version) + ) + ).fetchall() + return len(selected), sum(map(lambda r: unpack(r['amount']), selected)) + +def get_acct_cnt(acct, s_limit): + global engine + return engine.execute( + s_limit( + select( + [func.count(acct.distinct())] + ).where( + txs.c.version > 0 + ) + ) + ).scalar() + + ############# # DB Worker # ############# class TxDBWorker(Thread): - def __init__(self, db_path, rpc_server, mint_addr): + def __init__(self, config): Thread.__init__(self) - self.db_path = db_path - started = False - logger.info('transactions db worker starting') - while not started: + self.url = "{DB_DIALECT}+{DB_DRIVER}://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}".format(**config) + logger.info('sqlalchemy.url: {}'.format(self.url)) + self.db_backup_path = config['DB_BACKUP_PATH'] + running = False + while not running: try: - start_rpc_client_instance(rpc_server, mint_addr) - started = True + start_rpc_client_instance(config['RPC_SERVER'], config['MINT_ACCOUNT']) + running = True except: sleep(10) def run(self): + global engine while True: + logger.info('transactions db worker starting') + engine = create_engine(self.url) + metadata.create_all(engine) try: # get latest version in the db cur_ver = get_latest_version() @@ -131,8 +170,16 @@ def run(self): sleep(1) continue if cur_ver > bver: - sleep(1) - continue + if cur_ver > bver + 50: # for safety due to typical blockchain behavior + sleep(1) + continue + file_path = '{}_{}.gz'.format(self.db_backup_path, strftime('%Y%m%d%H%M%S')) + logger.info('saving database to {}'.format(file_path)) + with gzip.open(file_path, 'wb') as f: + f.write(dumps(engine.execute(select([txs])).fetchall())) + metadata.drop_all(engine) + metadata.create_all(engine) + break # batch update num = min(1000, bver - cur_ver) # at most 5000 records at once @@ -140,7 +187,7 @@ def run(self): # read records res = parse_raw_tx_lst(*tx_data) - if len(res) == 0: + if not res: sleep(5) continue @@ -160,4 +207,3 @@ def run(self): except: logger.exception('Major error in tx_db_worker') sleep(2) - logger.info('restarting tx_db_worker') diff --git a/stats.py b/stats.py index 85e75f6..075c96f 100644 --- a/stats.py +++ b/stats.py @@ -10,7 +10,7 @@ # Imports # ########### from datetime import datetime, timedelta -from db_funcs import get_tx_from_db_by_version, get_latest_version, engine, txs +from db_funcs import get_tx_from_db_by_version, get_latest_version, get_first_version, get_tx_cnt_sum, get_acct_cnt, txs from sqlalchemy import select, desc, func import struct @@ -18,8 +18,6 @@ # Funcs # ######### -unpack = lambda x: struct.unpack(' 0 - ) - ) - ).scalar() + first_version = get_first_version(s_limit) if not first_version: first_version = 1 logger.info('first ver = {}'.format(first_version)) @@ -59,39 +49,18 @@ def calc_stats(limit = None): logger.info('deltas: {} {}'.format(td, blocks_delta)) # mint p2p other - def get_tx_cnt_sum(whereclause): - selected = engine.execute( - s_limit( - select( - [txs.c.amount] - ).where( - whereclause - ).distinct(txs.c.version) - ) - ).fetchall() - return len(selected), sum(map(lambda r: unpack(r['amount']), selected)) - mint_count, mint_sum = get_tx_cnt_sum(txs.c.type == 'mint_transaction') + mint_count, mint_sum = get_tx_cnt_sum(txs.c.type == 'mint_transaction', s_limit) logger.info('mint {} {}'.format(mint_count, mint_sum)) - p2p_count, p2p_sum = get_tx_cnt_sum(txs.c.type == 'peer_to_peer_transaction') + p2p_count, p2p_sum = get_tx_cnt_sum(txs.c.type == 'peer_to_peer_transaction', s_limit) logger.info('p2p {} {}'.format(p2p_count, p2p_sum)) - other_count, other_sum = get_tx_cnt_sum((txs.c.type != 'mint_transaction') & (txs.c.type != 'peer_to_peer_transaction')) + other_count, other_sum = get_tx_cnt_sum((txs.c.type != 'mint_transaction') & (txs.c.type != 'peer_to_peer_transaction'), s_limit) # add 1 to account for the genesis block until it is added to DB other_count += 1 logger.info('others {} {}'.format(other_count, other_sum)) # unique accounts - def get_acct_cnt(acct): - return engine.execute( - s_limit( - select( - [func.count(acct.distinct())] - ).where( - txs.c.version > 0 - ) - ) - ).scalar() - count_dest = get_acct_cnt(txs.c.dest) - count_src = get_acct_cnt(txs.c.src) + count_dest = get_acct_cnt(txs.c.dest, s_limit) + count_src = get_acct_cnt(txs.c.src, s_limit) return (blocks_delta, *dhms, blocks_delta/td.total_seconds(), 100*mint_count/blocks_delta, 100*p2p_count/blocks_delta, 100*other_count/blocks_delta, mint_sum, p2p_sum, other_sum,