From e4cb908fe3388c29d5c034f71fdf90bfe45c3831 Mon Sep 17 00:00:00 2001 From: mwbrulhardt Date: Wed, 10 Jul 2019 19:18:42 -0400 Subject: [PATCH 1/5] Added the code for the Base and Dummy exchange classes to the exchange API. --- lib/data/features/indicators.py | 2 +- lib/env/TradingEnv.py | 115 ++++++------------------- lib/exchange/__init__.py | 0 lib/exchange/exchanges.py | 148 ++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 88 deletions(-) create mode 100644 lib/exchange/__init__.py create mode 100644 lib/exchange/exchanges.py diff --git a/lib/data/features/indicators.py b/lib/data/features/indicators.py index b5e8880..833e952 100644 --- a/lib/data/features/indicators.py +++ b/lib/data/features/indicators.py @@ -48,8 +48,8 @@ def add_indicators(df) -> pd.DataFrame: + wrapper = lambda func, args: func(*args) for name, f, arg_names in indicators: - wrapper = lambda func, args: func(*args) args = [df[arg_name] for arg_name in arg_names] df[name] = wrapper(f, args) df.fillna(method='bfill', inplace=True) diff --git a/lib/env/TradingEnv.py b/lib/env/TradingEnv.py index 4f1a643..2cd5e2e 100644 --- a/lib/env/TradingEnv.py +++ b/lib/env/TradingEnv.py @@ -10,6 +10,7 @@ from lib.env.reward import BaseRewardStrategy, IncrementalProfit from lib.data.providers import BaseDataProvider from lib.data.features.transform import max_min_normalize, mean_normalize, log_and_difference, difference +from lib.exchange.exchanges import BaseExchange, DummyExchange, ExchangeMode from lib.util.logger import init_logger @@ -26,23 +27,17 @@ class TradingEnv(gym.Env): def __init__(self, data_provider: BaseDataProvider, + exchange: BaseExchange = DummyExchange, reward_strategy: BaseRewardStrategy = IncrementalProfit, - initial_balance: int = 10000, - commission: float = 0.0025, **kwargs): super(TradingEnv, self).__init__() self.logger = kwargs.get('logger', init_logger(__name__, show_debug=kwargs.get('show_debug', True))) - self.base_precision: int = kwargs.get('base_precision', 2) - self.asset_precision: int = kwargs.get('asset_precision', 8) - self.min_cost_limit: float = kwargs.get('min_cost_limit', 1E-3) - self.min_amount_limit: float = kwargs.get('min_amount_limit', 1E-3) - self.data_provider = data_provider + self.exchange = DummyExchange(data_provider) if exchange == DummyExchange else exchange + self.reward_strategy = reward_strategy() - self.initial_balance = round(initial_balance, self.base_precision) - self.commission = commission self.render_benchmarks: List[Dict] = kwargs.get('render_benchmarks', []) self.normalize_obs: bool = kwargs.get('normalize_obs', True) @@ -59,73 +54,29 @@ def __init__(self, self.observations = pd.DataFrame(None, columns=self.data_provider.columns) - def _current_price(self, ohlcv_key: str = 'Close'): - return float(self.current_ohlcv[ohlcv_key]) - - def _make_trade(self, action: int, current_price: float): + def _take_action(self, action: int): action_type: TradingEnvAction = TradingEnvAction(action % 3) action_amount = float(1 / (action % 8 + 1)) - asset_bought = 0 - asset_sold = 0 - cost_of_asset = 0 - revenue_from_sold = 0 - - if action_type == TradingEnvAction.BUY and self.balance >= self.min_cost_limit: - buy_price = round(current_price * (1 + self.commission), self.base_precision) - cost_of_asset = round(self.balance * action_amount, self.base_precision) - asset_bought = round(cost_of_asset / buy_price, self.asset_precision) - - self.last_bought = self.current_step - self.asset_held += asset_bought - self.balance -= cost_of_asset - - self.trades.append({'step': self.current_step, 'amount': asset_bought, - 'total': cost_of_asset, 'type': 'buy'}) - - elif action_type == TradingEnvAction.SELL and self.asset_held >= self.min_amount_limit: - sell_price = round(current_price * (1 - self.commission), self.base_precision) - asset_sold = round(self.asset_held * action_amount, self.asset_precision) - revenue_from_sold = round(asset_sold * sell_price, self.base_precision) - - self.last_sold = self.current_step - self.asset_held -= asset_sold - self.balance += revenue_from_sold + if action_type == TradingEnvAction.BUY: + self.exchange.buy(action_amount) - self.trades.append({'step': self.current_step, 'amount': asset_sold, - 'total': revenue_from_sold, 'type': 'sell'}) - - return asset_bought, asset_sold, cost_of_asset, revenue_from_sold - - def _take_action(self, action: int): - current_price = self._current_price() - - asset_bought, asset_sold, cost_of_asset, revenue_from_sold = self._make_trade(action, current_price) - - current_net_worth = round(self.balance + self.asset_held * current_price, self.base_precision) - self.net_worths.append(current_net_worth) - - self.account_history = self.account_history.append({ - 'balance': self.balance, - 'asset_bought': asset_bought, - 'cost_of_asset': cost_of_asset, - 'asset_sold': asset_sold, - 'revenue_from_sold': revenue_from_sold, - }, ignore_index=True) + elif action_type == TradingEnvAction.SELL: + self.exchange.sell(action_amount) def _done(self): - lost_90_percent_net_worth = float(self.net_worths[-1]) < (self.initial_balance / 10) + lost_90_percent_net_worth = float(self.exchange.net_worths[-1]) < (self.exchange.initial_balance / 10) has_next_frame = self.data_provider.has_next_ohlcv() return lost_90_percent_net_worth or not has_next_frame def _reward(self): reward = self.reward_strategy.get_reward(observations=self.observations, - net_worths=self.net_worths, - account_history=self.account_history, - last_bought=self.last_bought, - last_sold=self.last_sold, - current_price=self._current_price()) + net_worths=self.exchange.net_worths, + account_history=self.exchange.account_history, + last_bought=self.exchange.last_bought, + last_sold=self.exchange.last_sold, + current_price=self.exchange.price()) reward = float(reward) if np.isfinite(float(reward)) else 0 @@ -158,9 +109,9 @@ def _next_observation(self): obs = observations.values[-1] if self.stationarize_obs: - scaled_history = log_and_difference(self.account_history, inplace=False) + scaled_history = log_and_difference(self.exchange.get_account_history(), inplace=False) else: - scaled_history = self.account_history + scaled_history = self.exchange.get_account_history() if self.normalize_obs: scaled_history = max_min_normalize(scaled_history, inplace=False) @@ -175,21 +126,9 @@ def _next_observation(self): def reset(self): self.data_provider.reset_ohlcv_index() - self.balance = self.initial_balance - self.net_worths = [self.initial_balance] - self.asset_held = 0 - self.current_step = 0 - self.last_bought = 0 - self.last_sold = 0 - - self.account_history = pd.DataFrame([{ - 'balance': self.balance, - 'asset_bought': 0, - 'cost_of_asset': 0, - 'asset_sold': 0, - 'revenue_from_sold': 0, - }]) - self.trades = [] + assert isinstance(self.exchange, DummyExchange) + + self.exchange.reset() self.rewards = [0] return self._next_observation() @@ -206,20 +145,22 @@ def step(self, action): return obs, reward, done, {} def render(self, mode='human'): + if mode == 'system': - self.logger.info('Price: ' + str(self._current_price())) - self.logger.info('Bought: ' + str(self.account_history[2][self.current_step])) - self.logger.info('Sold: ' + str(self.account_history[4][self.current_step])) - self.logger.info('Net worth: ' + str(self.net_worths[-1])) + account_history = self.exchange.account_history + self.logger.info('Price: ' + str(self.exchange.price())) + self.logger.info('Bought: ' + str(account_history[2][self.current_step])) + self.logger.info('Sold: ' + str(account_history[4][self.current_step])) + self.logger.info('Net worth: ' + str(self.exchange.net_worths[-1])) elif mode == 'human': if self.viewer is None: self.viewer = TradingChart(self.data_provider.data_frame) self.viewer.render(self.current_step, - self.net_worths, + self.exchange.net_worths, self.render_benchmarks, - self.trades) + self.exchange.trades) def close(self): if self.viewer is not None: diff --git a/lib/exchange/__init__.py b/lib/exchange/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/exchange/exchanges.py b/lib/exchange/exchanges.py new file mode 100644 index 0000000..3174758 --- /dev/null +++ b/lib/exchange/exchanges.py @@ -0,0 +1,148 @@ + + +import abc +import pandas as pd + +from enum import Enum + +from lib.data.providers.BaseDataProvider import BaseDataProvider + +class ExchangeMode(Enum): + TRAIN = 0 + TEST = 1 + PAPER = 2 + LIVE = 3 + +class BaseExchange(object, metaclass=abc.ABCMeta): + + def __init__(self, **kwargs): + self.product_id: str = kwargs.get('product_id', 'BTC-USD') + + self.base_precision: int = kwargs.get('base_precision', 2) + self.asset_precision: int = kwargs.get('asset_precision', 8) + self.min_cost_limit: float = kwargs.get('min_cost_limit', 1E-3) + self.min_amount_limit: float = kwargs.get('min_amount_limit', 1E-3) + + @abc.abstractmethod + def get_account_history(self): + raise NotImplementedError + + @abc.abstractmethod + def price(self): + raise NotImplementedError + + @abc.abstractmethod + def buy(self, amount): + raise NotImplementedError + + @abc.abstractmethod + def sell(self, amount): + raise NotImplementedError + + +class DummyExchange(BaseExchange): + + def __init__(self, data_provider: BaseDataProvider, initial_balance: int = 10000, + commission: float = 0.0025, **kwargs): + BaseExchange.__init__(self, **kwargs) + self.initial_balance = round(initial_balance, self.base_precision) + self.commission = commission + self.data_provider = data_provider + + self.current_ohlcv = self.data_provider.next_ohlcv() + self.balance = self.initial_balance + self.net_worths = [self.initial_balance] + self.asset_held = 0 + self.current_step = 0 + self.last_bought = 0 + self.last_sold = 0 + + self.account_history = pd.DataFrame([{ + 'balance': self.balance, + 'asset_bought': 0, + 'cost_of_asset': 0, + 'asset_sold': 0, + 'revenue_from_sold': 0, + }]) + self.trades = [] + + def get_account_history(self): + return self.account_history + + def price(self, ohlcv_key: str = 'Close'): + return float(self.current_ohlcv[ohlcv_key]) + + def buy(self, amount): + asset_bought = 0 + asset_sold = 0 + cost_of_asset = 0 + revenue_from_sold = 0 + + if self.balance >= self.min_cost_limit: + buy_price = round(self.price() * (1 + self.commission), self.base_precision) + cost_of_asset = round(self.balance * amount, self.base_precision) + asset_bought = round(cost_of_asset / buy_price, self.asset_precision) + + self.last_bought = self.current_step + self.asset_held += asset_bought + self.balance -= cost_of_asset + + self.trades.append({'step': self.current_step, 'amount': asset_bought, + 'total': cost_of_asset, 'type': 'buy'}) + current_net_worth = round(self.balance + self.asset_held * current_price, self.base_precision) + self.net_worths.append(current_net_worth) + + self.account_history = self.account_history.append({ + 'balance': self.balance, + 'asset_bought': asset_bought, + 'cost_of_asset': cost_of_asset, + 'asset_sold': asset_sold, + 'revenue_from_sold': revenue_from_sold, + }, ignore_index=True) + self.current_step += 1 + self.current_ohlcv = self.data_provider.next_ohlcv() + + def sell(self, amount): + + sell_price = round(self.price() * (1 - self.commission), self.base_precision) + asset_sold = round(self.asset_held * amount, self.asset_precision) + revenue_from_sold = round(asset_sold * sell_price, self.base_precision) + + self.last_sold = self.current_step + self.asset_held -= asset_sold + self.balance += revenue_from_sold + + self.trades.append({'step': self.current_step, 'amount': asset_sold, + 'total': revenue_from_sold, 'type': 'sell'}) + current_net_worth = round(self.balance + self.asset_held * current_price, self.base_precision) + self.net_worths.append(current_net_worth) + + self.account_history = self.account_history.append({ + 'balance': self.balance, + 'asset_bought': asset_bought, + 'cost_of_asset': cost_of_asset, + 'asset_sold': asset_sold, + 'revenue_from_sold': revenue_from_sold, + }, ignore_index=True) + self.current_step += 1 + self.current_ohlcv = self.data_provider.next_ohlcv() + + def reset(self): + self.data_provider.reset_ohlcv_index() + + + self.balance = self.initial_balance + self.net_worths = [self.initial_balance] + self.asset_held = 0 + self.current_step = 0 + self.last_bought = 0 + self.last_sold = 0 + + self.account_history = pd.DataFrame([{ + 'balance': self.balance, + 'asset_bought': 0, + 'cost_of_asset': 0, + 'asset_sold': 0, + 'revenue_from_sold': 0, + }]) + self.trades = [] From d4135a30a2199daa1fbd43d24123ba14c0fe827a Mon Sep 17 00:00:00 2001 From: mwbrulhardt Date: Sat, 27 Jul 2019 14:31:54 -0400 Subject: [PATCH 2/5] Added exchange code and integrated it with the TradeStrategy code. No tests have been run on this yet so there will definitely be some minor bugs or syntax errors. --- data/params.db | Bin 323584 -> 331776 bytes lib/env/TradingEnv.py | 127 +++++++++----------- lib/env/exchange/BaseExchange.py | 24 ++++ lib/env/exchange/LiveExchange.py | 17 +++ lib/env/exchange/SimulatedExchange.py | 89 ++++++++++++++ lib/env/exchange/__init__.py | 3 + lib/env/trade/BaseTradeStrategy.py | 5 +- lib/env/trade/SimulatedTradeStrategy.py | 6 +- lib/exchange/__init__.py | 0 lib/exchange/exchanges.py | 148 ------------------------ 10 files changed, 198 insertions(+), 221 deletions(-) create mode 100644 lib/env/exchange/BaseExchange.py create mode 100644 lib/env/exchange/LiveExchange.py create mode 100644 lib/env/exchange/SimulatedExchange.py create mode 100644 lib/env/exchange/__init__.py delete mode 100644 lib/exchange/__init__.py delete mode 100644 lib/exchange/exchanges.py diff --git a/data/params.db b/data/params.db index 76e553af7b39787527f513df8f8e39737e382e14..5f0448f8f244b855834a2b51e45be73e36047d43 100644 GIT binary patch delta 7079 zcmbtYdwdgB_RrjzykAYryA%Qi+G;6D+DV(bGC|t3=_7rBC{SqAHignQtxZddfT8RZ zBO*Ubg~3Hvly&8$R32NbAT6%AU>8BUDy}XltL(C&%R@zEh2NdIxq0Xxzx{lEd_KTE z-}60p=H{F;=bXFdV$7QDBc6>FzbX=mO1kt6@#>ouzxz-YrQHDmQAxMz%y>$>OS|Jh z@|iZR3PQzf1N^s$&FS8E;St3U&&tL#=~eZ$P0ddC5*P6v zL}PUnF$qj^H#W7nni{K?dPLvszee?t-vDc`bUX2*RtibyfsyJp^$-lB?cB8G9S|Qg0Dl z&LLurN=bnj0FvvS4a6X4+f7xeMF>jbp&oAY9_pkniUM%}B)Hry+~Q!0w= zp+uzrfG%n#@WxOe9soy;v$npx+2!;!x+ycl3c@iOVTIvX8p4Xgu~aPuVgV>BcegfF zx|%5yk{wt!BDoODUIPLOFfapcY7gHu1z~yN*mPu*hCk^P-}J!NdEJu^PTh3K?EQals`DkIGvJV= zX^YNxKf-X>(sYK-cOSxV=+bn$&Nm?mfjE9?+T1bImnght6|nJd$(1Nbvv!x>1nnJ+2o1XVLi*&`aR_plMp?n})DJjwCnJbehgL6@fTWXd1eD3c_%*(6p(y zVKw#KWUu^)=p*6;knfSdC>xePDQ}Zk%S+|6__Pl86kU4nj)Jk`$+1M z?v`GYt&>ic&6FLNR!Fx>FUXcj6|!jAercg}o%FPz76`wQtegZUmzLW(C!0Wq*41BaUN*FueCdX_d)QL_I*u{meO`= zGPR7hTeD2_sOBZDQT>T#Slg-r8k=ULHeP*6^QqRQ8L3Istk#ZHf1o+8eOUc()vKy2 z>eZ_I)DNoPCQhijRp-@C)dY2ddXK71)uH;6x=|Ia)~UCt?5ajpuX>J3qPnAgT4h#T zR8CbMRun7S7b?G3Y*w6Bj!^DaWGO3^-ze59jwngx3yKV7f%0QTqhhz>hH|4KS!q_D zP?Uj50OSTrD=ed7f1;^y9W1+`iCge8wYYcMD^#bHg3P9_u6i#jZ$+@Yfo`;R@as)z zUC6Jy(7J$MKZ~|43RQFPKL~P!K^qZN7zRCqpn@=HgGi+F3LQgHuwwvEqaMN$Le?Xs zFoZmXkb)56Lp8Yw*JSV{0vthL9Rdo2K&JqBK|v5Qj)3$85(|;I8c796T7?Q)k-HpzMVdTd zC6XOjejLe#SYCnT0xY*9$K~^mYeVZie!U#6?Qrb{tw_k_6)ng~`C(4-ASf>kYDSPf z40;TCBtPVlMug;r5H~{XA*4Y^dHIGhHw4uqH{}PNQip)NAg~Mp_8?G;fZQOk6nP;( z(hExvniq+BYY=RYgk1>DjYO-Fck+>Us*q$yQYDh|kmN*CE|MycU-GeE%8{H0Wv}4a z#R#xt!XhN+Vz~_YD2MmaLbT52*9*|v#;@liujGVzhe3}Z$c7;={4UHxj>!pm z;bF9I7QgRYq_u^#a}bgpLQ2uDS$J2tX$kU5PS7jG2(Sf#A|CJpkR3!EXydHV#)Zfa zIgx%SK&UMe%|~cm%wkd%!i8`?1|a>s1soE+?&EF{~o zJR8Z`SbhlYnT7V8g|t~%3x}W``>4hGc=d9TM%5$I4rzC4S8Gc(eVYGh6EyAWTN;b{ zkm`o2N1X;c(+w(;VPQkMHYUYn)*H>4sm9DyLkexqpiKso!DyLoNS}^0v?-G|rI=FF zjTRHba=P2Zh#BEFCQF(r6#GNcD>pe7^JV1TYjfr>O^dLkT=;aiO~1#RGO znhKAfk;vvEjo>@}eT^v=OImt{A(74gnGJl70d59uvVNg4(PbjfzvXUwq z!`iSGMLsk9Uc;AN6aL11nMXJ#)v?)7ognQ<0FkM{oIB6Fetcm`_w{p*v1}G3#Y#Kk z0nQBT@`3sL_oQ*g1ws?ekiL=D23A`WtnJdW>qH%BH-iT&WzT=v9=;{ql8ky=*{MDI=vXO4B80 zC3O-?{G!-QeMmXUn`9?B8oU9riO&fSp%LxscAl9xo^61SpD68!1ECtn^n0HwGVcC? zJDW}9##OTQm=daaOn=)4w~s{qg`4#ZWsha+AR~_dt%Eg?k$?KqrFS)X-1X%|;rJSM z8D<2FA`^9c!+{^SG-5_AW(2Dwqt!ls`1XZp%vg#U!LrFvCysu*Z_Y%40e!v%G6bJT z)=_3~mStbVnBQ<`D=3Gat--9wy2_k9y5ftCBQ^zCF3gH7v&^}Xg>5?vUI?(NF)Om{ zGN)38WFP*y9!6Q=m}<5PvV;ggWth2os{fsBr(?+pb)s~l>D=*Cstx`IlYwQQm+5V?!7p0DbB zQMa~?=-KeuqzeK4BCN+n-r8zjovHt9X+U3w^|_SM0;e(6s z0vLb1KKi?-DQ;FZkqeazu##^J82_RPLrW@3uyQ_D@_hp1XYS5lyhtG^CseVILggf3 z+febg#>5cA)=L*S=V8ho$3B87VT}ay{h{hLWf_YClzEsEY%G>~m>=G??T_kd<=)&( z*~haFLyC|bp+18d*juK*^iqY89Pku#Aw@VM-*_!i`W>+!f2>BOm zT^RY4cP8Dt*(YR1Yz!_Jr&qhiY%Q`SA*xoRbcV2!_O)nI- zfuDjx$Pj*zP`|?*yK~T~xX>b;24|~7$;Yh7hKLy~-SvBG z!@U4253?d0B<57bt;_oE@A>S4r?*3vaQa}w#KicXdS&+EA>p;*7dRI(g#1Gt6f@kn zth>AJvQR3vR0!YJDF=aw$)6?F;ydC^;^}a>uOjb~TS*fb1T{cP>?G`>D_vb%MwfNQz{d))!H=qpU$!A< zT;Hicgm*?`B|p0|{;J(O|9SE-R_d^lA7&YU#`6kX{3C+q(|6?~_*?Z`AApScQm=YSEnPUZK$F4Z|6Q;;9 zB{WSl{huU%o36Aj65GcW!n-NXD z|BKE|cZEQK3<+ighj2!&NUD7Dr_?|qb&4?~IF&O|LpMYzJGqyS;za7CFex~wGm?LO z{M#Z^wXlu8u9JieAxk1BcV@6jd+%BT*O5m&z!f`)m`Qb=08=B!dFIRs+yJvHg*OwOHALmHedzwCuZ;`=C%Ka$!O9K&9}F54*)+zU^D}P}{+*K(%Vdl(1V3 zYs~v+ckMf65h&x?$1o-AmV^1Obf-M$Yi{5?VTTWD!W4Y}2JhIIThh{X?wyAMHLnp; kgrp5!j4=Jr`KgZ!Cj>H*bz@5CriAIYy}o>a8zE5s7uS0w(*OVf delta 935 zcmZ{ie`u9e7{}l9oag*_-}BCMUuPK1>vlIPhuhpLltnS-TnUm>$~2_uy3SSJyItmN zg^Udp2U(`LJgp4W749z<$E&nqY0dT(lCwR15Gm+5Qz zVmdVWsV<2J?VJ!o7~p}%=C*awmd1^wFMxR-k_gOgZb`H?CR(B|Y_>APQ#eR7^bKDe zAiwJhG0PgEyQrZ2SN;?KhJVI={sHgd+j%>W@zs1OU&!y{_i)MnWLMZ2JMGIsvYyPN zOX(t7tbe1ql)@Bz2VcNRNJ1~{fHsK2DyV`5WCX(I;~~-xe7D(}#Oj$ILOwugH1=Ym zwV}DGk&Kv%VbWnP93l6kzQpz!`zR?j9}bg;{atBN34Ylj@+7f?Q|y1v=nnfNDDcBR z{rPP?gwvLJBmo&SwgNY&z0e@_r=LcdM+B#n4GcP748zZ z%$@5hbz5Cg<7!lWr25rcYMW|R8`K)LSUsrnRgN?5+;Dzy&N-hrNyj)loL8Kf^NjPj z^N`D8)eZn5F-?3kp-`X$R8||m;YP;Mnlo{KV)AFhq5L0rS2+JCIs#82K zdc}m?B0O0theVy|7Nhc6ep?cGP&_KOh!e6&FgaW95yk9BUdE5JHT+e6mF;JzxZv-x zO1_GZv38bXxA>c^lvnYyYy*3ZUErHogx|{tSS@~zml(wXyv{uIaRI)IkKtBai#u`c zCR|3prq|G;$LJrJPeRtr*as)6rX8T^O4lAd+0zFR7kiVc+_dy9{^ywfR%Zwew`cU@w@8<{fviP6WD_w<<3s4ekh?8^XrD3up z>p?V|e(KJwI6E6O&!tI)sX0o{dX`0N|08);lVfB?4#I4JqNav;yn%F@CytP!+?faw zfWo`-llDLxvWWP7DXcG0!KR@5E&Y}5PkZJ2H7(v=W4sD1GNTi?kml1KBPOvbFlagT JdoE+l{Tr$u2aEs! diff --git a/lib/env/TradingEnv.py b/lib/env/TradingEnv.py index 3765894..4f0b798 100644 --- a/lib/env/TradingEnv.py +++ b/lib/env/TradingEnv.py @@ -6,12 +6,12 @@ from enum import Enum from typing import List, Dict +from lib.env.exchange import BaseExchange, DummyExchange, ExchangeMode from lib.env.render import TradingChart from lib.env.reward import BaseRewardStrategy, IncrementalProfit, WeightedUnrealizedProfit from lib.env.trade import BaseTradeStrategy, SimulatedTradeStrategy from lib.data.providers import BaseDataProvider from lib.data.features.transform import max_min_normalize, mean_normalize, log_and_difference, difference -from lib.exchange.exchanges import BaseExchange, DummyExchange, ExchangeMode from lib.util.logger import init_logger @@ -20,20 +20,27 @@ class TradingEnvAction(Enum): SELL = 1 HOLD = 2 +class TradingMode(Enum): + TRAIN = 0 + TEST = 1 + PAPER = 2 + LIVE = 3 class TradingEnv(gym.Env): - '''A reinforcement trading environment made for use with gym-enabled algorithms''' + """A reinforcement trading environment made for use with gym-enabled algorithms""" metadata = {'render.modes': ['human', 'system', 'none']} viewer = None def __init__(self, data_provider: BaseDataProvider, - exchange: BaseExchange = DummyExchange, + exchange: BaseExchange = SimulatedExchange reward_strategy: BaseRewardStrategy = IncrementalProfit, trade_strategy: BaseTradeStrategy = SimulatedTradeStrategy, initial_balance: int = 10000, commissionPercent: float = 0.25, maxSlippagePercent: float = 2.0, + trading_mode: TradingMode = TradingMode.PAPER, + exchange_args: Dict **kwargs): super(TradingEnv, self).__init__() @@ -44,13 +51,29 @@ def __init__(self, self.min_cost_limit: float = kwargs.get('min_cost_limit', 1E-3) self.min_amount_limit: float = kwargs.get('min_amount_limit', 1E-3) - self.initial_balance = round(initial_balance, self.base_precision) self.commissionPercent = commissionPercent self.maxSlippagePercent = maxSlippagePercent - - self.data_provider = data_provider - self.exchange = DummyExchange(data_provider) if exchange == DummyExchange else exchange - + self.trading_mode = trading_mode + + if self.trading_mode == TradingMode.TRAIN or self.trading_mode == TradingMode.TEST: + assert type(data_provider) == 'StaticDataProvider' + assert type(exchange) == 'SimulatedExchange' + assert type(trade_strategy) == 'SimulatedTradeStrategy' + self.exchange = exchange(self, initial_balance, **exchange_args) + + elif self.trading_mode == TradingMode.PAPER: + assert type(data_provider) == 'ExchangeDataProvider' + assert type(exchange) == 'SimulatedExchange' + assert type(trade_strategy) == 'SimulatedTradeStrategy' + self.exchange = exchange(self, **exchange_args) + + elif self.trading_mode == TradingMode.LIVE: + assert type(data_provider) == 'ExchangeDataProvider' + assert type(exchange) == 'LiveExchange' + assert type(trade_strategy) == 'LiveTradeStrategy' + self.exchange = exchange(self, **exchange_args) + + self.data_provider = data_provider() self.reward_strategy = reward_strategy() self.trade_strategy = trade_strategy(commissionPercent=self.commissionPercent, maxSlippagePercent=self.maxSlippagePercent, @@ -58,7 +81,6 @@ def __init__(self, asset_precision=self.asset_precision, min_cost_limit=self.min_cost_limit, min_amount_limit=self.min_amount_limit) - self.render_benchmarks: List[Dict] = kwargs.get('render_benchmarks', []) self.normalize_obs: bool = kwargs.get('normalize_obs', True) self.stationarize_obs: bool = kwargs.get('stationarize_obs', True) @@ -68,7 +90,7 @@ def __init__(self, self.n_discrete_actions: int = kwargs.get('n_discrete_actions', 24) self.action_space = spaces.Discrete(self.n_discrete_actions) - self.n_features = 6 + len(self.data_provider.columns) + self.n_features = 5 + len(self.data_provider.columns) self.obs_shape = (1, self.n_features) self.observation_space = spaces.Box(low=0, high=1, shape=self.obs_shape, dtype=np.float16) @@ -84,57 +106,25 @@ def _get_trade(self, action: int): action_type: TradingEnvAction = TradingEnvAction(action % n_action_types) action_amount = float(1 / (action % n_amount_bins + 1)) + commission = self.commissionPercent / 100 + max_slippage = self.maxSlippagePercent / 100 + amount_asset_to_buy = 0 amount_asset_to_sell = 0 if action_type == TradingEnvAction.BUY and self.balance >= self.min_cost_limit: - price_adjustment = (1 + (self.commissionPercent / 100)) * (1 + (self.maxSlippagePercent / 100)) - buy_price = round(self._current_price() * price_adjustment, self.base_precision) - amount_asset_to_buy = round(self.balance * action_amount / buy_price, self.asset_precision) + price_adjustment = (1 + slippage) * (1 + max_slippage) + buy_price = self._current_price() * price_adjustment + buy_price = round(buy_price, self.base_precision) + amount_asset_to_buy = self.balance * action_amount / buy_price + amount_asset_to_buy = round(amount_asset_to_buy, self.asset_precision) + elif action_type == TradingEnvAction.SELL and self.asset_held >= self.min_amount_limit: - amount_asset_to_sell = round(self.asset_held * action_amount, self.asset_precision) + amount_asset_to_sell = self.asset_held * action_amount + amount_asset_to_sell = round(amount_asset_to_sell, self.asset_precision) return amount_asset_to_buy, amount_asset_to_sell - def _take_action(self, action: int): - amount_asset_to_buy, amount_asset_to_sell = self._get_trade(action) - - asset_bought, asset_sold, purchase_cost, sale_revenue = self.trade_strategy.trade(buy_amount=amount_asset_to_buy, - sell_amount=amount_asset_to_sell, - balance=self.balance, - asset_held=self.asset_held, - current_price=self._current_price) - - if asset_bought: - self.asset_held += asset_bought - self.balance -= purchase_cost - - self.trades.append({'step': self.current_step, - 'amount': asset_bought, - 'total': purchase_cost, - 'type': 'buy'}) - elif asset_sold: - self.asset_held -= asset_sold - self.balance += sale_revenue - - self.reward_strategy.reset_reward() - - self.trades.append({'step': self.current_step, - 'amount': asset_sold, - 'total': sale_revenue, - 'type': 'sell'}) - - current_net_worth = round(self.balance + self.asset_held * self._current_price(), self.base_precision) - self.net_worths.append(current_net_worth) - self.account_history = self.account_history.append({ - 'balance': self.balance, - 'asset_held': self.asset_held, - 'asset_bought': asset_bought, - 'purchase_cost': purchase_cost, - 'asset_sold': asset_sold, - 'sale_revenue': sale_revenue, - }, ignore_index=True) - def _done(self): lost_90_percent_net_worth = float(self.exchange.net_worths[-1]) < (self.exchange.initial_balance / 10) has_next_frame = self.data_provider.has_next_ohlcv() @@ -197,29 +187,26 @@ def _next_observation(self): def reset(self): self.data_provider.reset_ohlcv_index() - self.balance = self.initial_balance - self.net_worths = [self.initial_balance] + if self.trading_mode == TradingMode.TRAIN or self.trading_mode == TradingMode.TEST: + self.exchange.reset() + self.timestamps = [] - self.asset_held = 0 self.current_step = 0 self.reward_strategy.reset_reward() - self.account_history = pd.DataFrame([{ - 'balance': self.balance, - 'asset_held': self.asset_held, - 'asset_bought': 0, - 'purchase_cost': 0, - 'asset_sold': 0, - 'sale_revenue': 0, - }]) - self.trades = [] self.rewards = [0] return self._next_observation() def step(self, action): - self._take_action(action) + amount_asset_to_buy, amount_asset_to_sell = self._get_trade(action) + + if amount_asset_to_buy: + self.exchange.buy(amount_asset_to_buy) + elif amount_asset_to_sell: + self.exchange.sell(amount_asset_to_sell) + self.reward_strategy.reset_reward() self.current_step += 1 @@ -227,15 +214,15 @@ def step(self, action): reward = self._reward() done = self._done() - return obs, reward, done, {'net_worths': self.net_worths, 'timestamps': self.timestamps} + return obs, reward, done, {'net_worths': self.exchange.net_worths, 'timestamps': self.timestamps} def render(self, mode='human'): if mode == 'system': self.logger.info('Price: ' + str(self._current_price())) - self.logger.info('Bought: ' + str(self.account_history['asset_bought'][self.current_step])) - self.logger.info('Sold: ' + str(self.account_history['asset_sold'][self.current_step])) - self.logger.info('Net worth: ' + str(self.net_worths[-1])) + self.logger.info('Bought: ' + str(self.exchange.account_history['asset_bought'][self.current_step])) + self.logger.info('Sold: ' + str(self.exchange.account_history['asset_sold'][self.current_step])) + self.logger.info('Net worth: ' + str(self.exchange.net_worths[-1])) elif mode == 'human': if self.viewer is None: diff --git a/lib/env/exchange/BaseExchange.py b/lib/env/exchange/BaseExchange.py new file mode 100644 index 0000000..a58cb32 --- /dev/null +++ b/lib/env/exchange/BaseExchange.py @@ -0,0 +1,24 @@ + +import abc +import pandas as pd + +from enum import Enum +from lib.env import TradingEnv + +class BaseExchange(object, metaclass=abc.ABCMeta): + + @abc.abstractmethod + def __init__(self, env: TradingEnv, **kwargs): + pass + + @abc.abstractmethod + def get_account_history(self): + raise NotImplementedError + + @abc.abstractmethod + def buy(self, amount: float): + raise NotImplementedError + + @abc.abstractmethod + def sell(self, amount: float): + raise NotImplementedError diff --git a/lib/env/exchange/LiveExchange.py b/lib/env/exchange/LiveExchange.py new file mode 100644 index 0000000..c6f8447 --- /dev/null +++ b/lib/env/exchange/LiveExchange.py @@ -0,0 +1,17 @@ + + + +class LiveExchange(BaseExchange): + + def __init__(self, env: TradingEnv, **kwargs): + self.env = env + self.credentials = kwargs.get('credentials') + + def get_account_history(self): + pass + + def buy(self, amount: float): + pass + + def sell(self, amount: float): + pass diff --git a/lib/env/exchange/SimulatedExchange.py b/lib/env/exchange/SimulatedExchange.py new file mode 100644 index 0000000..f519246 --- /dev/null +++ b/lib/env/exchange/SimulatedExchange.py @@ -0,0 +1,89 @@ + +from lib.env import TradingEnv + + +class SimulatedExchange(BaseExchange): + + def __init__(self, env: TradingEnv, initial_balance: int = 10000, **kwargs): + self.env = env + self.initial_balance = round(initial_balance, self.base_precision) + self.balance = self.initial_balance + self.net_worths = [self.initial_balance] + self.asset_held = 0 + self.current_step = 0 + self.account_history = pd.DataFrame([{ + 'balance': self.balance, + 'asset_bought': 0, + 'cost_of_asset': 0, + 'asset_sold': 0, + 'revenue_from_sold': 0, + }]) + self.trades = [] + + def get_account_history(self): + return self.account_history + + def _add_trade(self, amount: float, total: float, side: str): + self.trades.append({'step': self.env.current_step, + 'amount': asset_bought, + 'total': purchase_cost, + 'type': side}) + current_net_worth = self.balance + self.asset_held * self.env._current_price() + current_net_worth = round(current_net_worth, self.base_precision) + self.net_worths.append(current_net_worth) + + def buy(self, amount: float): + trade = self.env.trade_strategy.trade(buy_amount=amount, + sell_amount=0, + balance=self.balance, + asset_held=self.asset_held, + current_price=self.env._current_price) + asset_bought, asset_sold, purchase_cost, sale_revenue = trade + self.asset_held += asset_bought + self.balance -= purchase_cost + self._add_trade(amount=asset_bought, total=purchase_cost, side='buy') + self.account_history = self.account_history.append({ + 'balance': self.balance, + 'asset_held': self.asset_held, + 'asset_bought': asset_bought, + 'purchase_cost': purchase_cost, + 'asset_sold': asset_sold, + 'sale_revenue': sale_revenue, + }, ignore_index=True) + + self.current_step += 1 + self.current_ohlcv = self.data_provider.next_ohlcv() + + def sell(self, amount: float): + trade = self.env.trade_strategy.trade(buy_amount=0, + sell_amount=amount, + balance=self.balance, + asset_held=self.asset_held, + current_price=self.env._current_price) + asset_bought, asset_sold, purchase_cost, sale_revenue = trade + self.asset_held -= asset_sold + self.balance += sale_revenue + self._add_trade(amount=asset_sold, total=sale_revenue, side='sell') + self.account_history = self.account_history.append({ + 'balance': self.balance, + 'asset_bought': asset_bought, + 'cost_of_asset': cost_of_asset, + 'asset_sold': asset_sold, + 'revenue_from_sold': revenue_from_sold, + }, ignore_index=True) + + def reset(self): + self.balance = self.initial_balance + self.net_worths = [self.initial_balance] + self.asset_held = 0 + self.current_step = 0 + self.last_bought = 0 + self.last_sold = 0 + self.account_history = pd.DataFrame([{ + 'balance': self.balance, + 'asset_bought': 0, + 'cost_of_asset': 0, + 'asset_sold': 0, + 'revenue_from_sold': 0, + }]) + self.trades = [] diff --git a/lib/env/exchange/__init__.py b/lib/env/exchange/__init__.py new file mode 100644 index 0000000..3c73a04 --- /dev/null +++ b/lib/env/exchange/__init__.py @@ -0,0 +1,3 @@ +from lib.env.exchange.BaseExchange import BaseExchange +from lib.env.exchange.SimulatedExchange import SimulatedExchange +from lib.env.exchange.LiveExchange import LiveExchange diff --git a/lib/env/trade/BaseTradeStrategy.py b/lib/env/trade/BaseTradeStrategy.py index 71f6696..733ab86 100644 --- a/lib/env/trade/BaseTradeStrategy.py +++ b/lib/env/trade/BaseTradeStrategy.py @@ -3,6 +3,7 @@ class BaseTradeStrategy(object, metaclass=ABCMeta): + @abstractmethod def __init__(self, commissionPercent: float, @@ -15,8 +16,8 @@ def __init__(self, @abstractmethod def trade(self, - action: int, - n_discrete_actions: int, + buy_amount: float, + sell_amount: float, balance: float, asset_held: float, current_price: Callable[[str], float]) -> Tuple[float, float, float, float]: diff --git a/lib/env/trade/SimulatedTradeStrategy.py b/lib/env/trade/SimulatedTradeStrategy.py index acbd985..2baef46 100644 --- a/lib/env/trade/SimulatedTradeStrategy.py +++ b/lib/env/trade/SimulatedTradeStrategy.py @@ -6,6 +6,7 @@ class SimulatedTradeStrategy(BaseTradeStrategy): + def __init__(self, commissionPercent: float, maxSlippagePercent: float, @@ -26,9 +27,11 @@ def trade(self, balance: float, asset_held: float, current_price: Callable[[str], float]) -> Tuple[float, float, float, float]: + current_price = current_price('Close') commission = self.commissionPercent / 100 - slippage = np.random.uniform(0, self.maxSlippagePercent) / 100 + max_slippage = self.maxSlippagePercent / 100 + slippage = np.random.uniform(0, max_slippage) asset_bought, asset_sold, purchase_cost, sale_revenue = buy_amount, sell_amount, 0, 0 @@ -36,6 +39,7 @@ def trade(self, price_adjustment = (1 + commission) * (1 + slippage) buy_price = round(current_price * price_adjustment, self.base_precision) purchase_cost = round(buy_price * buy_amount, self.base_precision) + elif sell_amount > 0 and asset_held >= self.min_amount_limit: price_adjustment = (1 - commission) * (1 - slippage) sell_price = round(current_price * price_adjustment, self.base_precision) diff --git a/lib/exchange/__init__.py b/lib/exchange/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/exchange/exchanges.py b/lib/exchange/exchanges.py deleted file mode 100644 index 3174758..0000000 --- a/lib/exchange/exchanges.py +++ /dev/null @@ -1,148 +0,0 @@ - - -import abc -import pandas as pd - -from enum import Enum - -from lib.data.providers.BaseDataProvider import BaseDataProvider - -class ExchangeMode(Enum): - TRAIN = 0 - TEST = 1 - PAPER = 2 - LIVE = 3 - -class BaseExchange(object, metaclass=abc.ABCMeta): - - def __init__(self, **kwargs): - self.product_id: str = kwargs.get('product_id', 'BTC-USD') - - self.base_precision: int = kwargs.get('base_precision', 2) - self.asset_precision: int = kwargs.get('asset_precision', 8) - self.min_cost_limit: float = kwargs.get('min_cost_limit', 1E-3) - self.min_amount_limit: float = kwargs.get('min_amount_limit', 1E-3) - - @abc.abstractmethod - def get_account_history(self): - raise NotImplementedError - - @abc.abstractmethod - def price(self): - raise NotImplementedError - - @abc.abstractmethod - def buy(self, amount): - raise NotImplementedError - - @abc.abstractmethod - def sell(self, amount): - raise NotImplementedError - - -class DummyExchange(BaseExchange): - - def __init__(self, data_provider: BaseDataProvider, initial_balance: int = 10000, - commission: float = 0.0025, **kwargs): - BaseExchange.__init__(self, **kwargs) - self.initial_balance = round(initial_balance, self.base_precision) - self.commission = commission - self.data_provider = data_provider - - self.current_ohlcv = self.data_provider.next_ohlcv() - self.balance = self.initial_balance - self.net_worths = [self.initial_balance] - self.asset_held = 0 - self.current_step = 0 - self.last_bought = 0 - self.last_sold = 0 - - self.account_history = pd.DataFrame([{ - 'balance': self.balance, - 'asset_bought': 0, - 'cost_of_asset': 0, - 'asset_sold': 0, - 'revenue_from_sold': 0, - }]) - self.trades = [] - - def get_account_history(self): - return self.account_history - - def price(self, ohlcv_key: str = 'Close'): - return float(self.current_ohlcv[ohlcv_key]) - - def buy(self, amount): - asset_bought = 0 - asset_sold = 0 - cost_of_asset = 0 - revenue_from_sold = 0 - - if self.balance >= self.min_cost_limit: - buy_price = round(self.price() * (1 + self.commission), self.base_precision) - cost_of_asset = round(self.balance * amount, self.base_precision) - asset_bought = round(cost_of_asset / buy_price, self.asset_precision) - - self.last_bought = self.current_step - self.asset_held += asset_bought - self.balance -= cost_of_asset - - self.trades.append({'step': self.current_step, 'amount': asset_bought, - 'total': cost_of_asset, 'type': 'buy'}) - current_net_worth = round(self.balance + self.asset_held * current_price, self.base_precision) - self.net_worths.append(current_net_worth) - - self.account_history = self.account_history.append({ - 'balance': self.balance, - 'asset_bought': asset_bought, - 'cost_of_asset': cost_of_asset, - 'asset_sold': asset_sold, - 'revenue_from_sold': revenue_from_sold, - }, ignore_index=True) - self.current_step += 1 - self.current_ohlcv = self.data_provider.next_ohlcv() - - def sell(self, amount): - - sell_price = round(self.price() * (1 - self.commission), self.base_precision) - asset_sold = round(self.asset_held * amount, self.asset_precision) - revenue_from_sold = round(asset_sold * sell_price, self.base_precision) - - self.last_sold = self.current_step - self.asset_held -= asset_sold - self.balance += revenue_from_sold - - self.trades.append({'step': self.current_step, 'amount': asset_sold, - 'total': revenue_from_sold, 'type': 'sell'}) - current_net_worth = round(self.balance + self.asset_held * current_price, self.base_precision) - self.net_worths.append(current_net_worth) - - self.account_history = self.account_history.append({ - 'balance': self.balance, - 'asset_bought': asset_bought, - 'cost_of_asset': cost_of_asset, - 'asset_sold': asset_sold, - 'revenue_from_sold': revenue_from_sold, - }, ignore_index=True) - self.current_step += 1 - self.current_ohlcv = self.data_provider.next_ohlcv() - - def reset(self): - self.data_provider.reset_ohlcv_index() - - - self.balance = self.initial_balance - self.net_worths = [self.initial_balance] - self.asset_held = 0 - self.current_step = 0 - self.last_bought = 0 - self.last_sold = 0 - - self.account_history = pd.DataFrame([{ - 'balance': self.balance, - 'asset_bought': 0, - 'cost_of_asset': 0, - 'asset_sold': 0, - 'revenue_from_sold': 0, - }]) - self.trades = [] From a375f8bf2c8bc847fef20cc3b8313157a6466d7a Mon Sep 17 00:00:00 2001 From: mwbrulhardt Date: Sat, 27 Jul 2019 14:46:08 -0400 Subject: [PATCH 3/5] Added some corrections for syntax errors. --- cli.py | 16 +++++++++++++--- lib/env/TradingEnv.py | 6 +++--- lib/env/exchange/LiveExchange.py | 3 ++- lib/env/exchange/SimulatedExchange.py | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/cli.py b/cli.py index 449d2c7..8438412 100644 --- a/cli.py +++ b/cli.py @@ -6,6 +6,7 @@ from lib.util.logger import init_logger from lib.cli.functions import download_data_async from lib.env.reward import BaseRewardStrategy, IncrementalProfit, WeightedUnrealizedProfit +from lib.env import TradingMode np.warnings.filterwarnings('ignore') @@ -19,7 +20,10 @@ def run_optimize(args, logger): from lib.RLTrader import RLTrader - trader = RLTrader(**vars(args), logger=logger, reward_strategy=reward_strategy) + trader = RLTrader(**vars(args), + logger=logger, + reward_strategy=reward_strategy, + trading_mode=TradingMode.TRAIN) trader.optimize(n_trials=args.trials) @@ -41,9 +45,11 @@ def run_optimize(args, logger): from lib.RLTrader import RLTrader - trader = RLTrader(**vars(args), logger=logger, reward_strategy=reward_strategy) - if args.command == 'train': + trader = RLTrader(**vars(args), + logger=logger, + reward_strategy=reward_strategy, + trading_mode=TradingMode.TRAIN) trader.train(n_epochs=args.epochs, save_every=args.save_every, test_trained_model=args.test_trained, @@ -51,6 +57,10 @@ def run_optimize(args, logger): render_report=args.render_report, save_report=args.save_report) elif args.command == 'test': + trader = RLTrader(**vars(args), + logger=logger, + reward_strategy=reward_strategy, + trading_mode=TradingMode.TEST) trader.test(model_epoch=args.model_epoch, render_env=args.render_env, render_report=args.render_report, diff --git a/lib/env/TradingEnv.py b/lib/env/TradingEnv.py index 4f0b798..036edef 100644 --- a/lib/env/TradingEnv.py +++ b/lib/env/TradingEnv.py @@ -6,7 +6,7 @@ from enum import Enum from typing import List, Dict -from lib.env.exchange import BaseExchange, DummyExchange, ExchangeMode +from lib.env.exchange import BaseExchange, SimulatedExchange from lib.env.render import TradingChart from lib.env.reward import BaseRewardStrategy, IncrementalProfit, WeightedUnrealizedProfit from lib.env.trade import BaseTradeStrategy, SimulatedTradeStrategy @@ -33,14 +33,14 @@ class TradingEnv(gym.Env): def __init__(self, data_provider: BaseDataProvider, - exchange: BaseExchange = SimulatedExchange + exchange: BaseExchange = SimulatedExchange, reward_strategy: BaseRewardStrategy = IncrementalProfit, trade_strategy: BaseTradeStrategy = SimulatedTradeStrategy, initial_balance: int = 10000, commissionPercent: float = 0.25, maxSlippagePercent: float = 2.0, trading_mode: TradingMode = TradingMode.PAPER, - exchange_args: Dict + exchange_args: Dict = {} **kwargs): super(TradingEnv, self).__init__() diff --git a/lib/env/exchange/LiveExchange.py b/lib/env/exchange/LiveExchange.py index c6f8447..d70313a 100644 --- a/lib/env/exchange/LiveExchange.py +++ b/lib/env/exchange/LiveExchange.py @@ -1,5 +1,6 @@ - +from lib.env import TradingEnv +from lib.env.exchange import BaseExchange class LiveExchange(BaseExchange): diff --git a/lib/env/exchange/SimulatedExchange.py b/lib/env/exchange/SimulatedExchange.py index f519246..687fe71 100644 --- a/lib/env/exchange/SimulatedExchange.py +++ b/lib/env/exchange/SimulatedExchange.py @@ -1,6 +1,6 @@ from lib.env import TradingEnv - +from lib.env.exchange import BaseExchange class SimulatedExchange(BaseExchange): From 20a9bcfa76e59811a20f10a36e591cc9d6627afe Mon Sep 17 00:00:00 2001 From: mwbrulhardt Date: Mon, 29 Jul 2019 16:57:42 -0400 Subject: [PATCH 4/5] Minor bugs fixes and updated .gitignore. --- .gitignore | 1 + lib/RLTrader.py | 6 +++--- lib/cli/RLTraderCLI.py | 2 ++ lib/env/TradingEnv.py | 29 ++++++++++----------------- lib/env/__init__.py | 2 +- lib/env/exchange/SimulatedExchange.py | 4 +++- 6 files changed, 21 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 77158db..4b6910c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea/ .vscode .ipynb_checkpoints .pytest_cache diff --git a/lib/RLTrader.py b/lib/RLTrader.py index 4a8ccc0..0b772c2 100644 --- a/lib/RLTrader.py +++ b/lib/RLTrader.py @@ -13,7 +13,7 @@ from stable_baselines.common import set_global_seeds from stable_baselines import PPO2 -from lib.env.TradingEnv import TradingEnv +from lib.env.TradingEnv import TradingEnv, TradingMode from lib.env.reward import BaseRewardStrategy, IncrementalProfit, WeightedUnrealizedProfit from lib.data.providers.dates import ProviderDateFormat from lib.data.providers import BaseDataProvider, StaticDataProvider, ExchangeDataProvider @@ -134,8 +134,8 @@ def optimize_params(self, trial, n_prune_evals_per_trial: int = 2, n_tests_per_e del test_provider - train_env = DummyVecEnv([lambda: TradingEnv(train_provider)]) - validation_env = DummyVecEnv([lambda: TradingEnv(validation_provider)]) + train_env = DummyVecEnv([lambda: TradingEnv(train_provider, trading_mode=TradingMode.TRAIN)]) + validation_env = DummyVecEnv([lambda: TradingEnv(validation_provider, trading_mode=TradingMode.TRAIN)]) model_params = self.optimize_agent_params(trial) model = self.Model(self.Policy, diff --git a/lib/cli/RLTraderCLI.py b/lib/cli/RLTraderCLI.py index d9966a9..4883b29 100644 --- a/lib/cli/RLTraderCLI.py +++ b/lib/cli/RLTraderCLI.py @@ -70,6 +70,8 @@ def __init__(self): test_parser.add_argument('--save-report', dest="save_report", action="store_true", help='Save the performance report as .html') + live_parser = subparsers.add_parser('live', description='Live model') + subparsers.add_parser('update-static-data', description='Update static data') self.parser.set_defaults(**defaults) diff --git a/lib/env/TradingEnv.py b/lib/env/TradingEnv.py index 036edef..954791f 100644 --- a/lib/env/TradingEnv.py +++ b/lib/env/TradingEnv.py @@ -40,7 +40,7 @@ def __init__(self, commissionPercent: float = 0.25, maxSlippagePercent: float = 2.0, trading_mode: TradingMode = TradingMode.PAPER, - exchange_args: Dict = {} + exchange_args: Dict = {}, **kwargs): super(TradingEnv, self).__init__() @@ -55,32 +55,25 @@ def __init__(self, self.maxSlippagePercent = maxSlippagePercent self.trading_mode = trading_mode + self.data_provider = data_provider + self.reward_strategy = reward_strategy() + self.trade_strategy = trade_strategy(commissionPercent=self.commissionPercent, + maxSlippagePercent=self.maxSlippagePercent, + base_precision=self.base_precision, + asset_precision=self.asset_precision, + min_cost_limit=self.min_cost_limit, + min_amount_limit=self.min_amount_limit) + if self.trading_mode == TradingMode.TRAIN or self.trading_mode == TradingMode.TEST: - assert type(data_provider) == 'StaticDataProvider' - assert type(exchange) == 'SimulatedExchange' - assert type(trade_strategy) == 'SimulatedTradeStrategy' self.exchange = exchange(self, initial_balance, **exchange_args) elif self.trading_mode == TradingMode.PAPER: - assert type(data_provider) == 'ExchangeDataProvider' - assert type(exchange) == 'SimulatedExchange' - assert type(trade_strategy) == 'SimulatedTradeStrategy' self.exchange = exchange(self, **exchange_args) elif self.trading_mode == TradingMode.LIVE: - assert type(data_provider) == 'ExchangeDataProvider' - assert type(exchange) == 'LiveExchange' - assert type(trade_strategy) == 'LiveTradeStrategy' self.exchange = exchange(self, **exchange_args) - self.data_provider = data_provider() - self.reward_strategy = reward_strategy() - self.trade_strategy = trade_strategy(commissionPercent=self.commissionPercent, - maxSlippagePercent=self.maxSlippagePercent, - base_precision=self.base_precision, - asset_precision=self.asset_precision, - min_cost_limit=self.min_cost_limit, - min_amount_limit=self.min_amount_limit) + self.render_benchmarks: List[Dict] = kwargs.get('render_benchmarks', []) self.normalize_obs: bool = kwargs.get('normalize_obs', True) self.stationarize_obs: bool = kwargs.get('stationarize_obs', True) diff --git a/lib/env/__init__.py b/lib/env/__init__.py index 4df6ba5..6cf32fb 100644 --- a/lib/env/__init__.py +++ b/lib/env/__init__.py @@ -1,2 +1,2 @@ -from lib.env.TradingEnv import TradingEnv +from lib.env.TradingEnv import TradingEnv, TradingMode from lib.env.render.TradingChart import TradingChart diff --git a/lib/env/exchange/SimulatedExchange.py b/lib/env/exchange/SimulatedExchange.py index 687fe71..247bd8b 100644 --- a/lib/env/exchange/SimulatedExchange.py +++ b/lib/env/exchange/SimulatedExchange.py @@ -1,4 +1,6 @@ +import pandas as pd + from lib.env import TradingEnv from lib.env.exchange import BaseExchange @@ -6,7 +8,7 @@ class SimulatedExchange(BaseExchange): def __init__(self, env: TradingEnv, initial_balance: int = 10000, **kwargs): self.env = env - self.initial_balance = round(initial_balance, self.base_precision) + self.initial_balance = round(initial_balance, env.base_precision) self.balance = self.initial_balance self.net_worths = [self.initial_balance] self.asset_held = 0 From 3bc6dd5ccfa17188095f83dbc855271baff53716 Mon Sep 17 00:00:00 2001 From: mwbrulhardt Date: Mon, 29 Jul 2019 20:24:30 -0400 Subject: [PATCH 5/5] Finished integrating the exchange api and is currently working. --- data/params.db | Bin 331776 -> 372736 bytes lib/RLTrader.py | 5 ++- lib/cli/RLTraderCLI.py | 2 ++ lib/env/TradingEnv.py | 36 +++++++++++---------- lib/env/exchange/BaseExchange.py | 5 +++ lib/env/exchange/LiveExchange.py | 4 +++ lib/env/exchange/SimulatedExchange.py | 43 ++++++++++++++++---------- 7 files changed, 62 insertions(+), 33 deletions(-) diff --git a/data/params.db b/data/params.db index 5f0448f8f244b855834a2b51e45be73e36047d43..6da8036e1cb9fac3b10bfb1dfcf21f798bb426c3 100644 GIT binary patch delta 24743 zcmb_kd3Y4Xwx90mnVx+T0$~k=HGmRkCbI@eCm|s~APIzoKoYWtgsdbXfT#nS3_%5n z2&1@xsGtI}2th>&Du}G2xLr{}+;0F;1iga1syP1U>Oc0`+0m@u(z z>h6XEw}Dm+o$5yJxz;Ao!mA0_acp^Crmz5yXk&YtI1t>8-K&@3hwLYc_pZ6vCOd;*w&v>7wZq z(_z!r-QCYUV#>vo%pM#Y#jWvDD^k= z|I~lAJ2WIipU5XG6|6?hHky*i1JBlXZaTbg$Nftc{CG@ItJwxV5gz9A3GmQpiYE_P z_JG#hxqHaXb1X_W_FBs`9=$Q%7-=*chZ|kSUKqh^F!LO%Vc5oG_`g00|F34)hVk(K z{6zS_F#-OskMEdhWcIhC_t4`V{08PThGq6+Mf;eJ$2Tz*de33hrcfp+W0ZrH;YuBM zjr)%Kkb9eZncK`g&MoH_aphbVm(01iQCu%hgRY}*(MRZA^agq!tv6jVeQr8o>NM>! zZNTr(W2!aHF->X9j2|1{H10NTG_E!-H8vV6jG4wHqt!Uj7;5AU zzZ*PX8$L9=W!PnS+OXDepJ9Qa+%Us1-VkFLX6S2B>96a*(Vx-3t?$rp)%)~q`euEZ zK1)AQKTbbf-&@b?uIawjeX2XDdqwwx?h)N`-9p_wU4d@0Zj5e(u9r@&{X_ec_A~8q z?W@}7wU25a&@R(fYxA|K+F0#KZG^U)=B7t;Npn_nQuBsphh~H3AI>=*)$gizt2e1vtM69Vs0-AY>T&8(>ON{Uf1Uq<|D1n^@8F-|AL5ts zjr?4G8lS*N^Zj_8>Za<7>NC|b)jrj8s!{WMd+ zbZpqoY-WsL27^U2U_8E*J*qeNf*1r)w!?rh3y9(^%8(rhX>9@rdz;af7kT z_<-@O@nz$k#%gT6dB!Q&ihCK=hTlva+m~ev3TizIYKqGmxC}xNd&B0-dc$P`El*CH z4zvO}ZJJ-9OdtN_71TmG9r*dQoCf8*0*@px6_n=6l}-WLEIBO|XtU+C$w14I(%~HOIxV}QJPpcW4V()I8C1a8u_KdPsv)c8 zk@ohtJ-}|H?DUm^7hM?w|A)3s#coI6Fg8zuwL2ZlhO|vlDcHgHjAnD! z419SglTMwukl@NEoXhq893Z`pAP#`8Z^3T>MRU8J8PyVD-pmI5^T zIkG!#p=}d@Hp@R0F`;e5mBYhBze;web=l~)B(V|6to++FXw9gXd;1ceV4X9Yni?vL z7c`Y6H#F2W49^=BHKU=Z6uX_twarnHb;a|_N}3`oiW(zp>mrM&Y-ChXNlD!T>~vOC zHa68YERM<>M8s`z7$Q-En6V~d@31w@p8u@=A#DlLOUokqeeI6_ z^^Fl<7>_#rR@#n1R{G&Yts6%^Mkm|M{l6gl!1LG%G->VNFwxeL zx`^8w<-tb&2J!1hW3S%8-Q<4cKH=WsUg5TJKJ3#samCy;ZanAY26Cb3Pt=7zLnqKd z^dfo;wTV9cDbrh~ZFpC9pJ|b4u4$HOs>x}JGIckx#w*4P*rPva+-ZCgyEjXWHO3-i zs&Ncg5NH>f%j5PE%==8VrKj=TzAJ)ID-=trw zU!iZ*m*_L}ar#mE0s1iAP2EM^KXgZQ9lGarPl&#JscyP1K^LQo(uL|6?IrD5?J@1^ z+MU|RwD)TlY0I@)+7#^=?NDtGEvvbz`C9Xl=CEe3<~faLoyLp(_!3Q)W|AgGGguR% z{!@KX{iXT?_3P?w>hK3;!K|nm@>I=O5?q=kMa@^SOK~ z@8XB?J$PRAhw2;EhpIPKJMdSwUDc|pQx&MDsA5$kRDD$@> zud;EHvQU|(9EX}w8JdYwksS@ie!POc%${dIWDl?}VL$#pb}?JU7O<08H}>PZvxxbX z`GNU>c}t;SD;1&QXBypsz4(KSZ~HFhCB2Sg!&r7ibx{qo!1PsQ`g8jh+*e0jI7I^|ib+YLI zJ0Sp@39!jBSZTparm%^=OWV2dJ^+(UU}ghMGQdpmJ++6^2i8xR6(V-l?z?-O3) z!V95l5~(@^swP3z1ZZIrZ6P}e8kiJl;8bYfcv%C-0?K%bGSWBhX|CVw@kGA>`p1L* z(V&04U%vzNj}Pc~gZ@OBe#tRRr22>Y+MnjaJA0^1K@;{diNbT$s7dc-gj+Z$lx z0kwg?UjWuYVVOSw*5QX`ZUL-a2Fn1fox)!9WjyEaoSk$IFQOz=Xa|%l zfMWNf{0=Dg0Lo23vB^*rfMTO4UC=pOaObW<=WL>FasX=s*q;E_=7;?aU~K`|8vq*< zfMxy!*cb}?Pk@aHhW!~}V@PLNK=s4`>?MGW@xxvN*q8w9zW~-MgZ&m@trYfsfVBq0 zehRQw49olguvS3*P_j*{ANC7?wFY3n0$7U-_8Wk;P}tJ|YYB$^2f$hg?2iCz0oaeA zvlfgx7DB+E1FR(gdmdoTGT5&H)=XjF^TloP&mc2_`5s`*0CNUl%>K@u1r&1to!nq}9Bbv1Q zr0>t2*x4w3-cKGy$R~VHJkJe~0*s>q3J&=!yBMnAb|QJ)H*Xh{-JO!q((u1F&1JQiEr0EvCD!7Mw#3-%F;1bSW=+C9VWVM4NE0m4 zE{DbLlGnlH7)Pu*Cfa3nTP!idTk6)dFAeVzsAI+2%yzqbcuOtegv&W*o7-h`C~qom zYj2N2!&|EVQr8mgbjP?!ORD@WamCu)v6h%M=vJE?zY+aLyF<;?L&~zGVgDAF%GN6As=~y=Zv+wqV1S!a|w#DF?Xc1ZpWjN%m z*rIXUotB`naK&2fv37fOjNM_4!DCVKmpb-ntIOetamm{z*0EWm9WIwU#v!y6lRD#J z{0I)NwOvNeRpXVVrgt$R-4m- z=Zg>T+q#R^PwF8sFSJ@MCxv0EKBT>`a(+`r-3tadYgf3nerx~^!e6Tek#tF!)A z2V0%lZT@>5@}+TFYug;*7yI7leanf*H?GhbzX6-zts^`TGTR@K-q| z_?v(Y)D|1#jCQ%44!c=Dn)WOEZ+Q4KZM6p3H?E8E@T_+`W2|`Wahyx~8aiVE4}Zd) z*j|I%Xd|t6#8|L0p#=wdBpbpj3bqt`wAKqpG87TrVN|t zaYUOf4$|TvY>T)OI9z6z(>1(h8ku?B$jqbI_~Y%cS$sRra_@HkZwxiVZ>atXW*h#f z3oTO!yl>SPoT}sW=UmJn6s2fc%PZ5EmiG4Oz9p^Qc^rA;mknk!Bo8CLdFV5p%WWfl zr`Iy6dwx`go7SCUxQt{6~&Msr8FV)O@Ei=x>h>}=*2rj_Zh*wfx#)VI7f7NbY-%c9sB z)IAaO8+S*AZSc)M$HlIR{H%NL(pEPX>d7w~!Oj#t8sXA~-IY&%^`h_mWF~E3RjUiL zqlk{IAfHG$lveV_)&Vnp8`a9RUad~5BwJGQUC8>u{a&C-98}4SU?oSN{LyK0+`s0^ z2Wfp;?N~@0tC{|>+OYn$r?kh#vBE92QIISExwu=sXwPm8lE$R>X^o-Pv;Eb#UaNbi zcpI&5rPXIhZ9M$#g`8!ASiN_vg;vk(gqcGBf-#FYNus z)?=&)(KD}gG!_WsmqoCd*h>?-!j_I)6ne(TFv`h2qA6>-$cl?^{AIvTz#2ta86s=Y z_Jt4F4+HCV%9tP|8f~U($LTW)2k50v>IAw}-OIejZltE853RifA24 zMWzQtglpfaY%2;EiX#15M^KSzGLh@8M~-^F@NLY-eP4P%XNqVYPQ@~0Vpo57>iy5Z zGWm9{d_>3qnLqLoF2|4P`nC?n65`Cy^v`_MZGXw2aa(*vKXYk) zTL)2r>3)Ik{ic8WN{t{145R|n`~sTA@2op~uS*h$qyiaKAnxYsqa78xNxm!1%v5YT z1F(c>I+?+<{zS>-?j!eR`%V{fX#-mOQ>p2}Qtvk$N$`~U&Sx@lzIwH?+kglvI4xN4 zy@|K2TXpquQC(zfKPr_GEOj{h!UO7gQIfs(#ZsctXG*5@-jSKzJKpl0FXqyFwf3Rh z=@Pf|TJqt??vuE^DR-J=M@M&bb5v!D+@7tyC^ti@`+ku&{d-@#mrKXr!=9Kc+IS|l zpvHImKRLoZ&o}K1Go^QHIOR{L{P|~YRbIH;xA7uJt*{4`nMP%rI=)Q0vvz`Lg&r-^ zDV}C;sd0EX89j{7a)Y^U$jLRDe&ilAeQY{pdfD_G{?xXZnoLEebW@zkY#L%R8viu@ z)A%V)GkDdw-S~*H&A7l=Vw{fiYFx%a#%_jNhKq)?hLeUj4O$JG9SfAHk_G%{YMW$Lr@q8jbozej$HYJyms4tx;F;X1Lq@MdYtN{>UVxT|FLQlzd}7g^_p^(@)K2=l2r{>tx-?l>?Qt zlsi>!?t=0MRio0Vv?=dW_27;v-&AFBm(gBM!L39ixm4~oREwTLmpKpW&)K;bQ7&4B zj&McnE&MCQ2Zk17gYlMOJMQ_<#x_Hl(QQ0$n2&#ESZ|2ZeXRe+kfm4S9~>GD8r?p9 z(m_Kqb4=e?U#$N_w^je5!LGZYyQUZPAL};gTMRvQZ)=a?-ykA&tC`nzTHR7zlJ=^u zUgyw$p(|u-uxpHTOybV=I7SNO&cPo(#By~ClHUUR@Dxm9bKyCkuqApPq2H1sJQKy~ z#}>kdJ|)N)*eMVR$PawVSSikv=O=j@St-;t$N!aK|0TuKQlzQJCc$<;KOiE7kn#eQ z>kuQDL%$R3H7-;EElBahHL^J%mM@8U%Gs+x$@5qIg#*e&KcxcjCkD`}p~v~bJ^l%( zdBN1n5X+bYV|7V;Ou@mhP85S>O~B3vb{CZLp!83OoJ^zxkq6~`TK)mbd9?gJD9$DC z=sS3xMV`Nf=h@`>8yJ*aaZo&Bk`((j@N)?N0`OB?kSP$ zVP{kJNf4h!%O{|mL(9kE9nB=~=oma_ljo!GzGlkC{aqks%ScC{Mi!~@4m{7m&mQ(L zyrY?c(LMx}>;UC$(3d6C_ZE<5$VhL(XwUSI_CX+I`w5+(IV+&qa{x#)0;D%!pl1dT z^y{E2%dcxch-C+hy#``_SDbwn1|8gP_7$iU@Ur3k?*k!ltJw|^3ihJez3@7y+sy8P zI^ZR2y&%|YAx4|>< zQQ569`kAuPe-TJB7nFSgNHUj`eI7`D_tI0*$ZmmGkQsOdo1uNwePDesBEcRO`xNvM zoE!Er7qjTEGcn*Mk~o-(`X%gI&>QTp zu}^?n@SfO5L9O4*WH*A^X`Vo*p9Q^i56M0ZdV@VY_Hob)ei!=)==D3N>;}*qaGBX> zV8qj9uj)ZSO2?L6KytI$B>v8CZy4i%N@{ywBQ!p11(4+EqDl8kOnQd8(NU&Z-EzDkQQjceSkW( zw$4M7aoLpsJC(vN2H2^=u*(2;DuHbW*r@<}7r;*S!?ptK)Bx-XfSn@itmgrMokC$3 z0qm4u*rfnFg}|-`*eL*eC%{he!?pnIlmKiSz^2Mz8v!-7)+7B|Spcx9!LV}yHkH6G z1lUx7Edkh6KWr7irUqc^0CutrwgF%#)6UN6;p?`I>mhl1lL<^Sz)S`hdQh znFlbF12DA!Gf4&`WpqxWF!|8ANx_{fgU(H=E=s9k7Xa)efGq~tNq*Q$fSnY8tpV7H zGT3_GxNTg|Ogy=VDLNFUhfE`kuNkuqR}5{20s2?;nYv556}pi)-=<7+3n#`Ws6SIL zz{!SN_(Yu0-lS3~pHf=5x479jdFWo$huyiGT7k8hB6*m+n&++q8@BY`>e=+?nGTg2@zZf`Et5pb7;&S9fo ztm>f=b@XYSO?CL)Cn0Pjf4PssQ`7iwgMNDM6DueEE!BztTN+4wex2*unml z@NQnyJ&!!H6-Sn)Vm&jd9>2dOs21NF@yO6CR8Kb56L7$YrzMd_;&Diu>ls)|++2fyCS2-NRb|H4&|p|5RTAv52_Fn!dSid?y^@mYR0%k6 z!ljM}2Nd?bD(%6iQ7$-i!lmyMR{ZjNlN1Wez+7>cO&z@W<`0{XDZ<@~MLu-#lD* zG$)~khRIT~fEXqt9wQEuwcP7*WDaY5 zH-*!EIJvjfgG4GKb714cyuWSO#2@ipS;S1lf!zcwBO0IIqZM@P4(L`+@E!aVN9@iO zGhukv$5Sc4n=7cJA6gQbSVv7%z!GAg0)DS>uxDJ_k`d*!PjOTv;0Fr_QXcD^_-3Xk zg5zA{u!v}b!H%(TQoCpG&_i1!8yrhp<9C#WhX$ZTzP_2-;25eW*mD++>6-R3Jw7FR z)~)7rsW#(>#yVq&VW(lbzDw`bN8+^^)2!87+AEIVrj{UZxpsAxBt2VCbU;Y z`9+k!aKc>k9n&^R_T*J!0ny?M#RWvT_;Ex2V^2FFwCv?*^+JF3f9<=~@PKdUEiTp< zqF086dl8lqO`<@Yd3{R~Hhfb|Cgr4XFH3nO79wetH z+daQ+9s?8Ji$f=%n*>XQu8`2E6Z=YG!h0x-#7=~+eCIp2ZM`9?jworpo2s4@P#xdx ztEWuyLw!4IaMNKqxeJSllc>PIJPMu7q1zwcZj)v`KAQw>lm%RbYj0)uiu-MkH0zg8 zkw5^ZQMk75P;v7LMjr7;0T@noU_(m+#$s3BsAxld~&74S!Z1m$BHs7vcReGf-| zoWZ#afD?QcQ#ks`JBuI8q`_XCi~+N=0A}Zpo9< zO&(2 z8& z8m>(GLGWE6V}~uxgnwTup-MVS>d`!4Hq}R>#tWTxidWxus zEM6vDJDgIGi{H^SvT~F%#&z`!H)V*bQeo=C2Y=r-IgDG3eGCs678yzn8HTa??f8TH zkdDL0K+kHAXkXTDz>)7o+EUG{I48WE-;W*Ihxw&^wdxhsvpBQ+E>$Jo>Gf9Sg|^Ly z7YdoKFcx8hfl~j zJoEs_zjs)qY*9%CIm;Q{gpZb#HNs($(6(v-rN_GLZhS(Z3TX6b7tSyVZJP%)e7q}4 zJRWFwK%_^!>~8C_VQm#+i(^>d`9b8ojoW7#$~w~ie~p&66|0^rsjd_cwz$Z- zkpGb8a!GTQILwLL(0@p8nWVP@T1-yL{6F8Hd?-TON@3*b;WwNQ71~w;G;#oroPV>s z@o|@8pwTmMc9%P}uB`}&^dOwwg(qYo(CA?}oCg!yRsb}5AkOZ>Crai3jUI}#yYPI; z2O2#XXLs3_Y1{Hh&zCJ>3#E63XMsItQf+D3BAB;P)%esBJ{pDzk#!52BJ0W{#k0oZ z*)VNeu2j85s!q;^{TtP1OVx{{>i7uMzf*mdRK1v1$4ACI|HsCMw&lQh&~tot7dDld zK%-~*>@I6)TQ<<>c|M$67uuEuG6�?K9dq{Ig|~#-!e%p3HyE z-^~wDbz*1w59L}h<)jFwoIH$%vHRF`d~o{?CRDMty*($Q*h_L5L}#!x;0(U8tJ|Xy z>6^iO@{*JW(Mc=wJ8Ak;nU_y&J0W>bUXs@!dQauS-qW2~+nlBUsDcf$m!vj`KKES7 zt9fJnJN5T`D|$8k%Dp75L4+yw!>H{q8-L3^NnuDvg9uX=40HN-w(ogYU+{apQBI7Ncwzj#SrgV>mIX#?=i`6cX_Zs3-9NlJt0^US4gN%QRR*w(s_M3aRbgJ9^7E(5RH$CND{75LJ{@6-_JmKD@DS6WxLgqAE(I;kdkZx`MY! zTM&}XAXYD<)fe$kFVB9-XPGTIKE1t>RHl^5%s=$$(v^isd5n@9ukkxtQ)G`A}Yxz92TUP;5#_bMeXH z?~$fV>gOfN3*zi83C!NklBesRidrpg54q^Oq{?afeGAwWY+eSF_3`b4Wk0Z0fFrX z6fw0_66i(+iu?lF+95YSdf{_vJLC<;5@JgWWi35B>W7~vpOQBB-ViJzT1iRp{6D;< z=!y7^4#`SPRH`^w>gf7i_dT=9EJ+!uR8g?hk?rWKla5iaE%F+$l-Th?+U&Sw-?R61 z@@;tALycy*Lp4BX2-Du11`;&+LwrQ9N^?uk8X3T!u| z&5&1v*Swa_3^5)kZ4UM)KV6XefIF1+oMEdTo=!%JjDJQ&jMN{)*Q_XX8+D1;YrYEyDt6*z{IBudCq4NcVgR^Cc}uxYIh;GdWuq(j!ji%G zoOupT=Ud4{DUK+L$!{j~Ux}AQ7{&QhC7VA#SDd$BT&`U*2p_ridGF(r!o4KaD9-f? zajuU`>wYb_FEu+7VHB0j3n+;XTc^3cso^cj>`0hV6sZh|2-jB6KVTX$0(RwI5^EHv zP*q?G9ZbHIT{`l(G}%e8QB+bPQ}WBAQ+t@#3MC~Z;wUPa7f^EG__i?TqF1R=k)Wd} zQW+5Gtoruhe~q~%Ei6;8gg9@i=)4g^KK(gWh>NDvox~kQ9ThSii-&k$ehVVfeZ3_7 zDC(H!*YV)9yb*6$wd-5S0>l z8=AI8Q6;TEBorxHOck}5=B9{_iy!-T&LF$na;6(LYZ8qVRa8hS_KhnleIH4SC<#Q0 zD(3lBoOtT2virA?I_4!&NReMj`HOO+&K~#>HqheU-b-STqD%#qX`Xo7kn)Ie(%#-n zLXe`t%oPnreKhR0OZ%z8#A2p6k>>^`a(uVXFAqEP(oQMV=BD+_#rpjE_g?>csk8>V zD6dT9g->3U*uY8)j}!C6nK?HwGY`D0xqS57?<6a7Q1Eg&_=?Bh9j6VE!0i;ER0J@; z+<)7Y@wD+aTE8q%U%2+ctmppxdVpkwF|-lofkt#b*laz$ZG>oIeY{pIA7#mHA_;~$Y#-2D@oT~p?f0sU7 z_mVC_`vLwH?z(1`W~h3&Na{pLfh-it3dh4Hy&x_B7sOzq*f*pu`Qjs75=LvLL!l(NKHUQ z_<2*OGDGvNBvMU9s%0YAEZ=`Ny|7c7&mTe!?U)boplTcs6j4&~NL zmhx81uBV^+L9(NK%B_*C?EMS6oOJwC8<)|`n@5?|wCe5+dA%Q(R<2yk6%DA48c@^2 z+wHyBLz02bru=25{)TkZsvQDZIYp7Q{Ft0r#jee&`;qKFN^XjMP5v|k-IC!pF<~lwAdM4s~W8^ z{e-y_LaCrKX_6wYA@d`dY zvtGLj2dFFX-6rF;R_zdNFMPMj&zdjQui#4xJJq*oUeK&p*Q+;cR%n)}2WZMPS@;^0 zwd%8)6m^)!q8X^3tiD%$OcSQ&)i?2#Cr%#m({22xs>S?T{<>IyzedrVcsd-(5E z&*Pg+PVw_qqxswT*HkGgpXwZ6q8fv5Jb6(y8^^(Ka#FHO10GTIAf%u)lIK;BP|^_i z(4dt-YLt;4fEx2jjr-xbo;=?N2`UZ#PXk&3l*RxB@_;<-e3`!G08uX^c_I0wff_h! z1wx~rAf*`152(Hu!0H2(dm!hgF~}UzvL2j=TpijFOkN5I^Zf`?7FoTYa5v=VG>Ybh z?t)SSl>`QnCVz4kFo89(;LaV39#mW>a8IK^5>a{A&oB3k3g~fy$xZ^lOH4;ROVL z@KFhjBz$U-bYEO8%{W7|p;GYo9~DC-_yQy8QaSqBL$jbx@M?gHpbmVOQ9iWRzkng> zDm?ma^q?Hj61-BNLeK&qaFhpH{Hq))g;yopZ=)G740K0>;z4ilwhv8&mruUDXeP$+ z;7c!H7lx7m#J^odQ{Wv2cG4&VU}ZaWG#y~+E(pZ|Z164-O#oQ(QAgR(HQ2U!P$Izk zcerRWzy|i-C=Fm`yM8ncVCm)xjRV-=Z6`_wSh8NAEP#dmAd>D7s`U76TDn9iu%AcL zB|@?|0ZIp0x?e+M0XBG}ipB#hiE^M!fQ3CI5}>pGT{iNhKyL#f0F(;7mF?<~bn6d| z*P@{S8@%5@(Ev;Kif9bLLNFSQ1X%wz30VPkZi$E98;hh{e`NbP`h5@X97Y+yr&JTcH)*)|Z_0G7sv(Lm^Ja9kV>2UxOuMJ|Aa za5WkNu>OEPx*cExTRdb2)Y4jy|NdKK0azMtMv(v;9B4;T0893?$PTa&;zol2)*s+U zBLFt=(*qg>u(H6i2lWG38oNVkfDMjsBJ4tt%fU#T8w~(hh&CcU!1}|os5`(0V#cT! zz{(=gNV<-Y@&j9UwN_zz)MPcjYb-SUY-qtr zz_043>3V#xyGPd#9~qpk`9<@vCR%+2Uz?}JuHirUo_Whj99{I(iGL_`*#|#Sd@llT zd^w4pi#*~l3SIrfA``wiA#I75lc2e{nUy(^@uA<|M1!}Nz-D$iiJFTdGKW&QcJ=YM zZki`U1Zg=5or@5}!4$f+oQm5Qsv2o`SSk|md4&UC{W;`jPNFC>uxdGjMa11}uoEmCzkY7?uH%bg zC%jxirGkB8;obTBVo#0hFNKRqo}Sn#zmF_19XHOL+cI6+n3G&RQNZsr3#tb;f5)#r xEp`e=SMS0S;^r5eXyMY?W%`CEan3I}^6MojdLkE`YT?qMajzfQmL*1r{|h!>x8(o; delta 5361 zcmY+IcYIYv7RT>BGq<-80%8F}C*u6I=_m%v0e&0EBX3jbD=FLmvnf{Hf z5o>SZmCX#p+_^3?6_m|y%-Pf?4lrsh!sNE7?Qz>DK(f(M``*Z~;V0&oE%KDsR+QCM zSCJK#LrPx?Lx6oUh{L&-e;r5@!;Fn~ggaZ*gOR?GaqKdqW#j2G6^x@cE-)?_N6oNS z_EN|7Gi|#0S{oaaVDa1fP`DjnRJmtSO>J4#f(rJ1i{9FYvV&o8Rdvmhikj;3g>}q% z&BAUirbW2w?dTKslG+h-y$gt2&s{IDHrcEBa&>msI&vP;d_pUGk_A%ld%c5wg*9}_ z-U@@NO6#m>b
hK=k()!PX!%I)A@=APx6xd*t##>d7^;}v6rvCe2PDvg;&zA?@i zWkea8enY>gpVU9m|Dkv2f774RSLhynmR_P~>9KmGu4~t|OWGOjGi|r_hW4V?tUajx zNt>_Tsby$!+U=U5f%+fyta?P9W)*eJs5#ZIzyr)<_Lfm2|H(SxS@Qr6?&}0`Z!7Ry-)aC%z$W7M~KmUeP1Y5c9nXn#t{59BM-N&ZPXNGoX;RQ?8kjql?3^E>%1{L}m@zJZ_5 zPvvK3k_K`gna!t=9FjtYl5oPp5AZb{f<3T}AIkUTIqny(n>zx}!)mC9#oUK*H)Mee zRQ3k@4g0zAB?GZcU+eS)4hs8`(Zzply0_6NvvM1Cfzf3~w9&EV^fsDhuG&c3%*;*H z6^_R)I&kc^_@-`Wx3H$-V~3c}b+EHlg@P4(q^GQs-EY3oM$0t@(Fhy0n@cv)14?v7 z)soWk>Wca7Mqj#*eNP-}ol}j%O#nj{lvP%ivEFbBg8`y~_PDHRH@jmu&{p~;X<*C= zH85|BXAWV*U?02%n_w+8f(K?nuCJkt+|QcJFB6&SBbi2X!WEL=vg3emUYtwr@U1K- z-Uf+ym}ki}hSZpz?;+6~eTMv=rpIaD=&E%df7gy`?b;fxRjbz)YFXMeEl$0vwrY{; zJ}pe`(irtEb*(x}ZB%pB8a2^6jA1IHT=Xsfk~FbqTBo>%L4ZY8r+$s19 z>gkSXue6If%zgmUtI`YNW$7_#nKVzjOG=j##Iw>sNfUn+c8RR`S7Eexw|H2n6SfLh z#U>$2OcnPEbA)Gv)8b-57Q@9ig(9Iz_*ASH`in8*OTswrC*IBP=N9pg^S^ShbBFl> z{5CG%x*9HUtGL~~z_)U#{A4eGf?LWx&z<8NxOjd9zl)nnPtr?V1l>h%a8Yz0O{1^T zV(S+2&<(VK&LZEE-)I#1n5s0693u=#CmUc2sVDEi7C1;ENi#%}xnwKUu-BoKgd6X| zJLE3o68jyDglk3{tRpVt1ltJ)JZx+P??gCje5kh@)Ob`MXSj_PeX%}Gf6;Jh$Mws` zQXTYcz1bL|b?cuS6?#8CMqgp{(>~Gn8uw^d)XnMzZH0P=Hc@+*{lL2TJGC-(kQS}I zYJKD$vOa>W)O(em<@NHXN`$gi&Qj(o z=jE01E`=x?C zDj)%;xa}$NI3?Xb#f4i<>q#TE=j$Kh@Jh|>xl$n(gJy&V9gXwcfr}PKnTNK}{=GS* z1#7mGyYMs+)`l;*572({bM1F^8Fc42SBdZY&G;RWkqWk0u41F7AD zdrkExb2Dx#HFQ%0a2u((jhk@URA<@#I64ML`=J_(Dgu2cTE1TU{RMpyjB_T3qn_aC zKB&h!It=xAM;oZeI9kV?Q|z2I%sC~HGpLwyij}e#pO~Ov&ME$!Wz0DxG-nC(NWnZr zTsFm7R={PG?PYnilWdzqJJGf&ZYQ~SJ1O?Zj^KELJq|dY6trCyQDP8c%&x6;fHxt4 z*c+HslE07abtH)a@;hdk>}1LQhAs&KmtWB($?x(Dfq&gUWv%kt+y&Wg{;@ zTo5~lZm~hA8+#BR=)qYet^oNGNnC)OK@uAvr?CO?{sw$ugP8y>KRJaiaemN+Al45~ zVk=ystvG=+K9qKvrLX%N92+`z92?@oh8#l`kLoj2aj1@>ibZtl#qr(}Yc{lCq+;9t3))4ty$P}+#CDCubhcu^Ijday@%p$O#)94tiDfajp3x9#v})Wwcoj=IRv z51=k|bUik%z;4_!wDWEIezfy!d#P=Ywq8%RSb|L|2x?LtqWmCKYnH`weS2RQ1%W11 zV_WhAqzXx1fK(#M4G<6Zp}^mm`w-;&!D0k?ez3>}L##df$wDlAL1^I@n0hSdT+Vr+ zV-+|fHO|#j587s(uf~&9dX(jicFt(JAJ* zm+3Gs);-5x<;e)L{h$QFBtIy|P+5UcMOf#Y&^i|)%?_mnNGJ8u%-&aXKCY7GUnLJ~ zpM$l}MU{;z2h}80*=F_2*6oeO&T)#Jg*w~OnW!f@Is;>5VT^Pf&2mQ5uymPr>D*{% z*mf$~>9#!)n~;f37-|124<;bX@HeIo#v@7(LgRL{Q#O*$hLv`WycTr(uUs diff --git a/lib/RLTrader.py b/lib/RLTrader.py index 0b772c2..dbf7b58 100644 --- a/lib/RLTrader.py +++ b/lib/RLTrader.py @@ -157,7 +157,7 @@ def optimize_params(self, trial, n_prune_evals_per_trial: int = 2, n_tests_per_e rewards = [] n_episodes, reward_sum = 0, 0.0 - trades = train_env.get_attr('trades') + trades = [exchange.trades for exchange in train_env.get_attr('exchange')] if len(trades[0]) < 1: self.logger.info(f'Pruning trial for not making any trades: {eval_idx}') @@ -293,3 +293,6 @@ def test(self, model_epoch: int = 0, render_env: bool = True, render_report: boo self.logger.info( f'Finished testing model ({self.study_name}__{model_epoch}): ${"{:.2f}".format(np.sum(rewards))}') + + def live(self, paper_mode: bool = True): + pass diff --git a/lib/cli/RLTraderCLI.py b/lib/cli/RLTraderCLI.py index 4883b29..92d9e84 100644 --- a/lib/cli/RLTraderCLI.py +++ b/lib/cli/RLTraderCLI.py @@ -71,6 +71,8 @@ def __init__(self): help='Save the performance report as .html') live_parser = subparsers.add_parser('live', description='Live model') + live_parser.add_argument('--paper-mode', dest="paper_mode", action="store_true", + help='Trade in paper mode') subparsers.add_parser('update-static-data', description='Update static data') diff --git a/lib/env/TradingEnv.py b/lib/env/TradingEnv.py index 954791f..6ea6c8b 100644 --- a/lib/env/TradingEnv.py +++ b/lib/env/TradingEnv.py @@ -20,12 +20,14 @@ class TradingEnvAction(Enum): SELL = 1 HOLD = 2 + class TradingMode(Enum): TRAIN = 0 TEST = 1 PAPER = 2 LIVE = 3 + class TradingEnv(gym.Env): """A reinforcement trading environment made for use with gym-enabled algorithms""" metadata = {'render.modes': ['human', 'system', 'none']} @@ -37,8 +39,8 @@ def __init__(self, reward_strategy: BaseRewardStrategy = IncrementalProfit, trade_strategy: BaseTradeStrategy = SimulatedTradeStrategy, initial_balance: int = 10000, - commissionPercent: float = 0.25, - maxSlippagePercent: float = 2.0, + commission_pct: float = 0.25, + max_slippage_pct: float = 2.0, trading_mode: TradingMode = TradingMode.PAPER, exchange_args: Dict = {}, **kwargs): @@ -51,14 +53,14 @@ def __init__(self, self.min_cost_limit: float = kwargs.get('min_cost_limit', 1E-3) self.min_amount_limit: float = kwargs.get('min_amount_limit', 1E-3) - self.commissionPercent = commissionPercent - self.maxSlippagePercent = maxSlippagePercent + self.commission_pct = commission_pct + self.max_slippage_pct = max_slippage_pct self.trading_mode = trading_mode self.data_provider = data_provider self.reward_strategy = reward_strategy() - self.trade_strategy = trade_strategy(commissionPercent=self.commissionPercent, - maxSlippagePercent=self.maxSlippagePercent, + self.trade_strategy = trade_strategy(commissionPercent=self.commission_pct, + maxSlippagePercent=self.max_slippage_pct, base_precision=self.base_precision, asset_precision=self.asset_precision, min_cost_limit=self.min_cost_limit, @@ -83,7 +85,7 @@ def __init__(self, self.n_discrete_actions: int = kwargs.get('n_discrete_actions', 24) self.action_space = spaces.Discrete(self.n_discrete_actions) - self.n_features = 5 + len(self.data_provider.columns) + self.n_features = 6 + len(self.data_provider.columns) self.obs_shape = (1, self.n_features) self.observation_space = spaces.Box(low=0, high=1, shape=self.obs_shape, dtype=np.float16) @@ -99,21 +101,21 @@ def _get_trade(self, action: int): action_type: TradingEnvAction = TradingEnvAction(action % n_action_types) action_amount = float(1 / (action % n_amount_bins + 1)) - commission = self.commissionPercent / 100 - max_slippage = self.maxSlippagePercent / 100 + commission = self.commission_pct / 100 + max_slippage = self.max_slippage_pct / 100 amount_asset_to_buy = 0 amount_asset_to_sell = 0 - if action_type == TradingEnvAction.BUY and self.balance >= self.min_cost_limit: - price_adjustment = (1 + slippage) * (1 + max_slippage) + if action_type == TradingEnvAction.BUY and self.exchange.balance >= self.min_cost_limit: + price_adjustment = (1 + commission) * (1 + max_slippage) buy_price = self._current_price() * price_adjustment buy_price = round(buy_price, self.base_precision) - amount_asset_to_buy = self.balance * action_amount / buy_price + amount_asset_to_buy = self.exchange.balance * action_amount / buy_price amount_asset_to_buy = round(amount_asset_to_buy, self.asset_precision) - elif action_type == TradingEnvAction.SELL and self.asset_held >= self.min_amount_limit: - amount_asset_to_sell = self.asset_held * action_amount + elif action_type == TradingEnvAction.SELL and self.exchange.asset_held >= self.min_amount_limit: + amount_asset_to_sell = self.exchange.asset_held * action_amount amount_asset_to_sell = round(amount_asset_to_sell, self.asset_precision) return amount_asset_to_buy, amount_asset_to_sell @@ -128,8 +130,8 @@ def _reward(self): reward = self.reward_strategy.get_reward(current_step=self.current_step, current_price=self._current_price, observations=self.observations, - account_history=self.account_history, - net_worths=self.net_worths) + account_history=self.exchange.account_history, + net_worths=self.exchange.net_worths) reward = float(reward) if np.isfinite(float(reward)) else 0 @@ -200,6 +202,8 @@ def step(self, action): elif amount_asset_to_sell: self.exchange.sell(amount_asset_to_sell) self.reward_strategy.reset_reward() + else: + self.exchange.hold() self.current_step += 1 diff --git a/lib/env/exchange/BaseExchange.py b/lib/env/exchange/BaseExchange.py index a58cb32..665ae5a 100644 --- a/lib/env/exchange/BaseExchange.py +++ b/lib/env/exchange/BaseExchange.py @@ -5,6 +5,7 @@ from enum import Enum from lib.env import TradingEnv + class BaseExchange(object, metaclass=abc.ABCMeta): @abc.abstractmethod @@ -22,3 +23,7 @@ def buy(self, amount: float): @abc.abstractmethod def sell(self, amount: float): raise NotImplementedError + + @abc.abstractmethod + def hold(self): + raise NotImplementedError diff --git a/lib/env/exchange/LiveExchange.py b/lib/env/exchange/LiveExchange.py index d70313a..7740804 100644 --- a/lib/env/exchange/LiveExchange.py +++ b/lib/env/exchange/LiveExchange.py @@ -2,6 +2,7 @@ from lib.env import TradingEnv from lib.env.exchange import BaseExchange + class LiveExchange(BaseExchange): def __init__(self, env: TradingEnv, **kwargs): @@ -16,3 +17,6 @@ def buy(self, amount: float): def sell(self, amount: float): pass + + def hold(self): + raise NotImplementedError diff --git a/lib/env/exchange/SimulatedExchange.py b/lib/env/exchange/SimulatedExchange.py index 247bd8b..5431c63 100644 --- a/lib/env/exchange/SimulatedExchange.py +++ b/lib/env/exchange/SimulatedExchange.py @@ -4,6 +4,7 @@ from lib.env import TradingEnv from lib.env.exchange import BaseExchange + class SimulatedExchange(BaseExchange): def __init__(self, env: TradingEnv, initial_balance: int = 10000, **kwargs): @@ -12,13 +13,13 @@ def __init__(self, env: TradingEnv, initial_balance: int = 10000, **kwargs): self.balance = self.initial_balance self.net_worths = [self.initial_balance] self.asset_held = 0 - self.current_step = 0 self.account_history = pd.DataFrame([{ 'balance': self.balance, + 'asset_held': self.asset_held, 'asset_bought': 0, - 'cost_of_asset': 0, + 'purchase_cost': 0, 'asset_sold': 0, - 'revenue_from_sold': 0, + 'sale_revenue': 0, }]) self.trades = [] @@ -27,11 +28,11 @@ def get_account_history(self): def _add_trade(self, amount: float, total: float, side: str): self.trades.append({'step': self.env.current_step, - 'amount': asset_bought, - 'total': purchase_cost, + 'amount': amount, + 'total': total, 'type': side}) current_net_worth = self.balance + self.asset_held * self.env._current_price() - current_net_worth = round(current_net_worth, self.base_precision) + current_net_worth = round(current_net_worth, self.env.base_precision) self.net_worths.append(current_net_worth) def buy(self, amount: float): @@ -53,9 +54,6 @@ def buy(self, amount: float): 'sale_revenue': sale_revenue, }, ignore_index=True) - self.current_step += 1 - self.current_ohlcv = self.data_provider.next_ohlcv() - def sell(self, amount: float): trade = self.env.trade_strategy.trade(buy_amount=0, sell_amount=amount, @@ -68,24 +66,37 @@ def sell(self, amount: float): self._add_trade(amount=asset_sold, total=sale_revenue, side='sell') self.account_history = self.account_history.append({ 'balance': self.balance, + 'asset_held': self.asset_held, 'asset_bought': asset_bought, - 'cost_of_asset': cost_of_asset, + 'purchase_cost': purchase_cost, 'asset_sold': asset_sold, - 'revenue_from_sold': revenue_from_sold, + 'sale_revenue': sale_revenue, + }, ignore_index=True) + + def hold(self): + self.account_history = self.account_history.append({ + 'balance': self.balance, + 'asset_held': self.asset_held, + 'asset_bought': 0, + 'purchase_cost': 0, + 'asset_sold': 0, + 'sale_revenue': 0, }, ignore_index=True) + current_net_worth = self.balance + self.asset_held * self.env._current_price() + current_net_worth = round(current_net_worth, self.env.base_precision) + self.net_worths.append(current_net_worth) + def reset(self): self.balance = self.initial_balance self.net_worths = [self.initial_balance] self.asset_held = 0 - self.current_step = 0 - self.last_bought = 0 - self.last_sold = 0 self.account_history = pd.DataFrame([{ 'balance': self.balance, + 'asset_held': 0, 'asset_bought': 0, - 'cost_of_asset': 0, + 'purchase_cost': 0, 'asset_sold': 0, - 'revenue_from_sold': 0, + 'sale_revenue': 0, }]) self.trades = []