pyrenko.py

'''
python3
renko brick calculation python implementation
Jack Boynton 2019
'''
from tqdm import tqdm
import numpy as np
import pandas as pd
from math import floor
import datetime
import sys
import requests
from data import new_trade
from engines import BitmexTrader, BinanceTrader, RobinhoodTrader, AlpacaTrader
import threading
from calculate_pred import main as pred
import statistics
from b_trader import trader as b_trader
"""
sys.path.insert(1, '/home/jayce/rest_api/')
from balance import update as update_  # not working
"""

class renko:
    def __init__(self, plot, j_backtest, fast, slow, signal_l, to_trade, strategy, ordtype):

        self.trade = BitmexTrader(
            trade=to_trade, leverage=10, tp=0.5, test=False, ord_type=ordtype)
        self.j_backtest = j_backtest
        self.fast = int(fast)
        self.slow = int(slow)
        self.signal_l = int(signal_l)
        self.source_prices = []
        self.returns = []
        self.trades_ = []
        self.renko_prices = []
        self.renko_directions = []
        self.out = []
        self.last_loaded = 0
        self.plot = plot  # unused
        self.timestamps = []
        self.macdaa = []
        self.n = 20 # ema--rsi
        self.smaa = []
        self.act_timestamps = []
        self.end_backtest = datetime.datetime.now()
        self.strategy = strategy
        self.use_ml = False
        self.b_ = b_trader(0.1) # 0.1 BTC starting balance  

    def set_brick_size(self, HLC_history=None, auto=True, brick_size=10):
        if auto:
            self.brick_size = self.__get_optimal_brick_size(
                HLC_history.iloc[:, [0, 1, 2]])
        else:
            self.brick_size = brick_size
        return self.brick_size

    def __renko_rule(self, last_price, ind):
        # determines if should plot new bricks
        # returns number of new bricks to plot
        # print(last_price,ind)
        try:
            gap_div = int(
                float(last_price - self.renko_prices[-1]) / self.brick_size)
        except Exception:
            gap_div = 0
        is_new_brick = False
        start_brick = 0
        num_new_bars = 0

        if gap_div != 0:
            if (gap_div > 0 and (self.renko_directions[-1] > 0 or self.renko_directions[-1] == 0)) or (gap_div < 0 and (self.renko_directions[-1] < 0 or self.renko_directions[-1] == 0)):
                num_new_bars = gap_div
                is_new_brick = True
                start_brick = 0
            elif np.abs(gap_div) >= 2:
                num_new_bars = gap_div
                num_new_bars -= np.sign(gap_div)
                start_brick = 2
                self.renko_prices.append(
                    self.renko_prices[-1] + 2 * self.brick_size * np.sign(gap_div))
                #print(self.timestamps[ind])
                self.act_timestamps.append(self.timestamps[ind+1])
                self.renko_directions.append(np.sign(gap_div))
            if is_new_brick:
                # add each brick
                for d in range(start_brick, np.abs(gap_div)):
                    self.renko_prices.append(
                        self.renko_prices[-1] + self.brick_size * np.sign(gap_div))
                    self.act_timestamps.append(self.timestamps[ind+1])
                    self.renko_directions.append(np.sign(gap_div))
        return num_new_bars

    def build_history(self, prices, timestamps):
        # builds backtest bricks
        
        print(len(self.renko_prices))
        self.orig_prices = prices
        print(prices[1].values[-1])
        if len(prices) > 0:
            self.timestamps = prices[1].values
            self.source_prices = pd.DataFrame(prices[2].values)
            self.renko_prices.append(prices[2].values[-1])
            self.act_timestamps.append(prices[1].values[-1])
            self.renko_directions.append(0)
            if self.last_loaded == 0:
                for n, p in tqdm(enumerate(self.source_prices[1:].values), total=len(self.source_prices[1:].values), desc='build renko'):  # takes long time
                    # print(type(p),p)
                    self.__renko_rule(p, n)  # performs __renko_rule on each price tick
                self.last_loaded = len(self.renko_prices)
                #self.source_prices = []
            else:
                for n, p in tqdm(enumerate(self.source_prices[1:].values), total=len(self.source_prices[1:].values), desc=f'build renko {self.last_loaded}:{self.last_loaded+len(self.source_prices[1:].values)}'):  # takes long time
                    # print(type(p),p)
                    self.__renko_rule(p, n)
                self.last_loaded = len(self.renko_prices)

            # map(lambda x: self.__renko_rule(x[1], x[0]), enumerate(self.source_prices[1:].values))
        return len(self.renko_prices)

    def do_next(self, last_price):
        # function that is used to plot live data
        # calls __renko_rule on each new live price tick
        last_price = float(last_price)
        if len(self.renko_prices) == 0:
            self.source_prices.append(last_price)
            self.renko_prices.append(last_price)
            self.renko_directions.append(0)
            return 1
        else:
            self.source_prices.append(last_price)
            return self.__renko_rule(last_price, 1)

    def get_renko_prices(self):
        return self.renko_prices

    def get_renko_directions(self):
        return self.renko_directions

    def plot_renko(self, col_up='g', col_down='r'):
        self.last_timestamp = datetime.datetime(
            year=2018, month=7, day=12, hour=7, minute=9, second=33)  # random day in the past to make sure all data gets loaded as backtest
        self.ys = []
        self.xs = []
        self.U = [] # ema--rsi
        self.D = [] # ema--rsi
        self.lll = 0
        self.prices = []
        self.next_brick = 0
        self.backtest = True
        self.backtest_bal_usd = 0.0028655
        self.init = self.backtest_bal_usd
        self.backtest_fee = 0.00075  # 0.075%
        self.backtest_slippage = 12 * 0.5  # ticks*tick_size=$slip
        self.leverage = 50
        self.w = 1
        self.l = 1
        self.runs = 0
        self.balances = []
        self.ff = True
        self.long = False
        self.short = False
        self.open = 0
        self.profit = 0

        self.first = True

        for i in tqdm(range(1, len(self.renko_prices)),total=len(self.renko_prices)): # start backtest
            self.col = col_up if self.renko_directions[i] == 1 else col_down
            self.x = i
            self.y = self.renko_prices[i] - self.brick_size if self.renko_directions[i] == 1 else self.renko_prices[i]
            self.last = self.renko_prices[-1]
            self.aaa = self.last
            self.animate(i)

        self.last = self.renko_prices[-1]
        self.backtest = False
        self.bricks = 0

        #self.source_prices = []
        # self.ys = [0]
        self.l = 1
        self.w = 1
        #print(self.returns)
        #self.returns = map(lambda x: x*100, self.returns)
        sr = pd.DataFrame(self.returns[2:]).cumsum()
        sra = (sr - sr.shift(1))/sr.shift(1)
        srb = sra.mean()/sra.std() * np.sqrt(365) # calculate sharpe ratio for 1 year trading every day
        #self.trade.end_backtest(self.pricea)
        self.b_.end(self.pricea,self.act_timestamps[-1])
        #print('net backtest profit: BTC ' + str(self.backtest_bal_usd - self.init) + ' :: ' + str(round(((self.backtest_bal_usd-self.init)/self.init)*100, 3)) + ' percent')
        #print('net backtest profit: BTC ' + str(self.backtest_bal_usd - self.init), 'max drawdown: ' + str(round(min(self.trades_), 8)) + ' BTC', 'max trade: ' + str(round(max(self.trades_), 8)) + ' BTC', 'average: ' + str(round(statistics.mean(self.trades_), 8)) + ' BTC', 'SR: ' + str(round(srb[0], 5)))
        if not self.j_backtest:
            self.trade.end_backtest(self.pricea)
            while True:
                # starts live trading
                self.check_for_new()
        else:
            return self.out
    
    def check_for_new(self):
        # connects to hosted BITMEX delta server running in nodejs on port 4444
        data = requests.get(
            'http://132.198.249.205:4444/quote?symbol=XBTUSD').json()

        for key in data:
            #print(str(key['timestamp']))
            if datetime.datetime.strptime(key['timestamp'].replace('T', ''), '%Y-%m-%d%H:%M:%S.%fZ') > self.last_timestamp:
                self.add_to_plot(float(key['bidPrice']), self.do_next(
                    np.array(float(key['bidPrice']), dtype=float)))
                update_()
                act_price = 0
                print(str(float(key['bidPrice'])) + ' brick: ' + str(self.last) + ' sma: ' + str(self.smaa[-1]) + ' macd: ' + str(
                    self.macdaa[-1]) + ' len: ' + str(len(self.ys)) + ' bricks: ' + str(self.bricks) + ' y: ' + str(self.renko_prices[-1]) + ' act: ' + str(act_price), end="\r")
                self.last_timestamp = datetime.datetime.strptime(
                    key['timestamp'].replace('T', ''), '%Y-%m-%d%H:%M:%S.%fZ')
                #print(self.last_timestamp)
                #self.bid = float(key['bidPrice'])

    def add_to_plot(self, price, bricks):
        self.aaa = self.last
        self.bricks = bricks
        self.prices.append(self.last)
        if bricks >= 1:
            bricks += 1
        for i in range(1, bricks):
            self.x = self.x + i
            #print()
            self.y = self.renko_prices[(-bricks + i) - 1] - self.brick_size if self.renko_directions[(
                -bricks + i) - 1] == 1 else self.renko_prices[(-bricks + i) - 1]
            self.last = self.renko_prices[(-bricks + i) - 1]
            self.aaa = self.last
            self.animate(1)
        self.last = self.renko_prices[-1]

    def animate(self, i):
        self.lll += 1
        self.ys.append(self.y)  # all bricks for indicator calculation
        self.xs.append(self.x)  # num bars
        if self.next_brick == 1:
            self.col = 'b'
        elif self.next_brick == 2:
            self.col = 'y'
        self.balances.append(self.profit)
        self.calc_indicator(i)  # calculates given indicator

    def ma_(self):
        # vanilla moving average
        return pd.DataFrame(self.ys).rolling(window=self.n).mean()

    def ma(self):
        # calculates simple moving averages on brick prices
        if (1==1):
            fast_ma = pd.DataFrame(self.ys).rolling(window=self.fast).mean()
            slow_ma = pd.DataFrame(self.ys).rolling(window=self.slow).mean()
            return fast_ma.values, slow_ma.values
        else:
            # doesnt work... calculates the mas for the entire source prices df not changing with time
            fast_ma = pd.DataFrame(self.source_prices).rolling(window=self.fast).mean()
            slow_ma = pd.DataFrame(self.source_prices).rolling(window=self.slow).mean()
            return fast_ma.values, slow_ma.values

    def macd(self):
        # calculated moving average convergence divergence on brick prices
        fast, slow = self.ma()
        macda = []
        for n, i in enumerate(fast):
            macda.append(i - slow[n])
        self.macdaa = macda
        return macda

    def sma(self):
        # simple moving average to compare against macd
        self.smaa = (pd.DataFrame(self.macd()).rolling(
            window=self.signal_l).mean()).values
        return (pd.DataFrame(self.macd()).rolling(window=self.signal_l).mean()).values

    def ema_(self, t, n):
        # exponential moving average
        return pd.DataFrame(t).ewm(span=n, adjust=False).mean().values

    def rsi(self):
        if len(self.ys) > 10:
            if self.ys[-1] > self.ys[-2]:
                self.U.append(self.ys[-1] - self.ys[-2])
                self.D.append(0)
            elif self.ys[-2] > self.ys[-1]:
                self.U.append(0)
                self.D.append(self.ys[-2] - self.ys[-1])
            else:
                self.D.append(0)
                self.U.append(0)
            try:
                RS = (self.ema_(self.U, self.n)/self.ema_(self.D, self.n))
            except:
                RS = 0
            if RS.any() != 0:
                return list(map(lambda x: 100-(100/(1+x)),RS))
    
    def wma(self,p):
        # weighted moving average
        ret = []
        times = []
        for n,num in enumerate(self.ys):
            if n > p:
                use = self.ys[n-p:n]
                times.append(self.act_timestamps[n])
                weights = [x/sum(list(range(1,p+1))) for x in range(1,len(use)+1)]
                ret.append(sum([use[x]*weights[x] for x in range(len(weights))]))
        return ret,times

    


    def cross(self, a, b):
        # determines if signal price and macd cross or had crossed one brick ago
        try:
            if (a[-2] > b[-2] and b[-1] > a[-1]) or (b[-2] > a[-2] and a[-1] > b[-1]) or (a[-2] > b[-2] and b[-1] == a[-1]) or (b[-2] > a[-2] and b[-1] == a[-1]):
                return True
            return False
        except Exception:
            return False

    def close_short(self, price):
        # calculates profit on close of short trade
        net = round(((1 / self.pricea - 1 / (self.open)) * floor(self.risk*self.open)*self.leverage), 8)
        self.profit += net
        fee = round((self.risk*self.leverage * self.backtest_fee), 8)
        fee += round((self.risk*self.leverage * self.backtest_fee), 8)
        self.profit -= fee
        self.backtest_bal_usd += round((net - fee), 8)
        ret = round((net - fee), 8)/self.risk
        self.returns.append(ret)
        try:
            per = ((self.w + self.l) - self.l) / (self.w + self.l)
        except Exception:
            per = 0
        self.trades_.append(round((net-fee), 8))
        print('trade: BTC ' + str(round((net - fee), 8)), 'net BTC: ' + str(round(self.profit, 8)),
              'closed at: ' + str(self.pricea), 'profitable?: ' + str('yes') if price < self.open else str('no'), 'balance: BTC ' + str(self.backtest_bal_usd), 'percentage profitable ' + str(round(per * 100, 3)) + '%', 'w:' + str(self.w), 'l:' + str(self.l))
        if price < self.open:
            self.w += 1
        else:
            self.l += 1

    def close_long(self, price):
        # calculates profit on close of long trade
        if price > self.open:
            self.w += 1
        else:
            self.l += 1
        net = round(((1 / self.open - 1 / (self.pricea)) * floor(self.risk*self.open)*self.leverage), 8)
        self.profit += net
        fee = round((self.risk*self.leverage * self.backtest_fee), 8)
        fee += round((self.risk*self.leverage * self.backtest_fee), 8)
        self.profit -= fee
        # print(type(net-fee))
        self.backtest_bal_usd += round((net - fee), 8)
        ret = round((net - fee), 8)/self.risk
        self.returns.append(ret)
        try:
            per = ((self.w + self.l) - self.l) / (self.w + self.l)
        except Exception:
            per = 0
        self.trades_.append(round((net - fee), 8))
        print('trade: BTC ' + str(round((net - fee), 8)), 'net BTC: ' + str(round(self.profit, 8)),
              'closed at: ' + str(self.pricea), 'profitable?: ' + str('no') if price < self.open else str('yes'), 'balance $' + str(self.backtest_bal_usd), 'percentage profitable: ' + str(round(per * 100, 3)) + '%', 'w:' + str(self.w), 'l:' + str(self.l))

    def calc_indicator(self, ind):
        # calculates indicator
        #print(f"using {self.strategy}")
        if self.strategy == "macd":  # can add more indicators by expanding if condition:
            self.pricea = self.y + self.brick_size  # calculates indicator on each new brick
            if self.cross(self.macd(), self.sma()) and self.macd()[-1] > self.sma()[-1] and not self.long:
                self.long = True
                self.short = False
                if self.runs > 0:
                    #print(f"closed trade at {self.trade.close_backtest_short(self.pricea):.8f} of profit")
                    self.b_.close(self.pricea)
                    if self.long:
                        side = 0
                    elif self.short:
                        side = 1
                    if len(self.renko_prices) > 10 and len(self.macd()) > 10:
                        # write trades to file
                        new_trade(past_bricks=self.ys_open, price_open=self.open, price_close=self.pricea, side=side, macd_open=self.macd_open, macd_close=self.macd()[-1], sma_open=self.sma_open, sma_close=self.sma()[-1], time_open=self.open_time, time_close=self.act_timestamps[ind])
                if self.end_backtest <= self.last_timestamp and not self.j_backtest and len(self.ys) > 35 and self.trade.backtest_over == True:
                    predi = pred(self.ys[-10:], self.macd()[-10:], self.sma()[-10:], self.pricea)
                    threading.Thread(target=self.trade.buy_long, args=("BITMEX", "XBT-USD", self.pricea, self.pricea-self.brick_size, predi, )).start()
                    if self.ff: # if done backtest: 
                        #self.trade.end_backtest(self.pricea)
                        self.b_.end(self.pricea)
                        print('net backtest profit: BTC ' + str(self.backtest_bal_usd) +
                              ' with $' + str(self.backtest_slippage) + ' of slippage per trade', 'max drawdown: ' + str(min(self.trades_)), 'max trade: ' + str(max(self.trades_)), 'average: ' + str(statistics.mean(self.trades_)))
                        print('proceeding to live...')
                        self.backtest_bal_usd = self.init
                        self.profit = 0
                        self.ff = False
                    act_price = float(requests.get("https://www.bitmex.com/api/v1/orderBook/L2?symbol=xbt&depth=1").json()[1]['price'])
                    print('BUY at: ' + str(self.pricea), ' act: ' + str(act_price),
                          str(datetime.datetime.now()), 'pred: ' + str(pred(self.ys[-10:], self.macd()[-10:], self.sma()[-10:], self.pricea)))
                else:
                    if ind != 1:
                        sss = self.act_timestamps[ind]
                    else:
                        sss = 'undef'
                    if len(self.macd()) > 10 and len(self.sma()) > 10 and len(self.ys) > 10:
                        if self.use_ml:
                            predi = pred(self.ys[-10:], self.macd()[-10:], self.sma()[-10:], self.pricea)
                        else:
                            predi = 1
                        self.risk = self.backtest_bal_usd * predi
                        #self.trade.backtest_buy(self.pricea)
                        self.b_.buy(self.pricea)
                        #print('backtest BUY at: ' + str(self.pricea), 'time: ' + str(sss), 'amount: ' + str(self.risk),
                        #     'fee: $' + str(round(((floor(self.risk*self.pricea)*self.leverage / self.pricea) * self.backtest_fee * self.pricea), 3)), 'pred: ' + str(predi))
                        self.out.append([1,sss,self.pricea])
                self.open = self.pricea
                self.open_time = self.act_timestamps[ind]
                self.macd_open = self.macd()[-10:]
                self.ys_open = self.ys[-10:]
                self.sma_open = self.sma()[-10:]
                self.next_brick = 1
                self.runs += 1
            elif self.cross(self.macd(), self.sma()) and self.sma()[-1] > self.macd()[-1] and not self.short:
                self.short = True
                self.long = False
                if self.runs > 0:
                    #print(f"closed trade at {self.trade.close_backtest_long(self.pricea):.8f} of profit")
                    self.b_.close(self.pricea)
                    if self.long:
                        side = 0
                    elif self.short:
                        side = 1
                    if len(self.renko_prices) > 10 and len(self.macd()) > 10 and len(self.ys) > 10:
                        # write trades to file
                        new_trade(past_bricks=self.ys_open, price_open=self.open, price_close=self.pricea, side=side, macd_open=self.macd_open, macd_close=self.macd()[-1], sma_open=self.sma_open, sma_close=self.sma()[-1], time_open=self.open_time, time_close=self.act_timestamps[ind])

                if self.end_backtest <= self.last_timestamp and not self.j_backtest and len(self.ys) > 35 and self.trade.backtest_over == True:
                    predi = pred(self.ys[-10:], self.macd()[-10:], self.sma()[-10:], self.pricea)
                    threading.Thread(target=self.trade.sell_short, args=("BITMEX", "XBT-USD", self.pricea, self.pricea+self.brick_size, predi, )).start()
                    if self.ff:
                        #self.trade.end_backtest(self.pricea)
                        self.b_.end(self.pricea)
                        print('net backtest profit: BTC ' + str(self.backtest_bal_usd) +
                              ' with $' + str(self.backtest_slippage) + ' of slippage per trade', 'max drawdown: ' + str(min(self.trades_)), 'max trade: ' + str(max(self.trades_)), 'average: ' + str(statistics.mean(self.trades_)))
                        print('proceeding to live...')
                        self.backtest_bal_usd = self.init
                        self.profit = 0
                        self.ff = False
                    act_price = float(requests.get("https://www.bitmex.com/api/v1/orderBook/L2?symbol=xbt&depth=1").json()[1]['price'])
                    print('SELL at: ' + str(self.pricea), 'act: ' + str(act_price),
                          str(datetime.datetime.now()), 'pred: ' + str(pred(self.ys[-10:], self.macd()[-10:], self.sma()[-10:], self.pricea)))

                else:
                    if ind != 1:
                        sss = self.act_timestamps[ind]
                    else:
                        sss = 'undef'
                    if len(self.macd()) > 10 and len(self.sma()) > 10 and len(self.ys) > 10:
                        if self.use_ml:
                            predi = pred(self.ys[-10:], self.macd()[-10:], self.sma()[-10:], self.pricea)
                        else:
                            predi = 1
                        self.risk = self.backtest_bal_usd * predi
                        #self.trade.backtest_sell(self.pricea)
                        self.b_.sell(self.pricea)
                        #print('backtest SELL at: ' + str(self.pricea), 'time: ' + str(sss), 'amount: ' + str(self.risk),
                              #'fee: $' + str(round(((floor(self.risk*self.pricea)*self.leverage / self.pricea) * self.backtest_fee * self.pricea), 3)), 'pred: ' + str(predi))
                        self.out.append([2,sss,self.pricea])
                self.open = self.pricea
                self.open_time = self.act_timestamps[ind]
                self.macd_open = self.macd()[-10:]
                self.ys_open = self.ys[-10:]
                self.sma_open = self.sma()[-10:]
                self.next_brick = 2
                self.runs += 1
            else:
                self.next_brick = 0
        elif self.strategy == "rsi":
            #print(ind)
            if self.rsi() is not None:
                self.pricea = self.y + self.brick_size
                rsi = []
                for i in self.rsi():
                    rsi.append(i[0])

                if rsi[-1] > 10 and rsi[-2] < 10 and not self.long:
                    if self.long or self.short:
                        self.b_.close(self.pricea, self.act_timestamps[ind])
                    self.b_.buy(self.pricea)
                    #print(f"BUY at {self.pricea} @ {self.act_timestamps[ind]}")
                    self.long = True
                    self.short = False
                    
                elif rsi[-1] < 70 and rsi[-2] > 70 and not self.short:
                    if self.long or self.short:
                        self.b_.close(self.pricea, self.act_timestamps[ind])
                    self.b_.sell(self.pricea)
                    #print(f"SELL at {self.pricea} @ {self.act_timestamps[ind]}")
                    self.short = True
                    self.long = False