Start working on strategy evaluation
This commit is contained in:
parent
d112dce5f5
commit
85a9c4fcb3
@ -75,6 +75,7 @@ def get_portfolio_value(holding, prices):
|
||||
def handle_order(date, order, holding, prices, commission, impact):
|
||||
"""Process the order."""
|
||||
symbol, order, shares = order
|
||||
assert(shares > 0) # Can only buy or sell positive amount of shares.
|
||||
adj_closing_price = prices[symbol]
|
||||
cost = shares * adj_closing_price
|
||||
# Charge commission and deduct impact penalty
|
||||
|
||||
36
strategy_evaluation/BenchmarkStrategy.py
Normal file
36
strategy_evaluation/BenchmarkStrategy.py
Normal file
@ -0,0 +1,36 @@
|
||||
import pandas as pd
|
||||
import util as ut
|
||||
import datetime as dt
|
||||
|
||||
|
||||
class BenchmarkStrategy:
|
||||
def __init__(self, verbose=False, impact=0.0, commission=0.0):
|
||||
self.verbose = verbose
|
||||
self.impact = impact
|
||||
self.commission = commission
|
||||
|
||||
def addEvidence(self, symbol=0, sd=0, ed=0, sv=0):
|
||||
"""Keep this so that API is valid."""
|
||||
pass
|
||||
|
||||
def testPolicy(self, symbol="IBM",
|
||||
sd=dt.datetime(2009, 1, 1),
|
||||
ed=dt.datetime(2010, 1, 1),
|
||||
sv=10000):
|
||||
"""Benchmark is to buy 1000 shares and hold."""
|
||||
dates = pd.date_range(sd, ed)
|
||||
prices = ut.get_data([symbol], dates) # automatically adds SPY
|
||||
|
||||
orders = pd.DataFrame(index=prices.index)
|
||||
orders["Symbol"] = symbol
|
||||
orders["Order"] = ""
|
||||
orders["Shares"] = 0
|
||||
|
||||
orders.iloc[0] = [symbol, "BUY", 1000]
|
||||
orders.iloc[-1] = [symbol, "SELL", 1000]
|
||||
orders = orders[orders["Shares"] != 0]
|
||||
|
||||
if self.verbose:
|
||||
print(type(orders)) # it better be a DataFrame!
|
||||
print(orders)
|
||||
return orders
|
||||
58
strategy_evaluation/ManualStrategy.py
Normal file
58
strategy_evaluation/ManualStrategy.py
Normal file
@ -0,0 +1,58 @@
|
||||
import datetime as dt
|
||||
import pandas as pd
|
||||
import util as ut
|
||||
|
||||
|
||||
class ManualStrategy:
|
||||
def __init__(self, verbose=False, impact=0.0, commission=0.0):
|
||||
self.verbose = verbose
|
||||
self.impact = impact
|
||||
self.commission = commission
|
||||
|
||||
# this method should create a QLearner, and train it for trading
|
||||
def addEvidence(self, symbol="IBM",
|
||||
sd=dt.datetime(2008, 1, 1),
|
||||
ed=dt.datetime(2009, 1, 1),
|
||||
sv=10000):
|
||||
|
||||
# add your code to do learning here
|
||||
|
||||
# example usage of the old backward compatible util function
|
||||
syms = [symbol]
|
||||
dates = pd.date_range(sd, ed)
|
||||
prices_all = ut.get_data(syms, dates) # automatically adds SPY
|
||||
prices = prices_all[syms] # only portfolio symbols
|
||||
# prices_SPY = prices_all['SPY'] # only SPY, for comparison later
|
||||
if self.verbose:
|
||||
print(prices)
|
||||
|
||||
# example use with new colname
|
||||
# automatically adds SPY
|
||||
volume_all = ut.get_data(syms, dates, colname="Volume")
|
||||
volume = volume_all[syms] # only portfolio symbols
|
||||
# volume_SPY = volume_all['SPY'] # only SPY, for comparison later
|
||||
if self.verbose:
|
||||
print(volume)
|
||||
|
||||
# this method should use the existing policy and test it against new data
|
||||
def testPolicy(self, symbol="IBM",
|
||||
sd=dt.datetime(2009, 1, 1),
|
||||
ed=dt.datetime(2010, 1, 1),
|
||||
sv=10000):
|
||||
dates = pd.date_range(sd, ed)
|
||||
prices = ut.get_data([symbol], dates) # automatically adds SPY
|
||||
orders = pd.DataFrame(index=prices.index)
|
||||
orders["Symbol"] = symbol
|
||||
orders["Order"] = ""
|
||||
orders["Shares"] = 0
|
||||
|
||||
# here we build a fake set of trades
|
||||
orders.iloc[0] = [symbol, "BUY", 1000]
|
||||
orders.iloc[40] = [symbol, "SELL", 1000]
|
||||
orders.iloc[41] = [symbol, "BUY", 1000]
|
||||
orders.iloc[60] = [symbol, "SELL", 2000]
|
||||
orders.iloc[61] = [symbol, "BUY", 2000]
|
||||
orders.iloc[-1] = [symbol, "SELL", 1000]
|
||||
orders = orders[orders["Shares"] != 0]
|
||||
return orders
|
||||
|
||||
@ -1,89 +1,88 @@
|
||||
"""
|
||||
Template for implementing StrategyLearner (c) 2016 Tucker Balch
|
||||
|
||||
Copyright 2018, Georgia Institute of Technology (Georgia Tech)
|
||||
Atlanta, Georgia 30332
|
||||
All Rights Reserved
|
||||
|
||||
Template code for CS 4646/7646
|
||||
|
||||
Georgia Tech asserts copyright ownership of this template and all derivative
|
||||
works, including solutions to the projects assigned in this course. Students
|
||||
and other users of this template code are advised not to share it with others
|
||||
or to make it available on publicly viewable websites including repositories
|
||||
such as github and gitlab. This copyright statement should not be removed
|
||||
or edited.
|
||||
|
||||
We do grant permission to share solutions privately with non-students such
|
||||
as potential employers. However, sharing with other current or future
|
||||
students of CS 7646 is prohibited and subject to being investigated as a
|
||||
GT honor code violation.
|
||||
|
||||
-----do not edit anything above this line---
|
||||
|
||||
Student Name: Tucker Balch (replace with your name)
|
||||
GT User ID: tb34 (replace with your User ID)
|
||||
GT ID: 900897987 (replace with your GT ID)
|
||||
"""
|
||||
|
||||
import datetime as dt
|
||||
import pandas as pd
|
||||
import util as ut
|
||||
import random
|
||||
|
||||
class StrategyLearner(object):
|
||||
|
||||
# constructor
|
||||
def __init__(self, verbose = False, impact=0.0, commission=0.0):
|
||||
self.verbose = verbose
|
||||
self.impact = impact
|
||||
self.commission = commission
|
||||
|
||||
# this method should create a QLearner, and train it for trading
|
||||
"""
|
||||
Template for implementing StrategyLearner (c) 2016 Tucker Balch
|
||||
|
||||
Copyright 2018, Georgia Institute of Technology (Georgia Tech)
|
||||
Atlanta, Georgia 30332
|
||||
All Rights Reserved
|
||||
|
||||
Template code for CS 4646/7646
|
||||
|
||||
Georgia Tech asserts copyright ownership of this template and all derivative
|
||||
works, including solutions to the projects assigned in this course. Students
|
||||
and other users of this template code are advised not to share it with others
|
||||
or to make it available on publicly viewable websites including repositories
|
||||
such as github and gitlab. This copyright statement should not be removed
|
||||
or edited.
|
||||
|
||||
We do grant permission to share solutions privately with non-students such
|
||||
as potential employers. However, sharing with other current or future
|
||||
students of CS 7646 is prohibited and subject to being investigated as a
|
||||
GT honor code violation.
|
||||
|
||||
-----do not edit anything above this line---
|
||||
|
||||
Student Name: Tucker Balch (replace with your name)
|
||||
GT User ID: tb34 (replace with your User ID)
|
||||
GT ID: 900897987 (replace with your GT ID)
|
||||
"""
|
||||
|
||||
import datetime as dt
|
||||
import pandas as pd
|
||||
import util as ut
|
||||
|
||||
class StrategyLearner(object):
|
||||
|
||||
# constructor
|
||||
def __init__(self, verbose = False, impact=0.0, commission=0.0):
|
||||
self.verbose = verbose
|
||||
self.impact = impact
|
||||
self.commission = commission
|
||||
|
||||
# this method should create a QLearner, and train it for trading
|
||||
def addEvidence(self, symbol = "IBM", \
|
||||
sd=dt.datetime(2008,1,1), \
|
||||
ed=dt.datetime(2009,1,1), \
|
||||
sv = 10000):
|
||||
|
||||
# add your code to do learning here
|
||||
|
||||
# example usage of the old backward compatible util function
|
||||
syms=[symbol]
|
||||
dates = pd.date_range(sd, ed)
|
||||
prices_all = ut.get_data(syms, dates) # automatically adds SPY
|
||||
prices = prices_all[syms] # only portfolio symbols
|
||||
prices_SPY = prices_all['SPY'] # only SPY, for comparison later
|
||||
if self.verbose: print(prices)
|
||||
|
||||
# example use with new colname
|
||||
volume_all = ut.get_data(syms, dates, colname = "Volume") # automatically adds SPY
|
||||
volume = volume_all[syms] # only portfolio symbols
|
||||
volume_SPY = volume_all['SPY'] # only SPY, for comparison later
|
||||
if self.verbose: print(volume)
|
||||
|
||||
# this method should use the existing policy and test it against new data
|
||||
sv = 10000):
|
||||
|
||||
# add your code to do learning here
|
||||
|
||||
# example usage of the old backward compatible util function
|
||||
syms=[symbol]
|
||||
dates = pd.date_range(sd, ed)
|
||||
prices_all = ut.get_data(syms, dates) # automatically adds SPY
|
||||
prices = prices_all[syms] # only portfolio symbols
|
||||
# prices_SPY = prices_all['SPY'] # only SPY, for comparison later
|
||||
if self.verbose: print(prices)
|
||||
|
||||
# example use with new colname
|
||||
volume_all = ut.get_data(syms, dates, colname = "Volume") # automatically adds SPY
|
||||
volume = volume_all[syms] # only portfolio symbols
|
||||
# volume_SPY = volume_all['SPY'] # only SPY, for comparison later
|
||||
if self.verbose: print(volume)
|
||||
|
||||
# this method should use the existing policy and test it against new data
|
||||
def testPolicy(self, symbol = "IBM", \
|
||||
sd=dt.datetime(2009,1,1), \
|
||||
ed=dt.datetime(2010,1,1), \
|
||||
sv = 10000):
|
||||
|
||||
# here we build a fake set of trades
|
||||
# your code should return the same sort of data
|
||||
dates = pd.date_range(sd, ed)
|
||||
prices_all = ut.get_data([symbol], dates) # automatically adds SPY
|
||||
trades = prices_all[[symbol,]] # only portfolio symbols
|
||||
trades_SPY = prices_all['SPY'] # only SPY, for comparison later
|
||||
trades.values[:,:] = 0 # set them all to nothing
|
||||
trades.values[0,:] = 1000 # add a BUY at the start
|
||||
trades.values[40,:] = -1000 # add a SELL
|
||||
trades.values[41,:] = 1000 # add a BUY
|
||||
trades.values[60,:] = -2000 # go short from long
|
||||
trades.values[61,:] = 2000 # go long from short
|
||||
trades.values[-1,:] = -1000 #exit on the last day
|
||||
if self.verbose: print(type(trades)) # it better be a DataFrame!
|
||||
if self.verbose: print(trades)
|
||||
if self.verbose: print(prices_all)
|
||||
return trades
|
||||
|
||||
if __name__=="__main__":
|
||||
print("One does not simply think up a strategy")
|
||||
sv = 10000):
|
||||
|
||||
# here we build a fake set of trades
|
||||
# your code should return the same sort of data
|
||||
dates = pd.date_range(sd, ed)
|
||||
prices_all = ut.get_data([symbol], dates) # automatically adds SPY
|
||||
trades = prices_all[[symbol,]] # only portfolio symbols
|
||||
# trades_SPY = prices_all['SPY'] # only SPY, for comparison later
|
||||
trades.values[:,:] = 0 # set them all to nothing
|
||||
trades.values[0,:] = 1000 # add a BUY at the start
|
||||
trades.values[40,:] = -1000 # add a SELL
|
||||
trades.values[41,:] = 1000 # add a BUY
|
||||
trades.values[60,:] = -2000 # go short from long
|
||||
trades.values[61,:] = 2000 # go long from short
|
||||
trades.values[-1,:] = -1000 #exit on the last day
|
||||
if self.verbose: print(type(trades)) # it better be a DataFrame!
|
||||
if self.verbose: print(trades)
|
||||
if self.verbose: print(prices_all)
|
||||
return trades
|
||||
|
||||
if __name__=="__main__":
|
||||
print("One does not simply think up a strategy")
|
||||
|
||||
141
strategy_evaluation/experiment1.py
Normal file
141
strategy_evaluation/experiment1.py
Normal file
@ -0,0 +1,141 @@
|
||||
import pandas as pd
|
||||
import datetime as dt
|
||||
import marketsim.marketsim as marketsim
|
||||
import indicators
|
||||
import util
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.widgets import MultiCursor
|
||||
from BenchmarkStrategy import BenchmarkStrategy
|
||||
from ManualStrategy import ManualStrategy
|
||||
|
||||
|
||||
def macd_strat(macd):
|
||||
def strat(x):
|
||||
print(x)
|
||||
|
||||
macd['macd_trigger'] = macd.rolling(2).apply(strat)
|
||||
|
||||
# for i, row in macd.iterrows():
|
||||
# if i == 0:
|
||||
# continue
|
||||
# print(row)
|
||||
# prev_macd, prev_signal, _ = row
|
||||
# cur_macd, cur_signal, _ = row
|
||||
# if cur_macd < -.5 and (prev_macd < prev_signal) \
|
||||
# and (cur_macd > cur_signal):
|
||||
# macd.iloc[i]['macd_buy_sell'] = 1
|
||||
|
||||
def experiment1():
|
||||
symbol = "JPM"
|
||||
start_value = 10000
|
||||
sd = dt.datetime(2008, 1, 1)
|
||||
ed = dt.datetime(2009, 12, 31)
|
||||
df = util.get_data([symbol], pd.date_range(sd, ed))
|
||||
df.drop(columns=["SPY"], inplace=True)
|
||||
# df = pd.DataFrame(index=df.index)
|
||||
|
||||
bs = BenchmarkStrategy()
|
||||
orders = bs.testPolicy(symbol, sd, ed, start_value)
|
||||
df["Benchmark"] = marketsim.compute_portvals(orders, start_value)
|
||||
|
||||
ms = ManualStrategy()
|
||||
orders = ms.testPolicy(symbol, sd, ed, start_value)
|
||||
df["Manual"] = marketsim.compute_portvals(orders, start_value)
|
||||
|
||||
# indicators.price_sma(df, symbol, 21)
|
||||
# sma = indicators.sma(df, symbol, [9, 21])
|
||||
# rsi = indicators.rsi(df, symbol)
|
||||
macd = indicators.macd(df, symbol).copy()
|
||||
# macd_strat(macd)
|
||||
|
||||
fig, ax = plt.subplots(2, sharex=True)
|
||||
df[symbol].plot(ax=ax[0])
|
||||
# sma.plot(ax=ax[0])
|
||||
macd.plot(ax=ax[1])
|
||||
# macd.iloc[:,0].plot(ax=ax[1])
|
||||
# rsi.plot(ax=ax[2])
|
||||
# df[["Benchmark", "Manual"]].plot(ax=ax[3])
|
||||
|
||||
# XXX: Plot where we buy and sell.
|
||||
|
||||
for a in ax: a.grid()
|
||||
multi = MultiCursor(fig.canvas, ax, color='r', lw=0.5)
|
||||
plt.show()
|
||||
|
||||
# df.plot(title="results", subplots=True)
|
||||
#sd = dt.datetime(2008, 1, 1)
|
||||
#ed = dt.datetime(2009, 12, 31)
|
||||
#df = get_data([symbol], pd.date_range(sd, ed))
|
||||
#df.drop(columns=["SPY"], inplace=True)
|
||||
# df_orig = df.copy()
|
||||
#df = indicators.normalize(df)
|
||||
#indicators.price_sma(df, symbol, 21)
|
||||
#df.plot(title="21 SMA and EMA")
|
||||
#plt.show()
|
||||
|
||||
# You may use data from other symbols (such as SPY) to inform both your
|
||||
# Manual Learner and Strategy Learner. The in-sample/development period is
|
||||
# January 1, 2008 to December 31 2009. The out-of-sample/testing period is
|
||||
# January 1, 2010 to December 31 2011.
|
||||
|
||||
|
||||
class BlittedCursor:
|
||||
"""
|
||||
A cross hair cursor using blitting for faster redraw.
|
||||
"""
|
||||
def __init__(self, ax):
|
||||
self.ax = ax
|
||||
self.background = None
|
||||
self.horizontal_line = ax.axhline(color='k', lw=0.8, ls='--')
|
||||
self.vertical_line = ax.axvline(color='k', lw=0.8, ls='--')
|
||||
# text location in axes coordinates
|
||||
self.text = ax.text(0.72, 0.9, '', transform=ax.transAxes)
|
||||
self._creating_background = False
|
||||
ax.figure.canvas.mpl_connect('draw_event', self.on_draw)
|
||||
|
||||
def on_draw(self, event):
|
||||
self.create_new_background()
|
||||
|
||||
def set_cross_hair_visible(self, visible):
|
||||
need_redraw = self.horizontal_line.get_visible() != visible
|
||||
self.horizontal_line.set_visible(visible)
|
||||
self.vertical_line.set_visible(visible)
|
||||
self.text.set_visible(visible)
|
||||
return need_redraw
|
||||
|
||||
def create_new_background(self):
|
||||
if self._creating_background:
|
||||
# discard calls triggered from within this function
|
||||
return
|
||||
self._creating_background = True
|
||||
self.set_cross_hair_visible(False)
|
||||
self.ax.figure.canvas.draw()
|
||||
self.background = self.ax.figure.canvas.copy_from_bbox(self.ax.bbox)
|
||||
self.set_cross_hair_visible(True)
|
||||
self._creating_background = False
|
||||
|
||||
def on_mouse_move(self, event):
|
||||
if self.background is None:
|
||||
self.create_new_background()
|
||||
if not event.inaxes:
|
||||
need_redraw = self.set_cross_hair_visible(False)
|
||||
if need_redraw:
|
||||
self.ax.figure.canvas.restore_region(self.background)
|
||||
self.ax.figure.canvas.blit(self.ax.bbox)
|
||||
else:
|
||||
self.set_cross_hair_visible(True)
|
||||
# update the line positions
|
||||
x, y = event.xdata, event.ydata
|
||||
self.horizontal_line.set_ydata(y)
|
||||
self.vertical_line.set_xdata(x)
|
||||
self.text.set_text('x=%1.2f, y=%1.2f' % (x, y))
|
||||
|
||||
self.ax.figure.canvas.restore_region(self.background)
|
||||
self.ax.draw_artist(self.horizontal_line)
|
||||
self.ax.draw_artist(self.vertical_line)
|
||||
self.ax.draw_artist(self.text)
|
||||
self.ax.figure.canvas.blit(self.ax.bbox)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
experiment1()
|
||||
0
strategy_evaluation/experiment2.py
Normal file
0
strategy_evaluation/experiment2.py
Normal file
120
strategy_evaluation/indicators.py
Normal file
120
strategy_evaluation/indicators.py
Normal file
@ -0,0 +1,120 @@
|
||||
import pandas as pd
|
||||
import datetime as dt
|
||||
import matplotlib.pyplot as plt
|
||||
from util import get_data
|
||||
|
||||
|
||||
def author():
|
||||
return "felixm"
|
||||
|
||||
|
||||
def normalize(timeseries):
|
||||
return timeseries / timeseries.iloc[0]
|
||||
|
||||
|
||||
def bollinger_band(df, symbol, period=20, m=2):
|
||||
boll_sma = df[symbol].rolling(period).mean()
|
||||
std = df[symbol].rolling(period).std()
|
||||
boll_up = boll_sma + m * std
|
||||
boll_lo = boll_sma - m * std
|
||||
df[f"{symbol}-Boll({period})-sma"] = boll_sma
|
||||
df[f"{symbol}-Boll({period})-up"] = boll_up
|
||||
df[f"{symbol}-Boll({period})-lo"] = boll_lo
|
||||
|
||||
|
||||
def sma(df, symbol, period):
|
||||
"""Adds SMA for one or multiple periods to df and returns SMAs"""
|
||||
if type(period) is int:
|
||||
period = [period]
|
||||
keys = []
|
||||
for p in period:
|
||||
key = f"{symbol}-sma({p})"
|
||||
df[key] = df[symbol].rolling(p).mean()
|
||||
keys.append(key)
|
||||
return df[keys]
|
||||
|
||||
|
||||
def ema(df, symbol, period):
|
||||
"""Adds a new column to the dataframe EMA(period)"""
|
||||
df[f"{symbol}-ema({period})"] = df[symbol].ewm(span=period).mean()
|
||||
|
||||
|
||||
def price_sma(df, symbol, period):
|
||||
"""Calculates SMA and adds new column price divided by SMA to the df."""
|
||||
sma = df[symbol].rolling(period).mean()
|
||||
df[f"{symbol}-price/sma({period})"] = df[symbol] / sma
|
||||
|
||||
|
||||
def rsi(df, symbol, period=14):
|
||||
"""Calculates relative strength index over given period."""
|
||||
|
||||
def rsi(x):
|
||||
pct = x.pct_change()
|
||||
avg_gain = pct[pct > 0].mean()
|
||||
avg_loss = pct[pct <= 0].abs().mean()
|
||||
rsi = 100 - (100 /
|
||||
(1 + ((avg_gain / period) /
|
||||
(avg_loss / period))))
|
||||
return rsi
|
||||
|
||||
key = f"rsi"
|
||||
# Add one to get 'period' price changes (first change is nan).
|
||||
period += 1
|
||||
df[key] = df[symbol].rolling(period).apply(rsi)
|
||||
return df[key]
|
||||
|
||||
|
||||
def macd(df, symbol):
|
||||
macd = df[symbol].ewm(span=12).mean() - df[symbol].ewm(span=26).mean()
|
||||
k1 = f"macd"
|
||||
k2 = k1 + "-signal"
|
||||
df[k1] = macd
|
||||
df[k2] = macd.rolling(9).mean()
|
||||
return df[[k1, k2]]
|
||||
|
||||
|
||||
def price_delta(df, symbol, period=1):
|
||||
"""Calculate delta between previous day and today."""
|
||||
df[f"{symbol}-diff({period})"] = df[symbol].diff(periods=period)
|
||||
|
||||
|
||||
def test_indicators():
|
||||
symbol = "JPM"
|
||||
|
||||
sd = dt.datetime(2008, 1, 1)
|
||||
ed = dt.datetime(2009, 12, 31)
|
||||
df = get_data([symbol], pd.date_range(sd, ed))
|
||||
df.drop(columns=["SPY"], inplace=True)
|
||||
df_orig = df.copy()
|
||||
# df = normalize(df)
|
||||
|
||||
sma(df, symbol, 21)
|
||||
ema(df, symbol, 21)
|
||||
df.plot(title="21 SMA and EMA")
|
||||
plt.savefig('figure_1.png')
|
||||
|
||||
df = df_orig.copy()
|
||||
sma(df, symbol, 8)
|
||||
price_sma(df, symbol, 8)
|
||||
df.plot(title="SMA and price / SMA", subplots=True)
|
||||
plt.savefig('figure_2.png')
|
||||
|
||||
df = df_orig.copy()
|
||||
bollinger_band(df, symbol)
|
||||
df.plot(title="Bollinger Band")
|
||||
plt.savefig('figure_3.png')
|
||||
|
||||
df = df_orig.copy()
|
||||
rsi(df, symbol)
|
||||
fig, axes = plt.subplots(nrows=2, sharex=True)
|
||||
df[symbol].plot(ax=axes[0], title="JPM price action")
|
||||
df["JPM-rsi(14)"].plot(ax=axes[1], title="RSI")
|
||||
plt.savefig('figure_4.png')
|
||||
|
||||
df = df_orig.copy()
|
||||
macd(df, symbol)
|
||||
fig, axes = plt.subplots(nrows=2, sharex=True)
|
||||
df[symbol].plot(ax=axes[0], title="JPM price action")
|
||||
df[["JPM-macd", "JPM-macd-signal"]].plot(ax=axes[1])
|
||||
plt.savefig('figure_5.png')
|
||||
|
||||
0
strategy_evaluation/testproject.py
Normal file
0
strategy_evaluation/testproject.py
Normal file
Loading…
Reference in New Issue
Block a user