local bench_utils = require 'bench_utils'

-- Output file
local output_path = 'baseline_st.md'
local output = io.open(output_path, "w")


output:write("## Results\n")
output:write("This file contains the results for the tests on the GTSRB dataset.\n\n")
output:write("| Command line arguments | accuracy on validation set |\n")
output:write("| --------------------- | ----- |\n")

print("Running best result with full dataset.")
local params = {}
table.insert(params, {'-n -1 --st --locnet 100,200,100', {true}})
bench_utils.run_test(params, output)

print("Running main tests.")
params = {}
table.insert(params, {'--val', {true}}) -- run tests on the validation set
table.insert(params, {'--no_lnorm', {false, true}})
table.insert(params, {'--no_cnorm', {false, true}})
table.insert(params, {'--st', {true}})
table.insert(params, {'--sca --tra', {false, true}})
table.insert(params, {'--locnet ', {false,
                                   "20,20,20",
                                   "60,60,60",
                                   "100,100,100",
                                   "50,100,50",
                                   "100,200,100",
                                   "100,100,50",
                                   "150,150,100"}})
bench_utils.run_test(params, output)

print("Running extra tests with bigger localization networks.")
params = {}
table.insert(params, {'--val', {true}}) -- run tests on the validation set
table.insert(params, {'--st', {true}})
table.insert(params, {'--sca --tra', {false, true}})
table.insert(params, {'--locnet ', {"141,141,141",
                                    "150,150,150",
                                    "200,200,200",
                                    "250,250,250",
                                    "150,250,150",
                                    "200,300,200"}})
bench_utils.run_test(params, output)

print("Running tests with multiple spatial transformers")
params = {}
table.insert(params, {'--val', {true}}) -- run tests on the validation set
table.insert(params, {'--st', {true}})
table.insert(params, {'--locnet ', {'""',
                                    "150,150,150",
                                    "150,250,150"}})
table.insert(params, {'--locnet2 ', {"150,150,150",
                                     "150,250,150",
                                     "75,75,75",
                                     "75,125,75"}})
bench_utils.run_test(params, output)

print("Running test with the idsia network.")
params = {}
table.insert(params, {'--val', {true}}) -- run tests on the validation set
table.insert(params, {'--net ', {"idsia_net.lua"}})
table.insert(params, {'--cnn ', {"100,150,250,300", "150,200,300,350"}})
table.insert(params, {'--st', {true}})
table.insert(params, {'--sca --tra', {false, true}})
table.insert(params, {'--locnet ', {"100,200,100",
                                    "150,250,150"}})
bench_utils.run_test(params, output)

print("Running tests with multiple spatial transformers on idsia network")
params = {}
table.insert(params, {'--val', {true}}) -- run tests on the validation set
table.insert(params, {'--st', {true}})
table.insert(params, {'--net ', {"idsia_net.lua"}})
table.insert(params, {'--cnn ', {"150,200,300,350",
                                 "200,250,350,400"}})
table.insert(params, {'--locnet ', {'""',
                                    "150,150,150",
                                    "200,300,200"}})
table.insert(params, {'--locnet2 ', {false,
                                     "150,150,150",
                                     "75,75,75"}})
table.insert(params, {'--locnet3 ', {false,
                                     "150,150,150",
                                     "75,75,75"}})
bench_utils.run_test(params, output)

output:write('\n\n')
output:write('This file is generated by the `baseline_st.lua` script.\n')
output:close()
