#!/bin/bash
# Copyright 2021 Alibaba Group Holding Limited. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================

set -eo pipefail

export TF_CUDA_COMPUTE_CAPABILITIES="7.5,8.0"
export TF_NEED_TENSORRT=0
export TF_NEED_ROCM=0
export TF_NEED_COMPUTECPP=0
export TF_NEED_OPENCL=0
export TF_NEED_OPENCL_SYCL=0
export TF_ENABLE_XLA=1
export TF_NEED_MPI=0

DESTDIR=$1

cd $DESTDIR
yes "" | bash ./configure || true

set -x

TF_ALL_TARGETS='//tensorflow/python/...'

# Disable failed UT cases temporarily.
export TF_BUILD_BAZEL_TARGET="$TF_ALL_TARGETS "\
"-//tensorflow/python/autograph/pyct:inspect_utils_test_par "\
"-//tensorflow/python/autograph/pyct:compiler_test "\
"-//tensorflow/python/autograph/pyct:cfg_test "\
"-//tensorflow/python/autograph/pyct:ast_util_test "\
"-//tensorflow/python/data/experimental/kernel_tests:prefetch_with_slack_test "\
"-//tensorflow/python:collective_ops_gpu_test_gpu "\
"-//tensorflow/python:collective_ops_gpu_test "\
"-//tensorflow/python/debug:local_cli_wrapper_test "\
"-//tensorflow/python/debug:debugger_cli_common_test "\
"-//tensorflow/python/debug:framework_test "\
"-//tensorflow/python/debug:source_remote_test "\
"-//tensorflow/python/debug:session_debug_grpc_test "\
"-//tensorflow/python/debug:grpc_large_data_test "\
"-//tensorflow/python/debug:dist_session_debug_grpc_test "\
"-//tensorflow/python/debug:dist_session_debug_grpc_test_gpu "\
"-//tensorflow/python/debug:dumping_wrapper_test "\
"-//tensorflow/python:deprecation_test "\
"-//tensorflow/python/distribute:values_test "\
"-//tensorflow/python/distribute:values_test_gpu "\
"-//tensorflow/python/distribute:parameter_server_strategy_test "\
"-//tensorflow/python/distribute:moving_averages_test_gpu "\
"-//tensorflow/python/distribute:moving_averages_test "\
"-//tensorflow/python/distribute:metrics_v1_test "\
"-//tensorflow/python/distribute:metrics_v1_test_gpu "\
"-//tensorflow/python/distribute:keras_metrics_test "\
"-//tensorflow/python/distribute:keras_metrics_test_gpu "\
"-//tensorflow/python/eager:remote_test "\
"-//tensorflow/python/eager:remote_test_gpu "\
"-//tensorflow/python/eager:def_function_xla_test_gpu "\
"-//tensorflow/python:embedding_variable_ops_test "\
"-//tensorflow/python:group_embedding_lookup_ops_test "\
"-//tensorflow/python/keras/distribute:multi_worker_fault_tolerance_test "\
"-//tensorflow/python/keras:callbacks_test "\
"-//tensorflow/python/keras:core_test "\
"-//tensorflow/python/keras/distribute:keras_dnn_correctness_test "\
"-//tensorflow/python/keras/distribute:keras_image_model_correctness_test "\
"-//tensorflow/python/keras/distribute:keras_lstm_model_correctness_test "\
"-//tensorflow/python/keras/distribute:keras_embedding_model_correctness_test_gpu "\
"-//tensorflow/python/keras/distribute:keras_lstm_model_correctness_test_gpu "\
"-//tensorflow/python/keras/distribute:keras_image_model_correctness_test_gpu "\
"-//tensorflow/python/keras/distribute:keras_dnn_correctness_test_gpu "\
"-//tensorflow/python/keras/distribute:multi_worker_test "\
"-//tensorflow/python/keras/distribute:keras_utils_test_gpu "\
"-//tensorflow/python/keras/distribute:keras_embedding_model_correctness_test "\
"-//tensorflow/python/keras/distribute:distribute_strategy_test_gpu "\
"-//tensorflow/python/keras/distribute:distribute_strategy_test "\
"-//tensorflow/python/keras/distribute:keras_utils_test "\
"-//tensorflow/python/keras/premade:wide_deep_test "\
"-//tensorflow/python/keras:simplernn_test "\
"-//tensorflow/python/keras:lstm_test "\
"-//tensorflow/python/keras:hdf5_format_test "\
"-//tensorflow/python/kernel_tests:unique_op_test "\
"-//tensorflow/python/kernel_tests:bias_op_test_gpu "\
"-//tensorflow/python/kernel_tests:bias_op_test "\
"-//tensorflow/python/kernel_tests:conv_ops_3d_test_gpu "\
"-//tensorflow/python/kernel_tests/signal:fft_ops_test "\
"-//tensorflow/python/kernel_tests:lu_op_test_gpu "\
"-//tensorflow/python/kernel_tests:conv_ops_3d_test "\
"-//tensorflow/python:layers_core_test "\
"-//tensorflow/python:optimizer_test "\
"-//tensorflow/python/profiler:model_analyzer_test_gpu "\
"-//tensorflow/python/profiler:model_analyzer_test "\
"-//tensorflow/python/tools/api/generator:output_init_files_test "\
"-//tensorflow/python/tpu:datasets_test "\
"-//tensorflow/python/training/tracking:util_xla_test_gpu "\
"-//tensorflow/python/kernel_tests:sparse_conditional_accumulator_test "\
"-//tensorflow/python/debug:session_debug_grpc_test_gpu "\
"-//tensorflow/python/debug:grpc_large_data_test_gpu "\
"-//tensorflow/python/keras:training_test "\
"-//tensorflow/python/keras:convolutional_test "\
"-//tensorflow/python/keras:lstm_v2_test "\
"-//tensorflow/python/keras:lstm_v2_test_gpu "\
"-//tensorflow/python/kernel_tests:normalize_op_test "\
"-//tensorflow/python/kernel_tests:svd_op_test "

for i in $(seq 1 3); do
    [ $i -gt 1 ] && echo "WARNING: cmd execution failed, will retry in $((i-1)) times later" && sleep 2
    ret=0
    bazel test -c opt --config=cuda --verbose_failures --test_env='NVIDIA_TF32_OVERRIDE=0' \
    --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute --config=opt \
    --test_timeout="300,450,1200,3600" --local_test_jobs=20 --test_output=errors \
    -- $TF_BUILD_BAZEL_TARGET && break || ret=$?
done

exit $ret
