#!/usr/bin/env bash

###############################################################################
# Copyright (c) 2017-25, Lawrence Livermore National Security, LLC
# and RAJA project contributors. See the RAJAPerf/LICENSE file for details.
#
# SPDX-License-Identifier: (BSD-3-Clause)
###############################################################################

if [[ $# -lt 2 ]]; then
  echo
  echo "You must pass 2 or more arguments to the script (in this order): "
  echo "   1) cray-mpich compiler version number"
  echo "   1) HIP compiler version number"
  echo "   2) HIP compute architecture"
  echo "   3...) optional arguments to cmake"
  echo
  echo "For example: "
  echo "    toss4_cray-mpich_amdclang_asan.sh 8.1.14 4.1.0 gfx906"
  exit
fi

MPI_VER=$1
COMP_VER=$2
COMP_ARCH=$3
shift 3

HOSTCONFIG="hip_3_X"

if [[ ${COMP_VER} == 4.* ]]
then
##HIP_CLANG_FLAGS="-mllvm -amdgpu-fixed-function-abi=1"
  HOSTCONFIG="hip_4_link_X"
elif [[ ${COMP_VER} == 3.* ]]
then
  HOSTCONFIG="hip_3_X"
else
  echo "Unknown hip version, using ${HOSTCONFIG} host-config"
fi

# if [[ ${COMP_ARCH} == gfx90a ]]
# then
  # note that unsafe atomics require use of coarse grain memory
##HIP_CLANG_FLAGS="-munsafe-fp-atomics"
# fi

BUILD_SUFFIX=lc_toss4-cray-mpich-${MPI_VER}-amdclang-${COMP_VER}-${COMP_ARCH}-asan
RAJA_HOSTCONFIG=../tpl/RAJA/host-configs/lc-builds/toss4/${HOSTCONFIG}.cmake

echo
echo "Creating build directory ${BUILD_SUFFIX} and generating configuration in it"
echo "Configuration extra arguments:"
echo "   $@"
echo
echo "To get cmake to work you may have to configure with"
echo "   -DHIP_PLATFORM=amd"
echo
echo "To use fp64 HW atomics you must configure with these options when using gfx90a and hip >= 5.2"
echo "   -DCMAKE_CXX_FLAGS=\"-munsafe-fp-atomics\""
echo
echo "To work around some issues where *_FUSED kernels crash add these options"
echo "   -DCMAKE_CXX_FLAGS=\"-fgpu-rdc\""
echo "   -DCMAKE_EXE_LINKER_FLAGS=\"-fgpu-rdc\""
echo
echo "To work around some issues where *_FUSED kernels perform poorly use this environment variable"
echo "   env HSA_SCRATCH_SINGLE_LIMIT=4000000000"
echo
echo "To work around some issues where the build fails with a weird error about max or fmax add these options"
echo "   -DCMAKE_CXX_FLAGS=\"--hip-version={hip_version:ex=6.1.2}\""
echo "   -DCMAKE_EXE_LINKER_FLAGS=\"--hip-version={hip_version:ex=6.1.2}\""
echo



rm -rf build_${BUILD_SUFFIX} >/dev/null
mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX}


module load cmake/3.23.1

# unload rocm to avoid configuration problems where the loaded rocm and COMP_VER
# are inconsistent causing the rocprim from the module to be used unexpectedly
module unload rocm rocmcc

if [[ "${COMP_VER}" == *-magic ]]; then
  ROCM_PATH="/usr/tce/packages/rocmcc/rocmcc-${COMP_VER}"
  MPI_ROCM_PATH="/usr/tce/packages/cray-mpich/cray-mpich-${MPI_VER}-rocmcc-${COMP_VER}"
else
  ROCM_PATH="/opt/rocm-${COMP_VER}"
  MPI_ROCM_PATH=/usr/tce/packages/cray-mpich-tce/cray-mpich-${MPI_VER}-rocmcc-${COMP_VER}
fi

COMP_HIP_VER="${COMP_VER%-magic}"
COMP_CLANG_MAJOR_VER="$(ls /opt/rocm-${COMP_HIP_VER}/lib/llvm/lib/clang/)"

cmake \
  -DCMAKE_BUILD_TYPE=Release \
  -DMPI_C_COMPILER="${MPI_ROCM_PATH}/bin/mpiamdclang" \
  -DMPI_CXX_COMPILER="${MPI_ROCM_PATH}/bin/mpiamdclang++" \
  -DCMAKE_PREFIX_PATH="${ROCM_PATH}/lib/cmake" \
  -DHIP_PLATFORM=amd \
  -DROCM_ROOT_DIR="${ROCM_PATH}" \
  -DHIP_ROOT_DIR="${ROCM_PATH}/hip" \
  -DHIP_PATH="${ROCM_PATH}/llvm/bin" \
  -DCMAKE_C_COMPILER="${ROCM_PATH}/llvm/bin/amdclang" \
  -DCMAKE_CXX_COMPILER="${ROCM_PATH}/llvm/bin/amdclang++" \
  -DCMAKE_HIP_ARCHITECTURES="${COMP_ARCH}:xnack+" \
  -DGPU_TARGETS="${COMP_ARCH}:xnack+" \
  -DAMDGPU_TARGETS="${COMP_ARCH}:xnack+" \
  -DCMAKE_C_FLAGS="-fsanitize=address -shared-libsan" \
  -DCMAKE_CXX_FLAGS="-fsanitize=address -shared-libsan" \
  -DCMAKE_HIP_FLAGS="-fsanitize=address -shared-libsan -fgpu-rdc --hip-version=${COMP_HIP_VER}" \
  -DCMAKE_EXE_LINKER_FLAGS="-L/opt/rocm-${COMP_HIP_VER}/lib/asan/ -L/opt/rocm-${COMP_HIP_VER}/llvm/lib/asan -Wl,-rpath,/opt/rocm-${COMP_HIP_VER}/lib/asan/:/opt/rocm-${COMP_HIP_VER}/llvm/lib/asan:/opt/rocm-${COMP_HIP_VER}/lib/llvm/lib/clang/${COMP_CLANG_MAJOR_VER}/lib/linux -fgpu-rdc --hip-version=${COMP_HIP_VER}" \
  -DBLT_CXX_STD=c++17 \
  -C ${RAJA_HOSTCONFIG} \
  -DENABLE_MPI=ON \
  -DENABLE_HIP=ON \
  -DENABLE_OPENMP=OFF \
  -DENABLE_CUDA=OFF \
  -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \
  "$@" \
  ..

echo
echo "***********************************************************************"
echo
echo "cd into directory build_${BUILD_SUFFIX} and run make to build RAJA"
echo
echo "  Please note that you have to have a consistent build environment"
echo "  when you make RAJA as cmake may reconfigure; unload the rocm module"
echo "  or load the appropriate rocm module (${COMP_VER}) when building."
echo
echo "    module unload rocm"
echo "    srun -n1 make"
echo
echo "  Please note that rocm requires libpgmath.so from rocm/llvm to run."
echo "  Until this is handled transparently in the build system you may add "
echo "  rocm/llvm to your LD_LIBRARY_PATH."
echo
echo "    export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/rocm-${COMP_VER}/llvm/lib"
echo
echo "***********************************************************************"
