run.sh
#!/bin/bash

(($#<4)) && echo "Usage $0 remove_path h5_output_path fs_name fs_ugi [vit_sf_path]" && exit -1
rank=${OMPI_COMM_WORLD_RANK:-0}
worldsize=${OMPI_COMM_WORLD_SIZE:-1}
echo "Running : ${rank}/${worldsize}"

remote=$1
remote_out=$2
afs_name=$3
afs_ugi=$4
vit_sf_path=$5

hdfs="hadoop fs -D hadoop.job.ugi=${afs_ugi} -D fs.default.name=${afs_name}"

$hdfs -test -e $remote_out
if [ $? -ne 0 ];then
	if [ $rank -eq 0 ];then
	$hdfs -mkdir $remote_out
	fi	
fi


echo "merging" ###############################################################

if [ -d merged_out ]; then
     rm -rf merged_out
fi
export PYTHONPATH=./ernie4:./utils:./third_party/ernie-core/src:./third_party/ernie-core/PaddleNLP:./third_party/data_processor:$PYTHONPATH

python3 script/supermer/supermer.py \
  --input $remote \
  --output merged_out  \
  --mp-degree 8 \
  --dp-moe-degree 0 \
  --safe-tensors  \
  --torch \
  --layer 56


if [ $? -ne 0 ]; then
    exit -1
fi

if [ ! -z "$vit_sf_path" ]; then
    python3 script/supermer/inject_vit.py merged_out ${vit_sf_path} ${rank}
fi

echo "uploading $remote_out" ###############################################################
for i in `ls merged_out`;do
   $hdfs -put merged_out/$i $remote_out &
done
wait

