#!/bin/bash
# Linted by https://www.shellcheck.net/

# This script aims to harvest .json files generated by logrotate,
# compress and upload them to S3, MinIO or NAS.

msg_type=$1 # trade, l2_event, etc.

if [[ -z "${DATA_DIR}" ]]; then
  echo "DATA_DIR must be set" >&2
  exit 1
fi

if [[ -z "${DEST_DIR}" && -z "${AWS_S3_DIR}" && -z "${MINIO_DIR}" && -z "${REDIS_URL}" ]]; then
  echo "At least one of DEST_DIR, AWS_S3_DIR, MINIO_DIR or REDIS_URL must be set" >&2
  exit 1
fi

num_destinations=0

if [[ -n "${AWS_S3_DIR}" ]]; then
  if [[ -z "${AWS_ACCESS_KEY_ID}"  ||  -z "${AWS_SECRET_ACCESS_KEY}" ]]; then
    echo "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set" >&2
  exit 1
  fi
  num_destinations=$((num_destinations+1))
fi

if [[ -n "${MINIO_DIR}" ]]; then
  if [[ -z "${MINIO_ACCESS_KEY_ID}"  ||  -z "${MINIO_SECRET_ACCESS_KEY}" ||  -z "${MINIO_ENDPOINT_URL}" ]]; then
    echo "MINIO_ACCESS_KEY_ID, MINIO_SECRET_ACCESS_KEY and MINIO_ENDPOINT_URL must be set" >&2
  exit 1
  fi
  num_destinations=$((num_destinations+1))
fi

if [[ -n "${DEST_DIR}" ]]; then
  mkdir -p "$DEST_DIR/$msg_type"
  num_destinations=$((num_destinations+1))
fi

# Infinite while loop
while :
do
  # Upload .json.gz and .json.xz files older than 1 minute
  if [[ -n "${AWS_S3_DIR}" ]]; then
    if [ $num_destinations -gt 1 ]; then sub_command="copy"; else sub_command="move"; fi
    if rclone --s3-region "${AWS_REGION:-us-east-1}" --immutable --contimeout=1s --retries 1 --low-level-retries 1 $sub_command "$DATA_DIR/$msg_type" "$AWS_S3_DIR/$msg_type" --include '*.json.gz' --include '*.json.xz' --min-age 1m --no-traverse --transfers=8; then
      num_destinations=$((num_destinations-1))
    fi
  fi
  if [[ -n "${MINIO_DIR}" ]]; then
    if [ $num_destinations -gt 1 ]; then sub_command="copy"; else sub_command="move"; fi
    if rclone --s3-access-key-id "$MINIO_ACCESS_KEY_ID" --s3-secret-access-key "$MINIO_SECRET_ACCESS_KEY" --s3-endpoint "$MINIO_ENDPOINT_URL" --immutable --contimeout=1s --retries 1 --low-level-retries 1 $sub_command "$DATA_DIR/$msg_type" "$MINIO_DIR/$msg_type" --include '*.json.gz' --include '*.json.xz' --min-age 1m --no-traverse --transfers=8; then
      num_destinations=$((num_destinations-1))
    fi
  fi
  if [[ -n "${DEST_DIR}" ]]; then
    if [ $num_destinations -gt 1 ]; then sub_command="copy"; else sub_command="move"; fi
    if rclone $sub_command "$DATA_DIR/$msg_type" "$DEST_DIR/$msg_type" --include '*.json.gz' --include '*.json.xz' --min-age 1m --no-traverse --transfers=8; then
      num_destinations=$((num_destinations-1))
    fi
  fi

  sleep 3
done
