#!/bin/bash

##
# That Shell script de-duplicates combined POR (point of reference) entries
# when Geonames has got the full details for every location type of the
# combined entry. For instance, a 'CA' (city and airport) entry may be split
# into a city and an airport when Geonames knows about those two POR
# individually:
# they can therefore be distinguished. The input data files are:
# - ../OPTD/optd_por_best_known_so_far.csv
# - dump_from_geonames.csv
#
# The generated file is:
# => optd_por_to_be_split.csv
#

##
# Temporary path
TMP_DIR="/tmp/por"

##
# Path of the executable: set it to empty when this is the current directory.
EXEC_PATH=`dirname $0`
# Trick to get the actual full-path
pushd ${EXEC_PATH} > /dev/null
EXEC_FULL_PATH=`popd`
popd > /dev/null
EXEC_FULL_PATH=`echo ${EXEC_FULL_PATH} | sed -e 's|~|'${HOME}'|'`
#
CURRENT_DIR=`pwd`
if [ ${CURRENT_DIR} -ef ${EXEC_PATH} ]
then
	EXEC_PATH="."
	TMP_DIR="."
fi
EXEC_PATH="${EXEC_PATH}/"
TMP_DIR="${TMP_DIR}/"

if [ ! -d ${TMP_DIR} -o ! -w ${TMP_DIR} ]
then
	\mkdir -p ${TMP_DIR}
fi

##
# Log level
LOG_LEVEL=3

##
# OpenTravelData directory
OPTD_DIR=`dirname ${EXEC_FULL_PATH}`
OPTD_DIR="${OPTD_DIR}/"

##
# OPTD sub-directories
DATA_DIR=${OPTD_DIR}opentraveldata/
TOOLS_DIR=${OPTD_DIR}tools/

##
# Log level
LOG_LEVEL=2

##
# Initial
OPTD_POR_FILENAME=optd_por_best_known_so_far.csv
OPTD_POR_FILE=${DATA_DIR}${OPTD_POR_FILENAME}

##
# Geonames (to be found, as temporary files, within the ../tools directory)
GEONAME_RAW_FILENAME=dump_from_geonames.csv
#
GEONAME_RAW_FILE=${TOOLS_DIR}${GEONAME_RAW_FILENAME}
# Geonames with primary key (generated by the
# ../tools/prepare_geonames_dump_file.sh script)
GEONAME_WPK_FILENAME=wpk_${GEONAME_RAW_FILENAME}
#
GEONAME_WPK_FILE=${TOOLS_DIR}${GEONAME_WPK_FILENAME}
# Sorted and cut (also generated by the above script)
GEONAME_SORTED_FILENAME=sorted_${GEONAME_WPK_FILENAME}
GEONAME_CUT_SORTED_FILENAME=cut_${GEONAME_SORTED_FILENAME}
#
GEONAME_SORTED_FILE=${TOOLS_DIR}${GEONAME_SORTED_FILENAME}
GEONAME_CUT_SORTED_FILE=${TOOLS_DIR}${GEONAME_CUT_SORTED_FILENAME}

##
# Target (generated files)
OPTD_POR_TOBESPLIT_FILENAME=optd_por_to_be_split.csv
OPTD_POR_NEW_FILENAME=new_${OPTD_POR_FILENAME}
#
OPTD_POR_TOBESPLIT_FILE=${TMP_DIR}${OPTD_POR_TOBESPLIT_FILENAME}
OPTD_POR_NEW_FILE=${TMP_DIR}${OPTD_POR_NEW_FILENAME}


##
# Sanity check
if [ ! -d ${TOOLS_DIR} ]
then
	echo
	echo "[$0:$LINENO] The tools/ sub-directory ('${TOOLS_DIR}') does not exist or is not accessible."
	echo "Check that your Git clone of the OpenTravelData is complete."
	echo
	exit -1
fi
if [ ! -f ${TOOLS_DIR}prepare_geonames_dump_file.sh ]
then
	echo
	echo "[$0:$LINENO] The Geonames dump file preparation script ('${TOOLS_DIR}prepare_geonames_dump_file.sh') does not exist or is not accessible."
	echo "Check that your Git clone of the OpenTravelData is complete."
	echo
	exit -1
fi


##
# Usage helper
#
if [ "$1" = "-h" -o "$1" = "--help" ]
then
	echo
	echo "That script generates a list of de-duplicated POR entries for which Geonames has the full details"
	echo
	echo "Usage: $0 [<log level (0: quiet; 5: verbose)>]"
	echo " - Default log level (from 0 to 5): ${LOG_LEVEL}"
	echo
	echo "* Input data files"
	echo "------------------"
	echo " - OPTD-maintained file of best known coordinates: '${OPTD_POR_FILE}'"
	echo " - Geonames data dump file: '${GEONAME_RAW_FILE}'"
	echo
	echo "* Output data file"
	echo "------------------"
	echo " - OPTD-maintained public file of POR: '${OPTD_POR_TOBESPLIT_FILE}'"
	echo
	exit
fi


##
# Cleaning
#
if [ "$1" = "--clean" ]
then
	\rm -f ${GEONAME_WPK_FILE} ${GEONAME_SORTED_FILE} ${GEONAME_CUT_SORTED_FILE}
	\rm -f ${OPTD_POR_TOBESPLIT_FILE} ${OPTD_POR_NEW_FILE}
	bash ${TOOLS_DIR}prepare_geonames_dump_file.sh --clean || exit -1
	exit
fi


##
# Log level
#
if [ "$1" != "" ]
then
	LOG_LEVEL="$1"
fi


##
# Preparation
#
bash ${TOOLS_DIR}prepare_geonames_dump_file.sh ${OPTD_DIR} ${LOG_LEVEL} || exit -1

##
#
if [ ! -f ${GEONAME_SORTED_FILE} ]
then
	echo
	echo "[$0:$LINENO] The '${GEONAME_SORTED_FILE}' file does not exist."
	echo
	exit -1
fi


##
# Split the combined POR entries for which Geonames has got the details
# for the individual entries.
SPLITTER=one-off/optd_por_splitter.awk
awk -F'^' -v log_level=${LOG_LEVEL} -f ${SPLITTER} \
	${OPTD_POR_FILE} ${GEONAME_RAW_FILE} > ${OPTD_POR_TOBESPLIT_FILE}

##
# Sanity check #1
# Display the number of occurences for every location types
#  1.1. Extract the primary keys (e.g., IEV-A or IEV-C)
#  1.2. Extract the location type (e.g., A or C)
LOC_TYPE_LIST=`
awk -F'^' '/^([A-Z]{3})-/ {print ($1)}' ${OPTD_POR_TOBESPLIT_FILE} \
	| awk -F'-' '{print ($2)}' | sort | uniq -c`

##
# Sanity check #2
# By construction, the generated file should have only de-duplicated entries.
# That is, every IATA code should correspond to at least two POR entries.
#  2.1. Extract the primary keys (e.g., IEV-A or IEV-C)
#  2.2. Extract the IATA codes (e.g., IEV)
#  2.3. Count the occurences of every IATA code (e.g., 2 IEV)
#  2.4. Check the IATA codes having just a single entry
#
SINGLE_POR_LIST=`
awk -F'^' '/^([A-Z]{3})-/ {print ($1)}' ${OPTD_POR_TOBESPLIT_FILE} \
	| awk -F'-' '{print ($1)}' | sort | uniq -c \
	| awk '{if ($1 != "2") {print ($0)}}'`
if [ "${SINGLE_POR_LIST}" != "" ]
then
	echo
	echo "[$0:$LINENO] List of POR having a single entry for a given IATA code:"
	echo "[$0:$LINENO] ${SINGLE_POR_LIST}"
	echo
	exit -1
fi

##
# Merge the file of best known coordinates (${OPTD_POR_FILE}, aka
# optd_por_best_known_so_far.csv) with the newly created splitted one
# (${OPTD_POR_TOBESPLIT_FILE}, aka optd_por_to_be_split.csv).
# They both have got the same format.
MERGER=optd_por_merger.awk
if [ -f ${OPTD_POR_TOBESPLIT_FILE} ]
then
	awk -F'^' -v log_level=${LOG_LEVEL} -f ${MERGER} \
		${OPTD_POR_TOBESPLIT_FILE} ${OPTD_POR_FILE} > ${OPTD_POR_NEW_FILE}
fi


##
# Reporting
#
echo
echo "Reporting Step"
echo "--------------"
echo
echo "wc -l ${OPTD_POR_FILE}"
if [ -f ${OPTD_POR_TOBESPLIT_FILE} ]
then
	NB_LINES_OPTD_POR=`wc -l ${OPTD_POR_FILE} | cut -d' ' -f1`
	NB_LINES_OPTD_TOBESPLIT=`wc -l ${OPTD_POR_TOBESPLIT_FILE} | cut -d' ' -f1`
	NB_LINES_OPTD_NEW=`wc -l ${OPTD_POR_NEW_FILE} | cut -d' ' -f1`
	NB_EXP_LINES_OPTD_NEW=`echo ${NB_LINES_OPTD_TOBESPLIT}/2 + ${NB_LINES_OPTD_POR} + 1 | bc 2> /dev/null`
	echo
	echo "See:"
	echo "  + '${OPTD_POR_FILE}' file, which contains ${NB_LINES_OPTD_POR} lines"
	echo "  + '${OPTD_POR_TOBESPLIT_FILE}' file, which contains ${NB_LINES_OPTD_TOBESPLIT} lines"
	echo "  + '${OPTD_POR_NEW_FILE}' file, which contains ${NB_LINES_OPTD_NEW} lines."
	if [ ${NB_EXP_LINES_OPTD_NEW} -eq ${NB_LINES_OPTD_NEW} ]
	then
		echo "    Hence, it is equal to the expected number of lines, which is ${NB_EXP_LINES_OPTD_NEW} (= ${NB_LINES_OPTD_TOBESPLIT}/2 + ${NB_LINES_OPTD_POR} + 1)"
	else
		echo "    [WARNING] Hence, it is not equal to the expected number of lines, which is ${NB_EXP_LINES_OPTD_NEW} (= ${NB_LINES_OPTD_TOBESPLIT}/2 + ${NB_LINES_OPTD_POR} + 1)"
	fi
	echo "head -3 ${OPTD_POR_FILE} ${OPTD_POR_TOBESPLIT_FILE} ${OPTD_POR_NEW_FILE}"
	echo "grep \"^NCE\" ${OPTD_POR_FILE} ${OPTD_POR_TOBESPLIT_FILE} ${OPTD_POR_NEW_FILE}"
	echo
	echo "List of location types for the de-duplicated POR entries:"
	echo "${LOC_TYPE_LIST}"
	echo
	echo "Next steps:"
	echo "-----------"
	echo "First, there should be no warning from the process above."
	echo "Then, double check that the generated file is comparable to the original one:"
	echo "diff -c ${OPTD_POR_FILE} ${OPTD_POR_NEW_FILE} | less"
	echo "Then, replace the original file and tell Git about it:"
	echo "cp ${OPTD_POR_NEW_FILE} ${OPTD_POR_FILE}"
	echo "git add ${OPTD_POR_FILE}"
	echo "git diff --cached ${OPTD_POR_FILE}"
	echo "git ci -m \"[RefData][OPTD] De-duplicated the POR (points of reference) entries of the OPTD-maintained file of best known coordinates.\" ${OPTD_POR_FILE}"
	echo
	echo "Finally, do some cleaning:"
	echo "$0 --clean"
	echo
fi
echo
