#! /bin/bash
# Copyright 2019 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail


function log_info() {
  echo >&2 "$(date "+%H:%M:%S") INFO: $1"
}

function extract_preprocessor() {
  log_info "Extracting preprocessing FST..."

  # Morphophonemics FST (first level of analysis) assumes its input to be all
  # lowercase and in Unicode codepoints, but not UTF-8 bytes (see below
  # comments of build_morphotactics for further details on UTF8_TO_UNICODE
  # conversion). This initial transformation on input words are done by the
  # preprocessing FST.
  $OPENFST_DIR/farextract $SRC_DIR/symbols/case_conversion.far
  $OPENFST_DIR/farextract $SRC_DIR/symbols/unicode.far

  $OPENFST_DIR/fstcompose  \
      UTF8_TO_UNICODE      \
      TO_LOWER             \
      $PREPROCESSOR_PATH

  $OPENFST_DIR/fstarcsort  \
      --sort_type=ilabel   \
      $PREPROCESSOR_PATH   \
      $PREPROCESSOR_PATH

  log_info "wrote preprocessor to '${PREPROCESSOR_PATH}'"
}


function extract_morphophonemics() {
  log_info "Extracting morphophonemics FST (first analysis layer)..."

  $OPENFST_DIR/farextract $SRC_DIR/morphophonemics/model.far

  mv                         \
      RESOLVE                \
      $MORPHOPHONEMICS_PATH

  # Grammars that are under //src/analyzer/morphophonemics implement the
  # inverse of the actual morphophonemics FST to simplify unit testing. Thus,
  # inverting the morphophonemics model FST before composing with the rest.
  $OPENFST_DIR/fstinvert     \
      $MORPHOPHONEMICS_PATH  \
      $MORPHOPHONEMICS_PATH

  $OPENFST_DIR/fstarcsort    \
      --sort_type=ilabel     \
      $MORPHOPHONEMICS_PATH  \
      $MORPHOPHONEMICS_PATH

  log_info "wrote morphophonemics to '${MORPHOPHONEMICS_PATH}'"
}

function build_morphotactics() {
  log_info "Compiling morphotactics FST (second analysis layer)..."

  $SRC_DIR/morphotactics/model_compile                  \
      --lexicon_dir=$SRC_DIR/lexicon/base               \
      --morphotactics_dir=$SRC_DIR/morphotactics/model  \
      --output_dir=$OUT_DIR

  # We use concatenation of two symbol tables in FST compilation:
  #   - base symbols: the set of symbols which can occur in Turkish words that
  #       are input to the morphological analyzer. These are enumerated in
  #       //src/analyzer/symbols/base_symbols.syms. If you use a custom lexicon
  #       to build the morphological analyzer, you need to make sure that all
  #       the characters of the strings that appear in 'word' and
  #       'morphophonemics' fields of all lexicon entries are in the base
  #       symbols table with their corresponding integer Unicode codepoints.
  #       Base symbols are converted to Unicode codepoints by the preprocessing
  #       FST, since morphophonemics FST (the first layer of analysis) assumes
  #       the input to be Unicode codepoints.
  #   - complex symbols: the set of complex symbols, usually multi-character
  #       (e.g. meta-morphemes, sub-parts of output analysis strings), which
  #       are used in defining morphotactics model. These are generated by
  #       //src/analyzer/morphotactics/model_compile.py, and it is assumed that
  #       none of these can occur in input words. Integer labels of these
  #       symbols place them in Unicode private use area, so that a possible
  #       clash with base symbols are avoided.
  cat                                     \
      $SRC_DIR/symbols/base_symbols.syms  \
      $OUT_DIR/complex_symbols.syms       \
      > $SYMBOLS_PATH

  log_info "wrote merged symbols table (base + complex) to '${SYMBOLS_PATH}'"

  $OPENFST_DIR/fstcompile         \
      --isymbols=$SYMBOLS_PATH    \
      --osymbols=$SYMBOLS_PATH    \
      $OUT_DIR/morphotactics.txt  \
      $MORPHOTACTICS_PATH

  # Morphotactics model definitions that are under
  # //src/analyzer/morphotactics/model implement the inverse of the actual
  # morphotactics FST to simplify unit testing and development. Thus, inverting
  # the morphotactics FST before composing with the rest.
  $OPENFST_DIR/fstinvert   \
      $MORPHOTACTICS_PATH  \
      $MORPHOTACTICS_PATH

  $OPENFST_DIR/fstarcsort  \
      --sort_type=ilabel   \
      $MORPHOTACTICS_PATH  \
      $MORPHOTACTICS_PATH

  log_info "wrote morphotactics model to '${MORPHOTACTICS_PATH}'"
}

function compose_and_create_far() {
  log_info "Composing analyzer components and creating FAR file..."

  $OPENFST_DIR/fstcompose    \
      $PREPROCESSOR_PATH     \
      $MORPHOPHONEMICS_PATH  \
      $OUT_DIR/$RULE_NAME

  $OPENFST_DIR/fstcompose  \
      $OUT_DIR/$RULE_NAME  \
      $MORPHOTACTICS_PATH  \
      $OUT_DIR/$RULE_NAME

  $OPENFST_DIR/fstrmepsilon      \
      $OUT_DIR/$RULE_NAME        \
  | $OPENFST_DIR/fstencode       \
      --encode_labels            \
      -                          \
      $OUT_DIR/labels.txt        \
  | $OPENFST_DIR/fstdeterminize  \
  | $OPENFST_DIR/fstminimize     \
  | $OPENFST_DIR/fstencode       \
      --decode                   \
      -                          \
      $OUT_DIR/labels.txt        \
  | $OPENFST_DIR/fstrmepsilon    \
  | $OPENFST_DIR/fstarcsort      \
      --sort_type=ilabel         \
  | $OPENFST_DIR/fstprint        \
      --isymbols=$SYMBOLS_PATH   \
      --osymbols=$SYMBOLS_PATH   \
  | $OPENFST_DIR/fstcompile      \
      --isymbols=$SYMBOLS_PATH   \
      --osymbols=$SYMBOLS_PATH   \
      --keep_isymbols            \
      --keep_osymbols            \
      -                          \
      $OUT_DIR/$RULE_NAME

  log_info "created analyzer FST with name '${RULE_NAME}'"

  $OPENFST_DIR/farcreate   \
      $OUT_DIR/$RULE_NAME  \
      $MODEL_PATH

  log_info "wrote the FAR file to '${MODEL_PATH}'"
}

function main() {
  readonly PROJECT_NAME=google_research_turkish_morphology
  readonly BASE_DIR=$(dirname $0)/build.runfiles/$PROJECT_NAME
  readonly SRC_DIR=$BASE_DIR/src/analyzer
  readonly OPENFST_DIR=$BASE_DIR/external/org_openfst

  readonly OUT_DIR=$1
  readonly PREPROCESSOR_PATH=$OUT_DIR/preprocessor.fst
  readonly MORPHOPHONEMICS_PATH=$OUT_DIR/morphophonemics.fst
  readonly MORPHOTACTICS_PATH=$OUT_DIR/morphotactics.fst
  readonly SYMBOLS_PATH=$OUT_DIR/alphabet.syms
  readonly RULE_NAME=turkish_morphological_analyzer
  readonly MODEL_PATH=$OUT_DIR/turkish.far

  extract_preprocessor
  extract_morphophonemics
  build_morphotactics
  compose_and_create_far
}

main "$@"
