launch.sh

#!/bin/bash
################################################################################
# ARB : Lancement du recalcul des indices du calque de plantabilité.
################################################################################
alias python=python3
source .env

action=$1
code_insee=$2
namespace_env=$3
DB_HOST=$4
DB_PORT=$5
DB_NAME=$6
DB_USER=$7

GRID_SIZE=5

# LISTE_COMMUNES=( ["ROCHETAILLEE-SUR-SAONE"]="69168" )
declare -A LISTE_COMMUNES
LISTE_COMMUNES=(["69003"]="ALBIGNY-SUR-SAONE" ["69029"]="BRON" ["69033"]="CAILLOUX-SUR-FONTAINES" ["69034"]="CALUIRE-ET-CUIRE" ["69040"]="CHAMPAGNE-AU-MONT-D'OR" ["69044"]="CHARBONNIERES-LES-BAINS" ["69046"]="CHARLY" ["69063"]="COLLONGES-AU-MONT-D'OR" ["69068"]="COUZON-AU-MONT-D'OR" ["69069"]="CRAPONNE" ["69071"]="CURIS-AU-MONT-D'OR" ["69072"]="DARDILLY" ["69081"]="ECULLY" ["69085"]="FLEURIEU-SUR-SAONE" ["69087"]="FONTAINES-SAINT-MARTIN" ["69088"]="FONTAINES-SUR-SAONE" ["69089"]="FRANCHEVILLE" ["69091"]="GIVORS" ["69096"]="GRIGNY" ["69100"]="IRIGNY" ["69116"]="LIMONEST" ["69117"]="LISSIEU" ["69127"]="MARCY-L'ETOILE" ["69142"]="LA-MULATIERE" ["69143"]="NEUVILLE-SUR-SAONE" ["69149"]="OULLINS" ["69152"]="PIERRE-BENITE" ["69153"]="POLEYMIEUX-AU-MONT-D'OR" ["69163"]="QUINCIEUX" ["69168"]="ROCHETAILLEE-SUR-SAONE" ["69191"]="SAINT-CYR-AU-MONT-D'OR" ["69194"]="SAINT-DIDIER-AU-MONT-D'OR" ["69199"]="SAINT-FONS" ["69202"]="SAINTE-FOY-LESLYON" ["69204"]="SAINT-GENIS-LAVAL" ["69205"]="SAINT-GENIS-LES-OLLIERES" ["69207"]="SAINT-GERMAIN-AU-MONT-D'OR" ["69233"]="SAINT-ROMAIN-AU-MONT-D'OR" ["69244"]="TASSIN-LA-DEMI-LUNE" ["69250"]="LA-TOUR-DE-SALVAGNY" ["69256"]="VAULX-EN-VELIN" ["69259"]="VENISSIEUX" ["69260"]="VERNAISON" ["69266"]="VILLEURBANNE" ["69271"]="CHASSIEU" ["69273"]="CORBAS" ["69275"]="DECINES-CHARPIEU" ["69276"]="FEYZIN" ["69278"]="GENAY" ["69279"]="JONAGE" ["69282"]="MEYZIEU" ["69283"]="MIONS" ["69284"]="MONTANAY" ["69286"]="RILLIEUX-LA-PAPE" ["69290"]="SAINT-PRIEST" ["69292"]="SATHONAY-CAMP" ["69293"]="SATHONAY-VILLAGE" ["69296"]="SOLAIZE" ["69381"]="LYON-1ER" ["69382"]="LYON-2EME" ["69383"]="LYON-3EME" ["69384"]="LYON-4EME" ["69385"]="LYON-5EME" ["69386"]="LYON-6EME" ["69387"]="LYON-7EME" ["69388"]="LYON-8EME" ["69389"]="LYON-9EME" )

# Si on travaille sur une seule commune (passée en paramètre)
if [ "x$code_insee" -eq "x" ]; then
  nom=${LISTE_COMMUNES[$code_insee]}
  LISTE_COMMUNES=( ["$nom"]="$code_insee" )
fi

DATA_REPO="https://forge.grandlyon.com/erasme/data-recalcul-calque.git"

scripts_dir="/app"
data_dir="/arb-data/source-files/data-recalcul-calque"
backup_dir="/arb-data/generated-files"
stage=1
line="\e[39m-----------------------------------------------"
need_update=1
today=$(date +"%Y%m%d")
dump_name="calque-plantabilite-$namespace_env-$today"
tag="1.0" # @TODO : should be parametric from last commit on data repo.
archive_version="v$tag-$today"

################################################################################
# functions
################################################################################

#---------------------------------------------------------------
# Functions
#---------------------------------------------------------------
# Logging
stage () {
  echo -e $line
  echo -e "\e[96m$stage. $1\e[39m"
  echo -e $line
  stage=$((stage+1))
}

# Formatting log line
comment () {
  echo -e "\e[39m\t-> $1\e[39m"
}

# Usage
usage () {
  comment "$0 [init-grid|init-datas|compute-factors|compute-indices|dump-datas|all] $namespace_env $DB_HOST $DB_PORT $DB_NAME $DB_USER"
}

# Check the last command return code (must be insterted just after the commend )
check () {
  if [ $? -eq 0 ]; then
   comment "\e[32mOk.\e[39m"
  else
   comment "\e[31mERROR !...\e[39m"
   exit 1
  fi;
}

# Overloading 'Exit' builtin function to get rid of 
# the running state every where in the code
exit () {
  error_code=$1
  echo "Exiting '$error_code'. (Sleeping for 1h for debug purpose)"
  sleep 3600
  builtin exit $error_code
}

#---------------------------------------------------------------
# M A I N
#---------------------------------------------------------------
stage "Launch ENV Initializations..."
cd $scripts_dir

# All the needed variables a given by parameter passing
comment "command line is '$0 $action $namespace_env $DB_HOST $DB_PORT $DB_NAME $DB_USER'"

comment "psql version..."
psql -V
check

comment "Postgres server says : "
pg_isready -d $DB_NAME -h $DB_HOST -p $DB_PORT -U $DB_USER
check 

# nb_try=1
# while [ $nb_try -lt 11 ]; do
#   is_pgready=$("pg_isready -d $DB_NAME -h $DB_HOST -p $DB_PORT -U $DB_USER")
#   if [ "x$is_pgready" != "x0" ]; then
#     comment "Try #$nb_try : PostGIS Database is not ready. Sleeping for 30s, before retry..."
#     sleep 1;
#     ((nb_try++))
#   else
#     break
#   fi  
# done
# if [ "x$is_pgready" != "x0" ]; then
#   is_pgready=$("pg_isready -d $DB_NAME -h $DB_HOST -p $DB_PORT -U $DB_USER")
#   check
# fi;

comment "Python parameters : "
python3 main.py displayEnv
 
comment "Checking action to do : "
case "$action" in 
  "init-grid"|"init-datas"|"compute-factors"|"compute-indices"|"dump-datas"|"all" )
    comment "Action is '$action'."
    check
  ;;
  *)
    comment "Actiion parameter is not recognized."
    usage
    exit 2
  ;;
esac

if [ $action == "init-grid"  ] || [ $action == "all"  ]; then
  # Do InitGrid, township by town ship to avoid memory overflow
  stage "init-grid"
  for CODE_INSEE in $( echo "${!LISTE_COMMUNES[@]}" | tr ' ' '\n' | sort ); do
      NOM_COMMUNE=${LISTE_COMMUNES[$CODE_INSEE]}
      stage "Init Grid : $NOM_COMMUNE"
      python3 main.py initGrid $GRID_SIZE $CODE_INSEE
    check
  done
fi

if [ $action == "init-datas"  ] || [ $action == "all"  ]; then
  stage "init-datas"
  python3 main.py initDatas
  check
fi

if [ $action == "compute-factors"  ] || [ $action == "all"  ]; then
  stage "Compute Factors & Indices"
  for CODE_INSEE in $( echo "${!LISTE_COMMUNES[@]}" | tr ' ' '\n' | sort ); do
      NOM_COMMUNE=${LISTE_COMMUNES[$CODE_INSEE]}
      stage "Compute Factors : $NOM_COMMUNE"
      python3 main.py computeFactors $CODE_INSEE
      check
  done
fi

if [ $action == "compute-indices"  ] || [ $action == "all"  ]; then
  stage "Compute Indices"
  python3 main.py computeIndices
  check
fi

# Launching everything, it is possible to give a list of townships
# python3 main.py computeAll

if [ $action == "dump-datas"  ] || [ $action == "all"  ]; then
  stage "Dumping result database"
  #
  # Option "--no-password"  is set not to have to provide password by prompt.
  # This requires the presnece of /root/.pgpass file (600 mode) with such a content : "hostname:port:database:username:password"
  # https://stackoverflow.com/questions/50404041/pg-dumpall-without-prompting-password
  #
  comment "pg_dump -n base -h ${DB_HOST} -U ${DB_USER} --no-password --clean --if-exists --file=$backup_dir/$dump_name.sql ${DB_NAME}"
  pg_dump -n base -h ${DB_HOST} -U ${DB_USER} --no-password --clean --if-exists --file=$backup_dir/$dump_name.sql ${DB_NAME}
  check

  comment "Commpressing dump as $dump_name.tgz"
  tar cvzf $backup_dir/$dump_name.tgz $backup_dir/$dump_name.sql
  check

  stage "Uploading archive in repo with tag $archive_version"
  comment "Upload to file server 'Geo'"

  stage "Cleanup backup dir '$backup_dir'"
  comment "old sql files"
  find $backup_dir -name "*.sql" -exec rm -f {} \;
  check

  comment "old tgz files"
  find $backup_dir -name "*.tgz" -mtime +5 -exec rm -f {} \;
  check
fi

stage "End of script."
exit 0