Try a manual deployment

.gitlab-ci.yml

@@ -70,6 +70,15 @@ variables:
   - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/jobs/master-calqul.yml
   - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/jobs/master-calqul.yml
 
+  # Job Compute Indices 
+  - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/jobs/compute-indices-calqul.yml
+  - sed -i "s/__POSTGRES_DB__/$POSTGRES_DB/g" deployment/jobs/compute-indices-calqul.yml 
+  - sed -i "s/__POSTGRES_PORT__/$POSTGRES_PORT/g" deployment/jobs/compute-indices-calqul.yml 
+  - sed -i "s/__POSTGRES_SERVER__/$POSTGRES_SERVER-$NAMESPACE_ENV/g" deployment/jobs/compute-indices-calqul.yml 
+  - sed -i "s/__POSTGRES_USER__/$POSTGRES_USER/g" deployment/jobs/compute-indices-calqul.yml 
+  - sed -i "s/__CALQUL_ACTION__/$CALQUL_ACTION/g" deployment/jobs/compute-indices-calqul.yml 
+  
+
   # Git-lfs deployment file
   - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/pods/git-calqul.yml
   - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/pods/git-calqul.yml
@@ -95,16 +104,20 @@ variables:
   # - oc delete pods --selector pod-name=calqul-git-lfs-$NAMESPACE_ENV #--grace-period=0 --force
   # - oc apply -f deployment/pods/git-calqul.yml
 
-  # Job
-  - oc delete jobs --selector job-name=calqul-master-job-$NAMESPACE_ENV 
-  - oc create -f deployment/jobs/master-calqul.yml
+  # # Job All
+  # - oc delete jobs --selector job-name=calqul-master-job-$NAMESPACE_ENV 
+  # - oc create -f deployment/jobs/master-calqul.yml
+  
+  # Job Compute indices
+  - oc delete jobs --selector job-name=calqul-compute-indices-$NAMESPACE_ENV 
+  - oc create -f deployment/jobs/compute-indices-calqul.yml
   
 ################################################################################
 #    CI/CD Stages
 ################################################################################
 stages:
   - Build Calqul
-  - Deploy Calqul
+  - Deploy Calqul Buttons
   - Deploy Calqul Production
 
 # *********************************************************************************************
@@ -180,11 +193,26 @@ Build Calqul Git-lfs:
 # * DEPLOY DEPLOY DEPLOY DEPLOY DEPLOY DEPLOY DEPLOY DEPLOY DEPLOY DEPLOY DEPLOY DEPLOY DEPLO *
 # *********************************************************************************************
 
-Deploy Calqul:
-  stage: Deploy Calqul
+# Deploy Calqul:
+#   stage: Deploy Calqul
+#   environment: dev
+#   variables:
+#     NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
+#   before_script:
+#     - *prepare_deployment
+#   script:
+#    - *apply_confs
+#   rules:
+#     - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+#   tags:
+#     - ns-arb-d01
+
+Deploy Calqul Buttons:
+  stage: Deploy Calqul Compute Indices
   environment: dev
   variables:
     NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
+    CALQUL_ACTION: "computeIndices"
   before_script:
     - *prepare_deployment
   script:
@@ -193,6 +221,8 @@ Deploy Calqul:
     - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
   tags:
     - ns-arb-d01
+  when:
+    - manual
 
 # *********************************************************************************************
 # * MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN  *

Dockerfiles/python/Dockerfile

@@ -56,11 +56,11 @@ COPY --chown=calquluser:calquluser .env ./
 RUN chmod 755 /app/*.sh
 
 # Arguments
-ARG NAMESPACE_ENV
-ARG DB_HOST_ENV
-ARG DB_PORT_ENV
-ARG DB_NAME_ENV
-ARG DB_USER_ENV
+# ARG NAMESPACE_ENV
+# ARG DB_HOST_ENV
+# ARG DB_PORT_ENV
+# ARG DB_NAME_ENV
+# ARG DB_USER_ENV
 
 # Getting some Args and Setting some ENV variables
 ENV CPLUS_INCLUDE_PATH="/usr/include/gdal"
@@ -71,12 +71,14 @@ ENV GDAL_DATA="/usr/local/lib/python3.7/dist-packages/fiona/gdal_data/"
 # Debug verbosity
 ENV PROJ_DEBUG=3
 
-ENV NS_ENV=$NAMESPACE_ENV
-ENV DB_HOST="$DB_HOST_ENV-$NAMESPACE_ENV"
-ENV DB_PORT=$DB_PORT_ENV
-ENV DB_NAME=$DB_NAME_ENV
-ENV DB_USER=$DB_USER_ENV
+# ENV NS_ENV=$NAMESPACE_ENV
+# ENV DB_HOST="$DB_HOST_ENV-$NAMESPACE_ENV"
+# ENV DB_PORT=$DB_PORT_ENV
+# ENV DB_NAME=$DB_NAME_ENV
+# ENV DB_USER=$DB_USER_ENV
 
 # Launch calculation
 # Documentation : https://stackoverflow.com/questions/40454470/how-can-i-use-a-variable-inside-a-dockerfile-cmd
-CMD [ "/bin/sh", "-c", "/app/launch.sh $NS_ENV $DB_HOST $DB_PORT $DB_NAME $DB_USER" ]
+# CMD [ "/bin/sh", "-c", "/app/launch.sh $NS_ENV $DB_HOST $DB_PORT $DB_NAME $DB_USER" ]
+
+CMD [ "/bin/bash" ]
\ No newline at end of file

Dockerfiles/python/scripts/launch.sh

@@ -5,11 +5,12 @@
 alias python=python3
 source .env
 
-namespace_env=$1
-DB_HOST=$2
-DB_PORT=$3
-DB_NAME=$4
-DB_USER=$5
+action=$1
+namespace_env=$2
+DB_HOST=$3
+DB_PORT=$4
+DB_NAME=$5
+DB_USER=$6
 
 # GRID_SIZE=50
 GRID_SIZE=5
@@ -51,6 +52,11 @@ comment () {
   echo -e "\e[39m\t-> $1\e[39m"
 }
 
+# Usage
+usage () {
+  comment "$0 [initDatas|computeFactors|computeIndices|dumpDatas|all] $namespace_env $DB_HOST $DB_PORT $DB_NAME $DB_USER"
+}
+
 # Check the last command return code (must be insterted just after the commend )
 check () {
   if [ $? -eq 0 ]; then
@@ -77,7 +83,7 @@ stage "Launch ENV Initializations..."
 cd $scripts_dir
 
 # All the needed variables a given by parameter passing
-comment "command line is '$0 $namespace_env $DB_HOST $DB_PORT $DB_NAME $DB_USER'"
+comment "command line is '$0 $action $namespace_env $DB_HOST $DB_PORT $DB_NAME $DB_USER'"
 
 comment "psql version..."
 psql -V
@@ -90,51 +96,71 @@ check
 comment "Python parameters : "
 python3 main.py displayEnv
  
-stage "InitDatas"
-python3 main.py initDatas
-check
-
-stage "Compute Factors & Indices"
-for NOM_COMMUNE in $( echo "${!LISTE_COMMUNES[@]}" | tr ' ' '\n' | sort ); do
-    CODE_INSEE=${LISTE_COMMUNES[$NOM_COMMUNE]}
-    stage "Compute Factors : $NOM_COMMUNE"
-    python3 main.py computeFactors $CODE_INSEE
+comment "Checking action to do : "
+case "$action" in 
+  "initDatas"|"computeFactors"|"computeIndices"|"dumpDatas"|"all" )
+    comment "Action is '$action'."
     check
-
-done
-
-stage "Compute Indices"
-python3 main.py computeIndices
-check
+  ;;
+  *)
+    comment "Actiion parameter is not recognized."
+    usage
+    exit 2
+  ;;
+esac
+
+if [ $action == "initDatas"  ] || [ $action == "all"  ]; then
+  stage "InitDatas"
+  python3 main.py initDatas
+  check
+fi
+
+if [ $action == "computeFactors"  ] || [ $action == "all"  ]; then
+  stage "Compute Factors & Indices"
+  for NOM_COMMUNE in $( echo "${!LISTE_COMMUNES[@]}" | tr ' ' '\n' | sort ); do
+      CODE_INSEE=${LISTE_COMMUNES[$NOM_COMMUNE]}
+      stage "Compute Factors : $NOM_COMMUNE"
+      python3 main.py computeFactors $CODE_INSEE
+      check
+  done
+fi
+
+if [ $action == "computeIndices"  ] || [ $action == "all"  ]; then
+  stage "Compute Indices"
+  python3 main.py computeIndices
+  check
+fi
 
 # Launching everything, it is possible to give a list of townships
 # python3 main.py computeAll
 
-stage "Dumping result database"
-#
-# Option "--no-password"  is set not to have to provide password by prompt.
-# This requires the presnece of /root/.pgpass file (600 mode) with such a content : "hostname:port:database:username:password"
-# https://stackoverflow.com/questions/50404041/pg-dumpall-without-prompting-password
-#
-comment "pg_dump -n base -h ${DB_HOST} -U ${DB_USER} --no-password --clean --if-exists --file=$backup_dir/$dump_name.sql ${DB_NAME}"
-pg_dump -n base -h ${DB_HOST} -U ${DB_USER} --no-password --clean --if-exists --file=$backup_dir/$dump_name.sql ${DB_NAME}
-check
-
-comment "Commpressing dump as $dump_name.tgz"
-tar cvzf $backup_dir/$dump_name.tgz $backup_dir/$dump_name.sql
-check
-
-stage "Uploading archive in repo with tag $archive_version"
-comment "Upload to file server 'Geo'"
-
-stage "Cleanup backup dir '$backup_dir'"
-comment "old sql files"
-find $backup_dir -name "*.sql" -exec rm -f {} \;
-check
-
-comment "old tgz files"
-find $backup_dir -name "*.tgz" -mtime +5 -exec rm -f {} \;
-check
+if [ $action == "dumpDatas"  ] || [ $action == "all"  ]; then
+  stage "Dumping result database"
+  #
+  # Option "--no-password"  is set not to have to provide password by prompt.
+  # This requires the presnece of /root/.pgpass file (600 mode) with such a content : "hostname:port:database:username:password"
+  # https://stackoverflow.com/questions/50404041/pg-dumpall-without-prompting-password
+  #
+  comment "pg_dump -n base -h ${DB_HOST} -U ${DB_USER} --no-password --clean --if-exists --file=$backup_dir/$dump_name.sql ${DB_NAME}"
+  pg_dump -n base -h ${DB_HOST} -U ${DB_USER} --no-password --clean --if-exists --file=$backup_dir/$dump_name.sql ${DB_NAME}
+  check
+
+  comment "Commpressing dump as $dump_name.tgz"
+  tar cvzf $backup_dir/$dump_name.tgz $backup_dir/$dump_name.sql
+  check
+
+  stage "Uploading archive in repo with tag $archive_version"
+  comment "Upload to file server 'Geo'"
+
+  stage "Cleanup backup dir '$backup_dir'"
+  comment "old sql files"
+  find $backup_dir -name "*.sql" -exec rm -f {} \;
+  check
+
+  comment "old tgz files"
+  find $backup_dir -name "*.tgz" -mtime +5 -exec rm -f {} \;
+  check
+fi
 
 stage "End of script."
 exit 0

deployment/jobs/compute-indices-calqul.yml

+# 
+# Job qui lance un pod de calcul des indices 
+#
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: calqul-job-compute-indices-__NAMESPACE_ENV__
+spec:
+  # maximum duration the job can run => 1h 
+  activeDeadlineSeconds: 3600 
+  template:         
+    metadata:
+      name: calqul-job-compute-indices-__NAMESPACE_ENV__
+    spec:
+      volumes:
+      - name: arb-data
+        persistentVolumeClaim: 
+          claimName: pvc-01-__NAMESPACE__-claim
+      containers:
+      - name: calqul-job-compute-indices-__NAMESPACE_ENV__
+        # Image Python de calcul du calque
+        image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-python-__NAMESPACE_ENV__
+        command: ["/bin/bash", "-c", "/app/launch.sh __CALQUL_ACTION__ __NAMESPACE_ENV__ __POSTGRES_SERVER__ __POSTGRES_PORT__ __POSTGRES_DB__ __POSTGRES_USER__" ]
+        volumeMounts:
+        - name: arb-data
+          mountPath: "/arb-data/source-files"
+        - name: arb-data
+          mountPath: "/arb-data/generated-files"
+        - name: arb-data
+          mountPath: "/arb-data/run" # storing a file telling export-job the state of python script
+      restartPolicy: Never
\ No newline at end of file