From 74fb6d4a42cbf0eafb15a4daba0d898656819cbe Mon Sep 17 00:00:00 2001
From: Pierre-Gilles Levallois <pierregilles.levallois@gmail.com>
Date: Fri, 24 Mar 2023 14:22:34 +0100
Subject: [PATCH] Try a deployment with parallel jobs

---
 .gitlab-ci.yml                       | 180 ++++++++++++---------------
 Dockerfiles/python/scripts/launch.sh |   2 +-
 deployment/jobs/stage-calqul.yml     |   4 +-
 3 files changed, 82 insertions(+), 104 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6a9b73d..8356399 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -119,7 +119,7 @@ variables:
 ################################################################################
 stages:
   - Build Calqul
-  - Auto Deploy
+  - Parallel Deploy
   - Deploy Calqul Buttons
   - Deploy Calqul Production
 
@@ -210,54 +210,15 @@ Build Calqul Git-lfs:
 #   tags:
 #     - ns-arb-d01
 
-# 1. Init Grid:
-#   stage: Deploy Calqul Buttons
-#   when: manual
-#   environment: dev
-#   variables:
-#     NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
-#     CALQUL_ACTION: "init-grid"
-#   before_script:
-#     - *prepare_deployment
-#   script:
-#    - *apply_confs
-#   # 
-#   # @TODO : Déployer 67 jobs avec des paramétrages différents 
-#   #         En fait les jobs ne démarreront pas tous en même temps si on adapte bien les ressoureces 
-#   #         (vérifier quel est le max de RAM consommé et règler les requests dessus)
-#   #  - INSEE="$codeInsee"
-#   #  - *DeployJobInitGrid 
-#   # parallel:
-#     # codeInsee: ["69001","69002","69003", ...]
-#   rules:
-#     - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
-#   tags:
-#     - ns-arb-d01
 1. Init Grid:
-  stage: Auto Deploy
+  stage: Parallel Deploy
   variables:
     NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
-  before_script:
-    - *prepare_deployment
-  script:
-   - *apply_confs
-  rules:
-    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
-  tags:
-    - ns-arb-d01
   parallel:
     matrix:
       - CALQUL_ACTION: init-grid
         CODE_INSEE: [69003, 69029, 69033, 69034, 69040, 69044, 69046, 69271, 69063, 69273, 69068, 69069, 69071, 69072, 69275, 69081, 69276, 69085, 69087, 69088, 69089, 69278, 69091, 69096, 69100, 69279, 69142, 69250, 69116, 69117, 69381, 69382, 69383, 69384, 69385, 69386, 69387, 69388, 69389, 69127, 69282, 69283, 69284, 69143, 69149, 69152, 69153, 69163, 69286, 69168, 69191, 69194, 69202, 69199, 69204, 69205, 69207, 69290, 69233, 69292, 69293, 69296, 69244, 69256, 69259, 69260, 69266]
   environment: dev $CALQUL_ACTION/$CODE_INSEE
-
-2. Init Datas:
-  stage: Deploy Calqul Buttons
-  when: manual
-  environment: dev
-  variables:
-    NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
-    CALQUL_ACTION: "init-datas"
   before_script:
     - *prepare_deployment
   script:
@@ -267,69 +228,86 @@ Build Calqul Git-lfs:
   tags:
     - ns-arb-d01
 
-3. Compute Factors:
-  stage: Deploy Calqul Buttons
-  when: manual
-  environment: dev
-  variables:
-    NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
-    CALQUL_ACTION: "compute-factors"
-  before_script:
-    - *prepare_deployment
-  script:
-   - *apply_confs
-  rules:
-    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
-  tags:
-    - ns-arb-d01
+# 2. Init Datas:
+#   stage: Deploy Calqul Buttons
+#   when: manual
+#   environment: dev
+#   variables:
+#     NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
+#     CALQUL_ACTION: "init-datas"
+#     CODE_INSEE: ""
+#   before_script:
+#     - *prepare_deployment
+#   script:
+#    - *apply_confs
+#   rules:
+#     - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+#   tags:
+#     - ns-arb-d01
 
-4. Compute Indices:
-  stage: Deploy Calqul Buttons
-  when: manual
-  environment: dev
-  variables:
-    NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
-    CALQUL_ACTION: "compute-indices"
-  before_script:
-    - *prepare_deployment
-  script:
-   - *apply_confs
-  rules:
-    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
-  tags:
-    - ns-arb-d01
+# 3. Compute Factors:
+#   stage: Deploy Calqul Buttons
+#   when: manual
+#   environment: dev
+#   variables:
+#     NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
+#     CALQUL_ACTION: "compute-factors"
+#   before_script:
+#     - *prepare_deployment
+#   script:
+#    - *apply_confs
+#   rules:
+#     - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+#   tags:
+#     - ns-arb-d01
 
-5. Dump Datas:
-  stage: Deploy Calqul Buttons
-  when: manual
-  environment: dev
-  variables:
-    NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
-    CALQUL_ACTION: "dump-datas"
-  before_script:
-    - *prepare_deployment
-  script:
-   - *apply_confs
-  rules:
-    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
-  tags:
-    - ns-arb-d01
+# 4. Compute Indices:
+#   stage: Deploy Calqul Buttons
+#   when: manual
+#   environment: dev
+#   variables:
+#     NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
+#     CALQUL_ACTION: "compute-indices"
+#   before_script:
+#     - *prepare_deployment
+#   script:
+#    - *apply_confs
+#   rules:
+#     - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+#   tags:
+#     - ns-arb-d01
 
-All Stages:
-  stage: Deploy Calqul Buttons
-  when: manual
-  environment: dev
-  variables:
-    NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
-    CALQUL_ACTION: "all"
-  before_script:
-    - *prepare_deployment
-  script:
-   - *apply_confs
-  rules:
-    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
-  tags:
-    - ns-arb-d01
+# 5. Dump Datas:
+#   stage: Deploy Calqul Buttons
+#   when: manual
+#   environment: dev
+#   variables:
+#     NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
+#     CALQUL_ACTION: "dump-datas"
+#   before_script:
+#     - *prepare_deployment
+#   script:
+#    - *apply_confs
+#   rules:
+#     - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+#   tags:
+#     - ns-arb-d01
+
+# All Stages:
+#   stage: Deploy Calqul Buttons
+#   when: manual
+#   environment: dev
+#   variables:
+#     NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
+#     CALQUL_ACTION: "all"
+#   before_script:
+#     - *prepare_deployment
+#   script:
+#    - *apply_confs
+#   rules:
+#     - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+#   tags:
+#     - ns-arb-d01
 
 # *********************************************************************************************
 # * MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN  *
diff --git a/Dockerfiles/python/scripts/launch.sh b/Dockerfiles/python/scripts/launch.sh
index f73b3a7..1075204 100644
--- a/Dockerfiles/python/scripts/launch.sh
+++ b/Dockerfiles/python/scripts/launch.sh
@@ -6,12 +6,12 @@ alias python=python3
 source .env
 
 action=$1
+code_insee=$2
 namespace_env=$3
 DB_HOST=$4
 DB_PORT=$5
 DB_NAME=$6
 DB_USER=$7
-code_insee=$8
 
 GRID_SIZE=5
 
diff --git a/deployment/jobs/stage-calqul.yml b/deployment/jobs/stage-calqul.yml
index 61ca812..5214480 100644
--- a/deployment/jobs/stage-calqul.yml
+++ b/deployment/jobs/stage-calqul.yml
@@ -4,7 +4,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: calqul-stage-__CALQUL_ACTION__-__NAMESPACE_ENV__
+  name: calqul-stage-__CALQUL_ACTION__-__CODE_INSEE__-__NAMESPACE_ENV__
 spec:
   # maximum duration the job can run => 40h 
   activeDeadlineSeconds: 144000 
@@ -23,7 +23,7 @@ spec:
       - name: calqul-stage-__CALQUL_ACTION__-__NAMESPACE_ENV__
         # Image Python de calcul du calque
         image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-python-__NAMESPACE_ENV__
-        command: ["/bin/bash", "-c", "/app/launch.sh __CALQUL_ACTION__ __NAMESPACE_ENV__ __POSTGRES_SERVER__ __POSTGRES_PORT__ __POSTGRES_DB__ __POSTGRES_USER__ __CODE_INSEE__" ]
+        command: ["/bin/bash", "-c", "/app/launch.sh __CALQUL_ACTION__ __CODE_INSEE__ __NAMESPACE_ENV__ __POSTGRES_SERVER__ __POSTGRES_PORT__ __POSTGRES_DB__ __POSTGRES_USER__" ]
         volumeMounts:
         - name: arb-data
           mountPath: "/arb-data/source-files"
-- 
GitLab