Launch script

.gitignore

-__pycache__/*
-logs/*
-.env
\ No newline at end of file
+__pycache__/*
+logs/
+tmp/
+Dockerfiles/python/.env
+! Dockerfiles/python/.env.example
+.env

.gitlab-ci.yml

@@ -9,8 +9,7 @@ variables:
 ################################################################################
 #    Anchors
 ################################################################################
-# Build and push anchor
-.build_and_push_docker_image: &build_and_push_docker_image
+.build_and_push_image: &build_and_push_image
   - docker build --pull --tag "$CI_REGISTRY_IMAGE/$IMAGE_NAME:latest" .
   - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:latest"
 
@@ -18,15 +17,52 @@ variables:
 .docker_login: &docker_login
   - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
 
+# Set_env : copying env sample file and replace values
+.set_env: &set_env
+    - cp .env.example .env
+    - echo "# Configuration generated by Gitlab CI. (#$CI_BUILD_ID)" > .env 
+    - echo "#" >> .env
+    - cat .env.example >> .env
+    - sed -i "s/__POSTGRES_DB__/$POSTGRES_DB/g" .env
+    - sed -i "s/__POSTGRES_PASSWORD__/$POSTGRES_PASSWORD/g" .env
+    - sed -i "s/__POSTGRES_PORT__/$POSTGRES_PORT/g" .env 
+    - sed -i "s/__POSTGRES_SERVER__/$POSTGRES_SERVER-$NAMESPACE_ENV/g" .env
+    - sed -i "s/__POSTGRES_USER__/$POSTGRES_USER/g" .env
+    - sed -i "s/__POSTGRES_SCHEMA__/$POSTGRES_SCHEMA/g" .env
+
 # Deployment preparation anchor
 .prepare_deployment: &prepare_deployment
   - export NO_PROXY=grandlyon.fr
   - export KUBECONFIG=$KUBECONFIG_DEV
   - echo Preparing Deployment on $NAMESPACE 
+
+  # DB deployment file
+  - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/db-calqul.yml
+  - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/db-calqul.yml
+  - sed -i "s/__POSTGRES_DB__/$POSTGRES_DB/g" deployment/db-calqul.yml 
+  - sed -i "s/__POSTGRES_PASSWORD__/$POSTGRES_PASSWORD/g" deployment/db-calqul.yml 
+  - sed -i "s/__POSTGRES_PORT__/$POSTGRES_PORT/g" deployment/db-calqul.yml 
+  - sed -i "s/__POSTGRES_SERVER__/$POSTGRES_SERVER-$NAMESPACE_ENV/g" deployment/db-calqul.yml 
+  - sed -i "s/__POSTGRES_USER__/$POSTGRES_USER/g" deployment/db-calqul.yml 
+  - sed -i "s/__POSTGRES_SCHEMA__/$POSTGRES_SCHEMA/g" deployment/db-calqul.yml 
+
+  # DB service file
+  - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/service-calqul.yml
+  - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/service-calqul.yml
+
+  # Job deployment file
+  - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/job-calqul.yml
   - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/job-calqul.yml
+  - sed -i "s/__POSTGRES_PORT__/$POSTGRES_PORT/g" deployment/job-calqul.yml
 
 # Applying openshift conf anchor
 .apply_confs: &apply_confs
+  # Db
+  - oc delete pods --selector pod-name=calqul-db-$NAMESPACE_ENV #--grace-period=0 --force
+  - oc apply -f deployment/db-calqul.yml
+  - oc apply -f deployment/service-calqul.yml
+
+  # Job
   - oc delete jobs --selector job-name=calqul-$NAMESPACE_ENV 
   - oc create -f deployment/job-calqul.yml
 
@@ -41,21 +77,48 @@ stages:
 # * BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD *
 # *********************************************************************************************
 
-Build Calqul:
+Build Calqul Python:
   stage: Build Calqul
   variables:
     DOCKER_TLS_CERTDIR: ""
     NAMESPACE_ENV: "d01"
-    IMAGE_NAME: calqul-$NAMESPACE_ENV
+    IMAGE_NAME: calqul-python-$NAMESPACE_ENV
+    IMAGE_DIR: ./Dockerfiles/python
+  services:
+    - docker:dind
+  script:
+    - *docker_login
+    - cd $IMAGE_DIR
+    - *set_env
+    - *build_and_push_image
   rules:
-    - if: $CI_COMMIT_BRANCH == "develop"
-    - changes: # build image only if sources have changed. 
-      - Dockerfile # Image definition has changed 
+    - if: '$CI_COMMIT_BRANCH == "launch-script"'
+      changes: # build image only if sources have changed. 
+      - Dockerfiles/python/Dockerfile # Image definition has changed 
+      - Dockerfiles/python/scripts/* # Python scripts have changed
+      - .gitlab-ci.yml
+  tags:
+    - build-push-to-registry
+
+
+Build Calqul Postgis:
+  stage: Build Calqul
+  variables:
+    DOCKER_TLS_CERTDIR: ""
+    NAMESPACE_ENV: "d01"
+    IMAGE_NAME: calqul-db-$NAMESPACE_ENV
+    IMAGE_DIR: ./Dockerfiles/db
   services:
     - docker:dind
   script:
     - *docker_login
-    - *build_and_push_docker_image
+    - cd $IMAGE_DIR
+    - *build_and_push_image
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "launch-script"'
+      changes: # build image only if sources have changed. 
+      - Dockerfiles/db/**/* # db scripts have changed
+      - .gitlab-ci.yml
   tags:
     - build-push-to-registry
 
@@ -65,8 +128,6 @@ Build Calqul:
 
 Deploy Calqul:
   stage: Deploy Calqul
-  rules:
-    - if: $CI_COMMIT_BRANCH == "develop"
   variables:
     NAMESPACE_ENV: 'd01'
     NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV"
@@ -74,6 +135,8 @@ Deploy Calqul:
     - *prepare_deployment
   script:
    - *apply_confs
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "launch-script"'
   tags:
     #- ns-arb-d01 @TODO : Change after when the right namespace is available  
-    - ns-test-d01
\ No newline at end of file
+    - ns-test-d01 # No possibility to put varaibles here :(
\ No newline at end of file

Dockerfile

-FROM python:3
-
-WORKDIR /usr/src/app
-
-RUN echo "-----------------------"
-RUN echo "    Python3 Image"
-RUN echo "-----------------------"
-
-COPY scripts/requirements.txt ./
-RUN pip install --no-cache-dir -r requirements.txt
-
-COPY ./scripts .
-
-CMD [ "python", "./hello-world.py" ]
\ No newline at end of file

Dockerfiles/db/Dockerfile

+#
+# ARB::Base de données postgis pour le calcul du calque de plantabilité
+#
+FROM postgis/postgis
+
+USER root
+
+ENV PGDATA=/arb-data/pgdata
+RUN mkdir -p $PGDATA && \
+chown -R postgres:0 /$PGDATA && \
+chmod 777 /$PGDATA
+
+# Entry point. 
+COPY ./db-init/ /docker-entrypoint-initdb.d/
+
+USER postgres
\ No newline at end of file

Dockerfiles/db/db-init/20_init_arb.sh

+#!/bin/bash
+# Boostraping de la base de données ARB
+# 
+
+# Working Directory
+cd /docker-entrypoint-initdb.d
+
+echo "--------------------------------------------------------------------------"
+echo " --> Boostraping de la base de donnees ARB"
+echo "     le repertoire est '$(pwd)'"
+echo "--------------------------------------------------------------------------"
+# Mettre ici le le schéma d'initailisation, les reprises de données, reéation de roles et grants, etc...
+echo "POSTGRES_DB=$POSTGRES_DB"         
+echo "POSTGRES_PASSWORD=$POSTGRES_PASSWORD"   
+echo "POSTGRES_PORT=$POSTGRES_PORT"       
+echo "POSTGRES_SERVER=$POSTGRES_SERVER"     
+echo "POSTGRES_USER=$POSTGRES_USER"       
+echo "POSTGRES_SCHEMA=$POSTGRES_SCHEMA"  

sql/create_tables.sql → Dockerfiles/db/db-init/30-create_tables.sql

@@ -4,6 +4,12 @@
 
 CREATE EXTENSION IF NOT EXISTS postgis;
 
+-- ------------------------
+--        POSTGIS
+-- ------------------------
+
+CREATE ROLE adm;
+
 -- ------------------------
 --         SCHEMA 
 -- ------------------------

Dockerfiles/db/db-init/90-end.sh

+#!/bin/bash
+# Boostraping de la base de données ARB
+#
+echo "--------------------------------------------------------------------------"
+echo "     Base de donnees ARB prete."
+echo "--------------------------------------------------------------------------"
\ No newline at end of file

.env.example → Dockerfiles/python/.env.example

 # DB settings
-DB_HOST="XXXXXXXXXXXXXX"
-DB_PORT=5432
-DB_USER="XXXXXXXXXXXXXX"
-DB_PWD="XXXXXXXXXXXXXX"
-DB_NAME="XXXXXXXXXXXXXX"
-DB_SCHEMA="XXXXXXXXXXXXXX"
+DB_HOST="__POSTGRES_SERVER__"
+DB_PORT=__POSTGRES_PORT__
+DB_USER="__POSTGRES_USER__"
+DB_PWD="__POSTGRES_PASSWORD__"
+DB_NAME="__POSTGRES_DB__"
+DB_SCHEMA="__POSTGRES_SCHEMA__"
 
 # Python settings
 PYTHON_LAUNCH="python"

Dockerfiles/python/Dockerfile

+##################################################################################
+#  Taken from https://github.com/Toblerity/Fiona/issues/1063
+##################################################################################
+FROM debian:buster-slim
+
+# Getting some Args and Setting some ENV variables
+ENV CPLUS_INCLUDE_PATH="/usr/include/gdal"
+ENV C_INCLUDE_PATH="/usr/include/gdal"
+ENV PROJ_LIB="/usr/local/lib/python3.7/dist-packages/pyproj/proj_dir/share/proj"
+ENV GDAL_DATA="/usr/local/lib/python3.7/dist-packages/fiona/gdal_data/"
+
+# Basic libs and install python3.7
+RUN apt-get update && apt-get upgrade -y \
+&& apt-get install -y procps wget python3.7 python3-pip gdal-bin libgdal-dev apt-utils git \
+&& apt-get clean \
+apt-get -y install proj proj-dev && \
+# https://stackoverflow.com/questions/70508775/error-could-not-build-wheels-for-pycairo-which-is-required-to-install-pyprojec
+apt-get install -y libcairo2 libcairo2-dev && \
+################################# Debug and diagnose tools #################################
+# apt-get install -qy apt-utils vim iputils-ping net-tools telnet curl postgresql-client &&\
+############################################################################################
+rm -rf /var/lib/apt/lists/*
+
+COPY ./scripts/requirements.txt requirements.txt
+
+# Pip
+RUN python3 -m pip install --no-cache-dir --upgrade pip \
+&& pip install --no-cache-dir --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` \
+&& pip install --no-cache-dir --ignore-installed PyYAML \
+# https://stackoverflow.com/questions/70508775/error-could-not-build-wheels-for-pycairo-which-is-required-to-install-pyprojec
+&& pip install --no-cache-dir manimlib \
+&& pip install --no-cache-dir manimce \
+# 
+&& pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Add a non root user (OpenShift Requirements)
+RUN addgroup calquluser && \ 
+useradd calquluser -g calquluser --home /app && \
+mkdir -p /app && \ 
+mkdir -p /app/logs && \
+mkdir -p /app/tmp && \
+chown -R calquluser:calquluser /app && \
+chmod 777 /app/logs /app/tmp
+
+
+WORKDIR /app
+
+USER calquluser
+
+# Add app 
+COPY --chown=calquluser:calquluser ./scripts/*.py ./
+COPY --chown=calquluser:calquluser --chmod= ./scripts/*.sh ./
+COPY --chown=calquluser:calquluser ./sql ./
+COPY --chown=calquluser:calquluser .env ./
+
+RUN chmod 755 ./*.sh
+
+# Launch calculation
+CMD [ "/bin/bash", "./launch.sh" ]
+
+# launch testing script
+# CMD [  "/bin/bash", "./test-env.sh" ]