diff --git a/.env.example b/.env.example deleted file mode 100644 index f49497fd6f6b6448e9a6e65cbb27fdb75021e32c..0000000000000000000000000000000000000000 --- a/.env.example +++ /dev/null @@ -1,16 +0,0 @@ -# DB settings -DB_HOST="XXXXXXXXXXXXXX" -DB_PORT=5432 -DB_USER="XXXXXXXXXXXXXX" -DB_PWD="XXXXXXXXXXXXXX" -DB_NAME="XXXXXXXXXXXXXX" -DB_SCHEMA="XXXXXXXXXXXXXX" - -# Python settings -PYTHON_LAUNCH="python" - -# Others settings -TARGET_PROJ="EPSG:2154" -REMOVE_TEMP_FILE=False -SKIP_EXISTING_DATA=True -ENABLE_TRUNCATE=False \ No newline at end of file diff --git a/.gitignore b/.gitignore index 12931954cc1105d47ae56861115e582b7a2e66eb..9c11743fbabec39899cfa752d0ba2111347e70f7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ -__pycache__/* -logs/* -.env \ No newline at end of file +__pycache__/* +logs/ +tmp/ +Dockerfiles/python/.env +! Dockerfiles/python/.env.example +.env diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ef9a421832c3eb57aa6998f6cc66d465bd3068a4..43e988babd771a2421c1693a94223c3c36fbc740 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,8 +9,7 @@ variables: ################################################################################ # Anchors ################################################################################ -# Build and push anchor -.build_and_push_docker_image: &build_and_push_docker_image +.build_and_push_image: &build_and_push_image - docker build --pull --tag "$CI_REGISTRY_IMAGE/$IMAGE_NAME:latest" . - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:latest" @@ -18,15 +17,52 @@ variables: .docker_login: &docker_login - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY +# Set_env : copying env sample file and replace values +.set_env: &set_env + - cp .env.example .env + - echo "# Configuration generated by Gitlab CI. (#$CI_BUILD_ID)" > .env + - echo "#" >> .env + - cat .env.example >> .env + - sed -i "s/__POSTGRES_DB__/$POSTGRES_DB/g" .env + - sed -i "s/__POSTGRES_PASSWORD__/$POSTGRES_PASSWORD/g" .env + - sed -i "s/__POSTGRES_PORT__/$POSTGRES_PORT/g" .env + - sed -i "s/__POSTGRES_SERVER__/$POSTGRES_SERVER-$NAMESPACE_ENV/g" .env + - sed -i "s/__POSTGRES_USER__/$POSTGRES_USER/g" .env + - sed -i "s/__POSTGRES_SCHEMA__/$POSTGRES_SCHEMA/g" .env + # Deployment preparation anchor .prepare_deployment: &prepare_deployment - export NO_PROXY=grandlyon.fr - export KUBECONFIG=$KUBECONFIG_DEV - echo Preparing Deployment on $NAMESPACE + + # DB deployment file + - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/db-calqul.yml + - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/db-calqul.yml + - sed -i "s/__POSTGRES_DB__/$POSTGRES_DB/g" deployment/db-calqul.yml + - sed -i "s/__POSTGRES_PASSWORD__/$POSTGRES_PASSWORD/g" deployment/db-calqul.yml + - sed -i "s/__POSTGRES_PORT__/$POSTGRES_PORT/g" deployment/db-calqul.yml + - sed -i "s/__POSTGRES_SERVER__/$POSTGRES_SERVER-$NAMESPACE_ENV/g" deployment/db-calqul.yml + - sed -i "s/__POSTGRES_USER__/$POSTGRES_USER/g" deployment/db-calqul.yml + - sed -i "s/__POSTGRES_SCHEMA__/$POSTGRES_SCHEMA/g" deployment/db-calqul.yml + + # DB service file + - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/service-calqul.yml + - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/service-calqul.yml + + # Job deployment file + - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/job-calqul.yml - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/job-calqul.yml + - sed -i "s/__POSTGRES_PORT__/$POSTGRES_PORT/g" deployment/job-calqul.yml # Applying openshift conf anchor .apply_confs: &apply_confs + # Db + - oc delete pods --selector pod-name=calqul-db-$NAMESPACE_ENV #--grace-period=0 --force + - oc apply -f deployment/db-calqul.yml + - oc apply -f deployment/service-calqul.yml + + # Job - oc delete jobs --selector job-name=calqul-$NAMESPACE_ENV - oc create -f deployment/job-calqul.yml @@ -41,21 +77,48 @@ stages: # * BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD BUILD * # ********************************************************************************************* -Build Calqul: +Build Calqul Python: stage: Build Calqul variables: DOCKER_TLS_CERTDIR: "" NAMESPACE_ENV: "d01" - IMAGE_NAME: calqul-$NAMESPACE_ENV + IMAGE_NAME: calqul-python-$NAMESPACE_ENV + IMAGE_DIR: ./Dockerfiles/python + services: + - docker:dind + script: + - *docker_login + - cd $IMAGE_DIR + - *set_env + - *build_and_push_image rules: - - if: $CI_COMMIT_BRANCH == "develop" - - changes: # build image only if sources have changed. - - Dockerfile # Image definition has changed + - if: '$CI_COMMIT_BRANCH == "launch-script"' + changes: # build image only if sources have changed. + - Dockerfiles/python/Dockerfile # Image definition has changed + - Dockerfiles/python/scripts/* # Python scripts have changed + - .gitlab-ci.yml + tags: + - build-push-to-registry + + +Build Calqul Postgis: + stage: Build Calqul + variables: + DOCKER_TLS_CERTDIR: "" + NAMESPACE_ENV: "d01" + IMAGE_NAME: calqul-db-$NAMESPACE_ENV + IMAGE_DIR: ./Dockerfiles/db services: - docker:dind script: - *docker_login - - *build_and_push_docker_image + - cd $IMAGE_DIR + - *build_and_push_image + rules: + - if: '$CI_COMMIT_BRANCH == "launch-script"' + changes: # build image only if sources have changed. + - Dockerfiles/db/**/* # db scripts have changed + - .gitlab-ci.yml tags: - build-push-to-registry @@ -65,8 +128,6 @@ Build Calqul: Deploy Calqul: stage: Deploy Calqul - rules: - - if: $CI_COMMIT_BRANCH == "develop" variables: NAMESPACE_ENV: 'd01' NAMESPACE: "ns-$TRIGRAMME-$NAMESPACE_ENV" @@ -74,6 +135,8 @@ Deploy Calqul: - *prepare_deployment script: - *apply_confs + rules: + - if: '$CI_COMMIT_BRANCH == "launch-script"' tags: #- ns-arb-d01 @TODO : Change after when the right namespace is available - - ns-test-d01 \ No newline at end of file + - ns-test-d01 # No possibility to put varaibles here :( \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 10f059c865def36bef4cd5569fb2be6e295934af..0000000000000000000000000000000000000000 --- a/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3 - -WORKDIR /usr/src/app - -RUN echo "-----------------------" -RUN echo " Python3 Image" -RUN echo "-----------------------" - -COPY scripts/requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt - -COPY ./scripts . - -CMD [ "python", "./hello-world.py" ] \ No newline at end of file diff --git a/Dockerfiles/db/Dockerfile b/Dockerfiles/db/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..da39ae6fda1818159124fb900bcc431da65b411b --- /dev/null +++ b/Dockerfiles/db/Dockerfile @@ -0,0 +1,16 @@ +# +# ARB::Base de données postgis pour le calcul du calque de plantabilité +# +FROM postgis/postgis + +USER root + +ENV PGDATA=/arb-data/pgdata +RUN mkdir -p $PGDATA && \ +chown -R postgres:0 /$PGDATA && \ +chmod 777 /$PGDATA + +# Entry point. +COPY ./db-init/ /docker-entrypoint-initdb.d/ + +USER postgres \ No newline at end of file diff --git a/Dockerfiles/db/db-init/20_init_arb.sh b/Dockerfiles/db/db-init/20_init_arb.sh new file mode 100644 index 0000000000000000000000000000000000000000..f27cbca971d17d9e9958024e60de0140669799b2 --- /dev/null +++ b/Dockerfiles/db/db-init/20_init_arb.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Boostraping de la base de données ARB +# + +# Working Directory +cd /docker-entrypoint-initdb.d + +echo "--------------------------------------------------------------------------" +echo " --> Boostraping de la base de donnees ARB" +echo " le repertoire est '$(pwd)'" +echo "--------------------------------------------------------------------------" +# Mettre ici le le schéma d'initailisation, les reprises de données, reéation de roles et grants, etc... +echo "POSTGRES_DB=$POSTGRES_DB" +echo "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" +echo "POSTGRES_PORT=$POSTGRES_PORT" +echo "POSTGRES_SERVER=$POSTGRES_SERVER" +echo "POSTGRES_USER=$POSTGRES_USER" +echo "POSTGRES_SCHEMA=$POSTGRES_SCHEMA" diff --git a/sql/create_tables.sql b/Dockerfiles/db/db-init/30-create_tables.sql similarity index 96% rename from sql/create_tables.sql rename to Dockerfiles/db/db-init/30-create_tables.sql index 0552ed93f771be8a20aed19782414cf66c53d4e7..746012edafbff5f99960a45444a7bcfa93d2822e 100644 --- a/sql/create_tables.sql +++ b/Dockerfiles/db/db-init/30-create_tables.sql @@ -4,6 +4,12 @@ CREATE EXTENSION IF NOT EXISTS postgis; +-- ------------------------ +-- POSTGIS +-- ------------------------ + +CREATE ROLE adm; + -- ------------------------ -- SCHEMA -- ------------------------ diff --git a/sql/insert_data.sql b/Dockerfiles/db/db-init/40-insert_data.sql similarity index 100% rename from sql/insert_data.sql rename to Dockerfiles/db/db-init/40-insert_data.sql diff --git a/Dockerfiles/db/db-init/90-end.sh b/Dockerfiles/db/db-init/90-end.sh new file mode 100644 index 0000000000000000000000000000000000000000..31c2d3f44305b4dd945ff0b55181c94711c9e6c9 --- /dev/null +++ b/Dockerfiles/db/db-init/90-end.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Boostraping de la base de données ARB +# +echo "--------------------------------------------------------------------------" +echo " Base de donnees ARB prete." +echo "--------------------------------------------------------------------------" \ No newline at end of file diff --git a/Dockerfiles/python/.env.example b/Dockerfiles/python/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..6ee418b8bfa58d8b8cc5586db5f7415f1fe58381 --- /dev/null +++ b/Dockerfiles/python/.env.example @@ -0,0 +1,16 @@ +# DB settings +DB_HOST="__POSTGRES_SERVER__" +DB_PORT=__POSTGRES_PORT__ +DB_USER="__POSTGRES_USER__" +DB_PWD="__POSTGRES_PASSWORD__" +DB_NAME="__POSTGRES_DB__" +DB_SCHEMA="__POSTGRES_SCHEMA__" + +# Python settings +PYTHON_LAUNCH="python" + +# Others settings +TARGET_PROJ="EPSG:2154" +REMOVE_TEMP_FILE=False +SKIP_EXISTING_DATA=True +ENABLE_TRUNCATE=False \ No newline at end of file diff --git a/Dockerfiles/python/Dockerfile b/Dockerfiles/python/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..b22e4d116686b2389441b3c9463391a62d04ada2 --- /dev/null +++ b/Dockerfiles/python/Dockerfile @@ -0,0 +1,62 @@ +################################################################################## +# Taken from https://github.com/Toblerity/Fiona/issues/1063 +################################################################################## +FROM debian:buster-slim + +# Getting some Args and Setting some ENV variables +ENV CPLUS_INCLUDE_PATH="/usr/include/gdal" +ENV C_INCLUDE_PATH="/usr/include/gdal" +ENV PROJ_LIB="/usr/local/lib/python3.7/dist-packages/pyproj/proj_dir/share/proj" +ENV GDAL_DATA="/usr/local/lib/python3.7/dist-packages/fiona/gdal_data/" + +# Basic libs and install python3.7 +RUN apt-get update && apt-get upgrade -y \ +&& apt-get install -y procps wget python3.7 python3-pip gdal-bin libgdal-dev apt-utils git \ +&& apt-get clean \ +apt-get -y install proj proj-dev && \ +# https://stackoverflow.com/questions/70508775/error-could-not-build-wheels-for-pycairo-which-is-required-to-install-pyprojec +apt-get install -y libcairo2 libcairo2-dev && \ +################################# Debug and diagnose tools ################################# +# apt-get install -qy apt-utils vim iputils-ping net-tools telnet curl postgresql-client &&\ +############################################################################################ +rm -rf /var/lib/apt/lists/* + +COPY ./scripts/requirements.txt requirements.txt + +# Pip +RUN python3 -m pip install --no-cache-dir --upgrade pip \ +&& pip install --no-cache-dir --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` \ +&& pip install --no-cache-dir --ignore-installed PyYAML \ +# https://stackoverflow.com/questions/70508775/error-could-not-build-wheels-for-pycairo-which-is-required-to-install-pyprojec +&& pip install --no-cache-dir manimlib \ +&& pip install --no-cache-dir manimce \ +# +&& pip install --no-cache-dir --upgrade -r requirements.txt + +# Add a non root user (OpenShift Requirements) +RUN addgroup calquluser && \ +useradd calquluser -g calquluser --home /app && \ +mkdir -p /app && \ +mkdir -p /app/logs && \ +mkdir -p /app/tmp && \ +chown -R calquluser:calquluser /app && \ +chmod 777 /app/logs /app/tmp + + +WORKDIR /app + +USER calquluser + +# Add app +COPY --chown=calquluser:calquluser ./scripts/*.py ./ +COPY --chown=calquluser:calquluser --chmod= ./scripts/*.sh ./ +COPY --chown=calquluser:calquluser ./sql ./ +COPY --chown=calquluser:calquluser .env ./ + +RUN chmod 755 ./*.sh + +# Launch calculation +CMD [ "/bin/bash", "./launch.sh" ] + +# launch testing script +# CMD [ "/bin/bash", "./test-env.sh" ] diff --git a/Pipfile b/Dockerfiles/python/scripts/Pipfile similarity index 100% rename from Pipfile rename to Dockerfiles/python/scripts/Pipfile diff --git a/aerien_enedis.py b/Dockerfiles/python/scripts/aerien_enedis.py similarity index 100% rename from aerien_enedis.py rename to Dockerfiles/python/scripts/aerien_enedis.py diff --git a/agricole.py b/Dockerfiles/python/scripts/agricole.py similarity index 100% rename from agricole.py rename to Dockerfiles/python/scripts/agricole.py diff --git a/arbre.py b/Dockerfiles/python/scripts/arbre.py similarity index 100% rename from arbre.py rename to Dockerfiles/python/scripts/arbre.py diff --git a/arbre_souche.py b/Dockerfiles/python/scripts/arbre_souche.py similarity index 100% rename from arbre_souche.py rename to Dockerfiles/python/scripts/arbre_souche.py diff --git a/arti.py b/Dockerfiles/python/scripts/arti.py similarity index 100% rename from arti.py rename to Dockerfiles/python/scripts/arti.py diff --git a/assainissement.py b/Dockerfiles/python/scripts/assainissement.py similarity index 100% rename from assainissement.py rename to Dockerfiles/python/scripts/assainissement.py diff --git a/batiment.py b/Dockerfiles/python/scripts/batiment.py similarity index 100% rename from batiment.py rename to Dockerfiles/python/scripts/batiment.py diff --git a/bus.py b/Dockerfiles/python/scripts/bus.py similarity index 100% rename from bus.py rename to Dockerfiles/python/scripts/bus.py diff --git a/facade.py b/Dockerfiles/python/scripts/facade.py similarity index 100% rename from facade.py rename to Dockerfiles/python/scripts/facade.py diff --git a/fibre.py b/Dockerfiles/python/scripts/fibre.py similarity index 100% rename from fibre.py rename to Dockerfiles/python/scripts/fibre.py diff --git a/foret.py b/Dockerfiles/python/scripts/foret.py similarity index 100% rename from foret.py rename to Dockerfiles/python/scripts/foret.py diff --git a/friche_nat.py b/Dockerfiles/python/scripts/friche_nat.py similarity index 100% rename from friche_nat.py rename to Dockerfiles/python/scripts/friche_nat.py diff --git a/gaz.py b/Dockerfiles/python/scripts/gaz.py similarity index 100% rename from gaz.py rename to Dockerfiles/python/scripts/gaz.py diff --git a/giratoire.py b/Dockerfiles/python/scripts/giratoire.py similarity index 100% rename from giratoire.py rename to Dockerfiles/python/scripts/giratoire.py diff --git a/scripts/hello-world.py b/Dockerfiles/python/scripts/hello-world.py similarity index 100% rename from scripts/hello-world.py rename to Dockerfiles/python/scripts/hello-world.py diff --git a/jeux.py b/Dockerfiles/python/scripts/jeux.py similarity index 100% rename from jeux.py rename to Dockerfiles/python/scripts/jeux.py diff --git a/Dockerfiles/python/scripts/launch.sh b/Dockerfiles/python/scripts/launch.sh new file mode 100644 index 0000000000000000000000000000000000000000..8aa29f1c0977c0ee7bf8dbbf277c0015a54652f9 --- /dev/null +++ b/Dockerfiles/python/scripts/launch.sh @@ -0,0 +1,92 @@ +#!/bin/bash +################################################################################ +# ARB : Lancement du recalcul des indices du calque de plantabilité. +################################################################################ +DATA_REPO="https://forge.grandlyon.com/erasme/data-recalcul-calque.git" + +scripts_dir="/app" +data_dir="/arb-data/source-files/data-recalcul-calque" +stage=1 +line="\e[39m-----------------------------------------------" +need_update=1 + +################################################################################ +# functions +################################################################################ + +#--------------------------------------------------------------- +# Functions +#--------------------------------------------------------------- +# Logging +stage () { + echo -e $line + echo -e "\e[96m$stage. $1\e[39m" + echo -e $line + stage=$((stage+1)) +} + +# Formatting log line +comment () { + echo -e "\e[39m\t-> $1\e[39m" +} + +# Check the last command return code (must be insterted just after the commend ) +check () { + if [ $? -eq 0 ]; then + comment "\e[32mOk.\e[39m" + else + comment "\e[31mERROR !...\e[39m" + exit 1 + fi; +} + +################################################################################ +mkdir -p $data_dir + +stage "Check source files in $data_dir" +cd $data_dir + +current_commit=$(git rev-parse --short HEAD) +if [ $? -eq 128 ]; then + comment "Data repository is empty. Is this your first time, young Padawan ?" + comment "Cloning repo, it can take a while..." + cd .. + git clone $DATA_REPO + need_update=0 +else + comment "Current commit is $current_commit" + cd $data_dir +fi; + +# Checking for update... +[ $(git rev-parse HEAD) = $(git ls-remote $(git rev-parse --abbrev-ref @{u} | sed 's/\// /g') | cut -f1) ] && need_update=0 || need_update=1 + +if [ $need_update -eq 0 ]; then + # We are up to date + comment "\e[32mData are up to date." +else + comment "\e[93mNew version of source data is available !\e[39m" + git pull origin main + new_commit=$(git rev-parse --short HEAD) + comment "New commit is : \e[93m'$new_commit'\e[39m" + git diff --compact-summary $current_commit $new_commit +fi; + +stage "Launch computations..." +cd $scripts_dir +comment "Init communes" +python3 main.py initCommunes + +python3 main.py initGrid +python3 main.py initDatas + +# Multiprocessing task +# python3 main.py computeFactors + +# python3 main.py computeIndices + +# Launching everything, it is possible to give a list of township +# python3 main.py computeAll + +stage "Sleeping a while for debug purpose..." +sleep 300 \ No newline at end of file diff --git a/main.py b/Dockerfiles/python/scripts/main.py similarity index 100% rename from main.py rename to Dockerfiles/python/scripts/main.py diff --git a/metro_funiculaire.py b/Dockerfiles/python/scripts/metro_funiculaire.py similarity index 100% rename from metro_funiculaire.py rename to Dockerfiles/python/scripts/metro_funiculaire.py diff --git a/parc.py b/Dockerfiles/python/scripts/parc.py similarity index 100% rename from parc.py rename to Dockerfiles/python/scripts/parc.py diff --git a/parcelle.py b/Dockerfiles/python/scripts/parcelle.py similarity index 100% rename from parcelle.py rename to Dockerfiles/python/scripts/parcelle.py diff --git a/pip_geopandas.txt b/Dockerfiles/python/scripts/pip_geopandas.txt similarity index 100% rename from pip_geopandas.txt rename to Dockerfiles/python/scripts/pip_geopandas.txt diff --git a/piste_cyclable.py b/Dockerfiles/python/scripts/piste_cyclable.py similarity index 100% rename from piste_cyclable.py rename to Dockerfiles/python/scripts/piste_cyclable.py diff --git a/plan_eau.py b/Dockerfiles/python/scripts/plan_eau.py similarity index 100% rename from plan_eau.py rename to Dockerfiles/python/scripts/plan_eau.py diff --git a/pont.py b/Dockerfiles/python/scripts/pont.py similarity index 100% rename from pont.py rename to Dockerfiles/python/scripts/pont.py diff --git a/requirements.txt b/Dockerfiles/python/scripts/requirements.txt similarity index 100% rename from requirements.txt rename to Dockerfiles/python/scripts/requirements.txt diff --git a/rsx_chaleur.py b/Dockerfiles/python/scripts/rsx_chaleur.py similarity index 100% rename from rsx_chaleur.py rename to Dockerfiles/python/scripts/rsx_chaleur.py diff --git a/slt.py b/Dockerfiles/python/scripts/slt.py similarity index 100% rename from slt.py rename to Dockerfiles/python/scripts/slt.py diff --git a/souterrain_enedis.py b/Dockerfiles/python/scripts/souterrain_enedis.py similarity index 100% rename from souterrain_enedis.py rename to Dockerfiles/python/scripts/souterrain_enedis.py diff --git a/strate_arboree.py b/Dockerfiles/python/scripts/strate_arboree.py similarity index 100% rename from strate_arboree.py rename to Dockerfiles/python/scripts/strate_arboree.py diff --git a/strate_basse.py b/Dockerfiles/python/scripts/strate_basse.py similarity index 100% rename from strate_basse.py rename to Dockerfiles/python/scripts/strate_basse.py diff --git a/Dockerfiles/python/scripts/test-env.sh b/Dockerfiles/python/scripts/test-env.sh new file mode 100644 index 0000000000000000000000000000000000000000..5c0483de0f5edc646083a32ff24ca21c23511a99 --- /dev/null +++ b/Dockerfiles/python/scripts/test-env.sh @@ -0,0 +1,10 @@ +# !/bin/bash +# Script de test de l'environnement +############################################################ +python3 main.py help +sleep 5 +python3 main.py test +sleep 5 +python3 main.py testDB +# Let's keep time to read logs if necessary before the pod is tarminated +sleep 3000 diff --git a/test.py b/Dockerfiles/python/scripts/test.py similarity index 100% rename from test.py rename to Dockerfiles/python/scripts/test.py diff --git a/tram.py b/Dockerfiles/python/scripts/tram.py similarity index 100% rename from tram.py rename to Dockerfiles/python/scripts/tram.py diff --git a/transport.py b/Dockerfiles/python/scripts/transport.py similarity index 100% rename from transport.py rename to Dockerfiles/python/scripts/transport.py diff --git a/utils.py b/Dockerfiles/python/scripts/utils.py similarity index 100% rename from utils.py rename to Dockerfiles/python/scripts/utils.py diff --git a/velo.py b/Dockerfiles/python/scripts/velo.py similarity index 100% rename from velo.py rename to Dockerfiles/python/scripts/velo.py diff --git a/velov.py b/Dockerfiles/python/scripts/velov.py similarity index 100% rename from velov.py rename to Dockerfiles/python/scripts/velov.py diff --git a/voie_ferree.py b/Dockerfiles/python/scripts/voie_ferree.py similarity index 100% rename from voie_ferree.py rename to Dockerfiles/python/scripts/voie_ferree.py diff --git a/sql/UPDATE_all_tiles_with_sum_indice.sql b/Dockerfiles/python/sql/UPDATE_all_tiles_with_sum_indice.sql similarity index 100% rename from sql/UPDATE_all_tiles_with_sum_indice.sql rename to Dockerfiles/python/sql/UPDATE_all_tiles_with_sum_indice.sql diff --git a/sql/copy_db.sql b/Dockerfiles/python/sql/copy_db.sql similarity index 100% rename from sql/copy_db.sql rename to Dockerfiles/python/sql/copy_db.sql diff --git a/sql/detect_duplicate_geom_tiles.sql b/Dockerfiles/python/sql/detect_duplicate_geom_tiles.sql similarity index 100% rename from sql/detect_duplicate_geom_tiles.sql rename to Dockerfiles/python/sql/detect_duplicate_geom_tiles.sql diff --git a/sql/list_active_query.sql b/Dockerfiles/python/sql/list_active_query.sql similarity index 100% rename from sql/list_active_query.sql rename to Dockerfiles/python/sql/list_active_query.sql diff --git a/sql/select_area_and_factors_mult.sql b/Dockerfiles/python/sql/select_area_and_factors_mult.sql similarity index 100% rename from sql/select_area_and_factors_mult.sql rename to Dockerfiles/python/sql/select_area_and_factors_mult.sql diff --git a/sql/select_count_currfactor_and_insee.sql b/Dockerfiles/python/sql/select_count_currfactor_and_insee.sql similarity index 100% rename from sql/select_count_currfactor_and_insee.sql rename to Dockerfiles/python/sql/select_count_currfactor_and_insee.sql diff --git a/sql/sum_indice_area_factors.sql b/Dockerfiles/python/sql/sum_indice_area_factors.sql similarity index 100% rename from sql/sum_indice_area_factors.sql rename to Dockerfiles/python/sql/sum_indice_area_factors.sql diff --git a/README.md b/README.md index 007b39a866a92142a77730f97757d6422849f2be..aa4b095e63e34463891050fedb36e93cae4ccceb 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ pip install -r requirements.txt * Lancer une première fois le script pour afficher la documentation ```bash -python main.py +python - main.py ``` **Bravo ! Vous êtes désormais prêt à lancer un nouveau calcul du calque de plantabilité !** 🎉 @@ -34,7 +34,7 @@ python main.py La documentation du script vous aidera à comprendre les arguments à passer pour lancer chaque étape du calcul : ```bash -$ python main.py help +$ python - main.py help Args: initCommunes Insert Communes in database from a geoJSON file path (with geometry and insee column) @@ -45,8 +45,8 @@ Args: Can be launch on certain "communes" with one <inseeCode> or in all territory by default (no parameter) computeIndices Compute the plantability indices on each tile with database informations. computeAll <gridSize: int, listInseeCode: int> Generate all the plantability layer (launch all previous steps). - List of inseeCode must be separated with comma (,) and without space (e.g. python main.py 5 69266,69388,69256) - but you can launch treatments for only one commune (e.g. python main.py 5 69266) + List of inseeCode must be separated with comma (,) and without space (e.g. python - main.py 5 69266,69388,69256) + but you can launch treatments for only one commune (e.g. python - main.py 5 69266) help Show this documentation ``` @@ -117,8 +117,17 @@ Vous trouverez le détail de ce projet sur les documents suivants : ## Intégration continue & Déploiement ## Build -Image de base Python : https://hub.docker.com/_/python - +Image de base Debian : https://hub.docker.com/_/debian:buster-slim + +### Configuration de Gitlab +les variables comportant les données de connexion à la base PostGIS doivent être initiialisées dans Gitlab. +Sous la rubrique Settings > CI/CD > Variables : +POSTGRES_DB calque_planta_temp +POSTGRES_PASSWORD xxxxxx +POSTGRES_PORT 5432 +POSTGRES_SERVER calqul-db-service (Le service OpenShift qui est routé vers la base PostGIS) +POSTGRES_USER calqul +POSTGRES_SCHEMA calqul ## Deploy ### Déploiemet d'un Job Openshift @@ -131,4 +140,23 @@ Doc : - https://docs.openshift.com/container-platform/4.11/nodes/jobs/nodes-nodes-jobs.html ### Suppression d'un job - - https://access.redhat.com/documentation/en-us/openshift_container_platform/3.4/html/developer_guide/dev-guide-scheduled-jobs \ No newline at end of file + - https://access.redhat.com/documentation/en-us/openshift_container_platform/3.4/html/developer_guide/dev-guide-scheduled-jobs + +### Commandes lancées par le job +- main.py initCommunes + +- main.py initGrid + +- main.py initDatas + +- main.py computeFactors + +- main.py computeIndices + +- main.py computeAll + +- main.py test + +- main.py testDB + +- main.py help \ No newline at end of file diff --git a/deployment/db-calqul.yml b/deployment/db-calqul.yml new file mode 100644 index 0000000000000000000000000000000000000000..0e06ff3fbcf2324889d492ea5b6cc09aef57e969 --- /dev/null +++ b/deployment/db-calqul.yml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calqul-db-__NAMESPACE_ENV__ + namespace: __NAMESPACE__ + labels: + apps: calqul-db-__NAMESPACE_ENV__ +spec: + replicas: 1 + selector: + matchLabels: + app: calqul-db-__NAMESPACE_ENV__ + role: primary + tier: database + template: + metadata: + labels: + app: calqul-db-__NAMESPACE_ENV__ + role: primary + tier: database + spec: + imagePullSecrets: + - name: forge-secret + volumes: + - name: db-data + persistentVolumeClaim: + claimName: pvc-01-__NAMESPACE__-claim + containers: + - name: calqul-db-__NAMESPACE_ENV__ + image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-db-__NAMESPACE_ENV__ + imagePullPolicy: Always + ports: + - containerPort: 5432 + protocol: TCP + readinessProbe: + tcpSocket: + port: 5432 + initialDelaySeconds: 120 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 5432 + initialDelaySeconds: 15 + periodSeconds: 20 + ressources: + requests: + memory: "256M" + cpu: "200mi" + limits: + memory: "512M" + cpu: "500mi" + env: + - name: POSTGRES_DB + value: "__POSTGRES_DB__" + - name: POSTGRES_PASSWORD + value: "__POSTGRES_PASSWORD__" + - name: POSTGRES_PORT + value: "__POSTGRES_PORT__" + - name: POSTGRES_SERVER + value: "__POSTGRES_SERVER__" + - name: POSTGRES_USER + value: "__POSTGRES_USER__" + - name: POSTGRES_SCHEMA + value: "__POSTGRES_SCHEMA__" + # need to define PGDATA env var because not using the default /var/lib/postgresql/data, but a subpath under it. + - name: PGDATA + value: /arb-data/pgdata + volumeMounts: + # need a subpath as postgres wants an empty folder (the mounted folder `$PGDATA` has a `lost+found` directory) + # see https://stackoverflow.com/questions/51168558/how-to-mount-a-postgresql-volume-using-aws-ebs-in-kubernete + - name: db-data + mountPath: "/arb-data" + subpath: pgdata + diff --git a/deployment/job-calqul.yml b/deployment/job-calqul.yml index bd8edabf0f7a5f3457e18459ac7b34001b2e2855..5f986c8778f7c21d636c73c5eed3ac4534b2f016 100644 --- a/deployment/job-calqul.yml +++ b/deployment/job-calqul.yml @@ -3,12 +3,40 @@ kind: Job metadata: name: calqul-__NAMESPACE_ENV__ spec: - activeDeadlineSeconds: 144000 # maximum duration the job can run => 40h + # maximum duration the job can run => 40h + activeDeadlineSeconds: 144000 template: metadata: name: calqul-__NAMESPACE_ENV__ spec: + volumes: + - name: arb-data + persistentVolumeClaim: + claimName: pvc-01-__NAMESPACE__-claim containers: - - name: calqul-__NAMESPACE_ENV__ - image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-__NAMESPACE_ENV__ - restartPolicy: Never #OnFailure \ No newline at end of file + - name: calqul-python-__NAMESPACE_ENV__ + image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-python-__NAMESPACE_ENV__ + volumeMounts: + - name: arb-data + mountPath: "/arb-data/source-files" + # subpath: source-files + - name: arb-data + mountPath: "/arb-data/generated-files" + # subpath: out + restartPolicy: Never + + # Waiting for the Database POD to be ready. + # ~~ NOT SURE IT IS WORKING ??? ~~ + # + # initContainers: + # - name: wait-for-database + # image: busybox:latest + # imagePullPolicy: IfNotPresent + # command: ['sh', '-c', 'until nc -vz ${POD_NAME}.${POD_NAMESPACE} __POSTGRES_PORT__; do echo "Waiting for ${POD_NAME} to get ready..."; sleep 3; done;'] + # env: + # - name: POD_NAME + # value: calqul-db-__NAMESPACE_ENV__ + # - name: POD_NAMESPACE + # valueFrom: + # fieldRef: + # fieldPath: metadata.namespace \ No newline at end of file diff --git a/deployment/service-calqul.yml b/deployment/service-calqul.yml new file mode 100644 index 0000000000000000000000000000000000000000..88624bff5f5963bbcf24ed401e07cdae02abf7cf --- /dev/null +++ b/deployment/service-calqul.yml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: calqul-db-service-__NAMESPACE_ENV__ + namespace: __NAMESPACE__ +spec: + selector: + app: calqul-db-__NAMESPACE_ENV__ + tier: database + ports: + - name: http-5432 + protocol: TCP + port: 5432 + targetPort: 5432 \ No newline at end of file diff --git a/launch.sh b/launch.sh deleted file mode 100644 index ac4101e4e0548155e989eba276b64b055383d65c..0000000000000000000000000000000000000000 --- a/launch.sh +++ /dev/null @@ -1 +0,0 @@ -screen -dmS calque_metro python main.py computeAll 5 diff --git a/scripts/requirements.txt b/scripts/requirements.txt deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000