diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8bb706a50b5e4557830fc12788576de581802dc3..27740528975565d92de52d193879187c801a410d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,14 +9,14 @@ # le 'stage' de déploiement en production est forcément différent du fait qu'il DOIT être manuel variables: - TRIGRAMME: test #arb @TODO : Change after when the right namespace is available + TRIGRAMME: arb REPLICA_ENV: 1 ################################################################################ # Anchors ################################################################################ .build_and_push_image: &build_and_push_image - - docker build --pull --tag "$CI_REGISTRY_IMAGE/$IMAGE_NAME:latest" . + - docker build --pull --tag "$CI_REGISTRY_IMAGE/$IMAGE_NAME:latest" --build-arg NAMESPACE_ENV=$NAMESPACE_ENV --build-arg DB_HOST_ENV=$POSTGRES_SERVER --build-arg DB_PORT_ENV=$POSTGRES_PORT --build-arg DB_NAME_ENV=$POSTGRES_DB --build-arg DB_USER_ENV=$POSTGRES_USER . - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:latest" # Docker Authentification anchor @@ -25,7 +25,6 @@ variables: # Set_env : copying env sample file and replace values .set_env: &set_env - - cp .env.example .env - echo "# Configuration generated by Gitlab CI. (#$CI_BUILD_ID)" > .env - echo "#" >> .env - cat .env.example >> .env @@ -42,24 +41,34 @@ variables: - export KUBECONFIG=$KUBECONFIG_DEV - echo "Preparing Deployment on $NAMESPACE ($NAMESPACE_ENV)" + # Secrets + - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/secrets/database-secret.yml + + # PVC Access + - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/pods/pvc-calqul.yml + - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/pods/pvc-calqul.yml + # DB deployment file - - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/db-calqul.yml - - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/db-calqul.yml - - sed -i "s/__POSTGRES_DB__/$POSTGRES_DB/g" deployment/db-calqul.yml - - sed -i "s/__POSTGRES_PASSWORD__/$POSTGRES_PASSWORD/g" deployment/db-calqul.yml - - sed -i "s/__POSTGRES_PORT__/$POSTGRES_PORT/g" deployment/db-calqul.yml - - sed -i "s/__POSTGRES_SERVER__/$POSTGRES_SERVER-$NAMESPACE_ENV/g" deployment/db-calqul.yml - - sed -i "s/__POSTGRES_USER__/$POSTGRES_USER/g" deployment/db-calqul.yml - - sed -i "s/__POSTGRES_SCHEMA__/$POSTGRES_SCHEMA/g" deployment/db-calqul.yml + - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/pods/db-calqul.yml + - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/pods/db-calqul.yml + - sed -i "s/__POSTGRES_DB__/$POSTGRES_DB/g" deployment/pods/db-calqul.yml + - sed -i "s/__POSTGRES_PASSWORD__/$POSTGRES_PASSWORD/g" deployment/pods/db-calqul.yml + - sed -i "s/__POSTGRES_PORT__/$POSTGRES_PORT/g" deployment/pods/db-calqul.yml + - sed -i "s/__POSTGRES_SERVER__/$POSTGRES_SERVER-$NAMESPACE_ENV/g" deployment/pods/db-calqul.yml + - sed -i "s/__POSTGRES_USER__/$POSTGRES_USER/g" deployment/pods/db-calqul.yml + - sed -i "s/__POSTGRES_SCHEMA__/$POSTGRES_SCHEMA/g" deployment/pods/db-calqul.yml # DB service file - - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/service-calqul.yml - - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/service-calqul.yml + - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/services/db-calqul.yml + - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/services/db-calqul.yml # Job deployment file - - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/job-calqul.yml - - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/job-calqul.yml - - sed -i "s/__POSTGRES_PORT__/$POSTGRES_PORT/g" deployment/job-calqul.yml + - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/jobs/master-calqul.yml + - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/jobs/master-calqul.yml + + # Git-lfs deployment file For test only + # - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/pods/git-calqul.yml + # - sed -i "s/__NAMESPACE_ENV__/$NAMESPACE_ENV/g" deployment/pods/git-calqul.yml # Git-lfs deployment file - sed -i "s/__NAMESPACE__/$NAMESPACE/g" deployment/git-calqul.yml @@ -67,19 +76,31 @@ variables: # Applying openshift conf anchor .apply_confs: &apply_confs + # Secrets + - oc delete secret database-secret-$NAMESPACE_ENV + - oc apply -f deployment/secrets/database-secret.yml + + # PVC Access + - oc delete pods --selector pod-name=calqul-pvc-access-$NAMESPACE_ENV #--grace-period=0 --force + - oc apply -f deployment/pods/pvc-calqul.yml + # Db - oc delete pods --selector pod-name=calqul-db-$NAMESPACE_ENV #--grace-period=0 --force - - oc apply -f deployment/db-calqul.yml - - oc apply -f deployment/service-calqul.yml + - oc apply -f deployment/pods/db-calqul.yml + - oc apply -f deployment/services/db-calqul.yml + + # Git-lfs for test only + # - oc delete pods --selector pod-name=calqul-git-lfs-$NAMESPACE_ENV #--grace-period=0 --force + # - oc apply -f deployment/pods/git-calqul.yml # Git-lfs # - oc delete pods --selector pod-name=calqul-git-lfs-$NAMESPACE_ENV #--grace-period=0 --force # - oc apply -f deployment/git-calqul.yml # Job - - oc delete jobs --selector job-name=calqul-job-$NAMESPACE_ENV - - oc create -f deployment/job-calqul.yml - + - oc delete jobs --selector job-name=calqul-master-job-$NAMESPACE_ENV + - oc create -f deployment/jobs/master-calqul.yml + ################################################################################ # CI/CD Stages ################################################################################ @@ -173,8 +194,7 @@ Deploy Calqul: rules: - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH tags: - #- ns-arb-d01 @TODO : Change after when the right namespace is available - - ns-test-d01 # No possibility to put variables here :( + - ns-arb-d01 # ********************************************************************************************* # * MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN BRANCH MAIN * @@ -195,7 +215,6 @@ Deploy Calqul Production: only: - main tags: - #- ns-arb-d01 @TODO : Change after when the right namespace is available - # In production, the namespace will be different, as we can't to put variables here :( - # we need anorher stage only for procudtion banch. - - ns-test-d01 + # In production, the namespace will be different, as we can't put variables here :(, + # we will need anorher stage only for procudtion branch. + - ns-arb-d01 diff --git a/Dockerfiles/db/Dockerfile b/Dockerfiles/db/Dockerfile index da39ae6fda1818159124fb900bcc431da65b411b..f7e3a4bb96ac28f7803fc56d2ff52e25d9ec29d4 100644 --- a/Dockerfiles/db/Dockerfile +++ b/Dockerfiles/db/Dockerfile @@ -1,7 +1,10 @@ # # ARB::Base de données postgis pour le calcul du calque de plantabilité # -FROM postgis/postgis +FROM postgis/postgis:11-3.3-alpine + +# Arguments +ARG NAMESPACE_ENV USER root diff --git a/Dockerfiles/db/db-init/20_init_arb.sh b/Dockerfiles/db/db-init/20_init_arb.sh index f27cbca971d17d9e9958024e60de0140669799b2..1dd178ad0e72aa1992de999e004850408ca33419 100644 --- a/Dockerfiles/db/db-init/20_init_arb.sh +++ b/Dockerfiles/db/db-init/20_init_arb.sh @@ -4,15 +4,22 @@ # Working Directory cd /docker-entrypoint-initdb.d +# id => uid=1001230000(1001230000) gid=0(root) groups=1001230000 +userid=$(id | cut -d'(' -f1 | cut -d'=' -f2) echo "--------------------------------------------------------------------------" -echo " --> Boostraping de la base de donnees ARB" +echo " --> Boostraping de la base de donnees '$POSTGRES_DB'" echo " le repertoire est '$(pwd)'" echo "--------------------------------------------------------------------------" # Mettre ici le le schéma d'initailisation, les reprises de données, reéation de roles et grants, etc... -echo "POSTGRES_DB=$POSTGRES_DB" -echo "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" -echo "POSTGRES_PORT=$POSTGRES_PORT" -echo "POSTGRES_SERVER=$POSTGRES_SERVER" -echo "POSTGRES_USER=$POSTGRES_USER" -echo "POSTGRES_SCHEMA=$POSTGRES_SCHEMA" +# psql -U $POSTGRES_USER -d $POSTGRES_DB -c "CREATE ROLE $userid" + +psql -U $POSTGRES_USER -d $POSTGRES_DB -f /docker-entrypoint-initdb.d/sql/30-create_tables.sql +psql -U $POSTGRES_USER -d $POSTGRES_DB -f /docker-entrypoint-initdb.d/sql/40-insert_data.sql + +# echo "POSTGRES_DB=$POSTGRES_DB" +# echo "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" +# echo "POSTGRES_PORT=$POSTGRES_PORT" +# echo "POSTGRES_SERVER=$POSTGRES_SERVER" +# echo "POSTGRES_USER=$POSTGRES_USER" +# echo "POSTGRES_SCHEMA=$POSTGRES_SCHEMA" diff --git a/Dockerfiles/db/db-init/30-create_tables.sql b/Dockerfiles/db/db-init/sql/30-create_tables.sql similarity index 100% rename from Dockerfiles/db/db-init/30-create_tables.sql rename to Dockerfiles/db/db-init/sql/30-create_tables.sql diff --git a/Dockerfiles/db/db-init/40-insert_data.sql b/Dockerfiles/db/db-init/sql/40-insert_data.sql similarity index 100% rename from Dockerfiles/db/db-init/40-insert_data.sql rename to Dockerfiles/db/db-init/sql/40-insert_data.sql diff --git a/Dockerfiles/git/Dockerfile b/Dockerfiles/git/Dockerfile index 72c95a3b315edd2cda5442d67891908c49a75c22..39bb34b75e92c70de6effa7f4aaaf18f6cb4ac24 100644 --- a/Dockerfiles/git/Dockerfile +++ b/Dockerfiles/git/Dockerfile @@ -1,5 +1,8 @@ FROM alpine/git:v2.36.2 +# Arguments +ARG NAMESPACE_ENV + RUN apk add git-lfs RUN git lfs install && \ diff --git a/Dockerfiles/python/.env.example b/Dockerfiles/python/.env.example index 2913a7752394055874ffc9dbac251f626d0a668a..14a12421cf721d4ba3e2aea58a2b7722b7bfa2bb 100644 --- a/Dockerfiles/python/.env.example +++ b/Dockerfiles/python/.env.example @@ -16,4 +16,4 @@ COMMUNES_FILE_PATH="/arb-data/source-files/data-recalcul-calque/communes_gl.geoj TARGET_PROJ="EPSG:2154" REMOVE_TEMP_FILE=False SKIP_EXISTING_DATA=True -ENABLE_TRUNCATE=False \ No newline at end of file +ENABLE_TRUNCATE=False diff --git a/Dockerfiles/python/Dockerfile b/Dockerfiles/python/Dockerfile index b22e4d116686b2389441b3c9463391a62d04ada2..6941d34f5cee4d227f7e7c3348d2cc8f997b4042 100644 --- a/Dockerfiles/python/Dockerfile +++ b/Dockerfiles/python/Dockerfile @@ -1,62 +1,71 @@ -################################################################################## -# Taken from https://github.com/Toblerity/Fiona/issues/1063 -################################################################################## -FROM debian:buster-slim - -# Getting some Args and Setting some ENV variables -ENV CPLUS_INCLUDE_PATH="/usr/include/gdal" -ENV C_INCLUDE_PATH="/usr/include/gdal" -ENV PROJ_LIB="/usr/local/lib/python3.7/dist-packages/pyproj/proj_dir/share/proj" -ENV GDAL_DATA="/usr/local/lib/python3.7/dist-packages/fiona/gdal_data/" - -# Basic libs and install python3.7 -RUN apt-get update && apt-get upgrade -y \ -&& apt-get install -y procps wget python3.7 python3-pip gdal-bin libgdal-dev apt-utils git \ -&& apt-get clean \ -apt-get -y install proj proj-dev && \ -# https://stackoverflow.com/questions/70508775/error-could-not-build-wheels-for-pycairo-which-is-required-to-install-pyprojec -apt-get install -y libcairo2 libcairo2-dev && \ -################################# Debug and diagnose tools ################################# -# apt-get install -qy apt-utils vim iputils-ping net-tools telnet curl postgresql-client &&\ -############################################################################################ -rm -rf /var/lib/apt/lists/* - -COPY ./scripts/requirements.txt requirements.txt - -# Pip -RUN python3 -m pip install --no-cache-dir --upgrade pip \ -&& pip install --no-cache-dir --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` \ -&& pip install --no-cache-dir --ignore-installed PyYAML \ -# https://stackoverflow.com/questions/70508775/error-could-not-build-wheels-for-pycairo-which-is-required-to-install-pyprojec -&& pip install --no-cache-dir manimlib \ -&& pip install --no-cache-dir manimce \ -# -&& pip install --no-cache-dir --upgrade -r requirements.txt - -# Add a non root user (OpenShift Requirements) -RUN addgroup calquluser && \ -useradd calquluser -g calquluser --home /app && \ -mkdir -p /app && \ -mkdir -p /app/logs && \ -mkdir -p /app/tmp && \ -chown -R calquluser:calquluser /app && \ -chmod 777 /app/logs /app/tmp - - -WORKDIR /app - -USER calquluser - -# Add app -COPY --chown=calquluser:calquluser ./scripts/*.py ./ -COPY --chown=calquluser:calquluser --chmod= ./scripts/*.sh ./ -COPY --chown=calquluser:calquluser ./sql ./ -COPY --chown=calquluser:calquluser .env ./ - -RUN chmod 755 ./*.sh - -# Launch calculation -CMD [ "/bin/bash", "./launch.sh" ] - -# launch testing script -# CMD [ "/bin/bash", "./test-env.sh" ] +################################################################################## +# Taken from https://github.com/Toblerity/Fiona/issues/1063 +################################################################################## +FROM debian:buster-slim + +# Arguments +ARG NAMESPACE_ENV +ARG DB_HOST_ENV +ARG DB_PORT_ENV +ARG DB_NAME_ENV +ARG DB_USER_ENV + +# Getting some Args and Setting some ENV variables +ENV CPLUS_INCLUDE_PATH="/usr/include/gdal" +ENV C_INCLUDE_PATH="/usr/include/gdal" +ENV PROJ_LIB="/usr/local/lib/python3.7/dist-packages/pyproj/proj_dir/share/proj" +ENV GDAL_DATA="/usr/local/lib/python3.7/dist-packages/fiona/gdal_data/" + +ENV NS_ENV=$NAMESPACE_ENV +ENV DB_HOST="$DB_HOST_ENV-$NAMESPACE_ENV" +ENV DB_PORT=$DB_PORT_ENV +ENV DB_NAME=$DB_NAME_ENV +ENV DB_USER=$DB_USER_ENV + +# Basic libs and install python3.7 +RUN apt-get update && apt-get upgrade -y \ +&& apt-get install -y procps wget python3.7 python3-pip gdal-bin libgdal-dev apt-utils git \ +&& apt-get clean \ +apt-get -y install proj proj-dev && \ +# https://stackoverflow.com/questions/70508775/error-could-not-build-wheels-for-pycairo-which-is-required-to-install-pyprojec +apt-get install -y libcairo2 libcairo2-dev && \ +################################# Debug and diagnose tools ################################# +apt-get install -qy apt-utils vim iputils-ping net-tools telnet curl && \ +apt-get install -qy postgresql-client && \ +############################################################################################ +rm -rf /var/lib/apt/lists/* + +COPY ./scripts/requirements.txt requirements.txt + +# Pip +RUN python3 -m pip install --no-cache-dir --upgrade pip \ +&& pip install --no-cache-dir --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` \ +&& pip install --no-cache-dir --ignore-installed PyYAML \ +# https://stackoverflow.com/questions/70508775/error-could-not-build-wheels-for-pycairo-which-is-required-to-install-pyprojec +&& pip install --no-cache-dir manimlib \ +&& pip install --no-cache-dir manimce \ +&& pip install --no-cache-dir --upgrade -r requirements.txt + +# Add a non root user (OpenShift Requirements) +RUN addgroup calquluser && \ +useradd calquluser -g calquluser --home /app && \ +mkdir -p /app && \ +mkdir -p /app/logs && \ +mkdir -p /app/tmp && \ +chown -R calquluser:calquluser /app && \ +chmod 777 /app/logs /app/tmp /app + +WORKDIR /app +USER calquluser + +# Add app +COPY --chown=calquluser:calquluser ./scripts/*.py ./ +COPY --chown=calquluser:calquluser ./scripts/*.sh ./ +COPY --chown=calquluser:calquluser ./sql ./ +COPY --chown=calquluser:calquluser .env ./ + +RUN chmod 755 ./*.sh + +# Launch calculation +# Documentation : https://stackoverflow.com/questions/40454470/how-can-i-use-a-variable-inside-a-dockerfile-cmd +CMD [ "/bin/sh", "-c", "/app/launch.sh $NS_ENV $DB_HOST $DB_PORT $DB_NAME $DB_USER" ] \ No newline at end of file diff --git a/Dockerfiles/python/scripts/launch.sh b/Dockerfiles/python/scripts/launch.sh index 32c4f94e67be394eacb8ce3f52c8518e7b4dd9f1..1134f0b1a0ccf885903138f0aa972c465c7f6bcc 100644 --- a/Dockerfiles/python/scripts/launch.sh +++ b/Dockerfiles/python/scripts/launch.sh @@ -2,13 +2,24 @@ ################################################################################ # ARB : Lancement du recalcul des indices du calque de plantabilité. ################################################################################ +namespace_env=$1 +DB_HOST=$2 +DB_PORT=$3 +DB_NAME=$4 +DB_USER=$5 + DATA_REPO="https://forge.grandlyon.com/erasme/data-recalcul-calque.git" scripts_dir="/app" data_dir="/arb-data/source-files/data-recalcul-calque" +backup_dir="/arb-data/generated-files" stage=1 line="\e[39m-----------------------------------------------" need_update=1 +today=$(date +"%Y%m%d") +dump_name="calque-plantabilite-$namespace_env-$today" +tag="1.0" # @TODO : should be parametric from last commit on data repo. +archive_version="v$tag-$today" ################################################################################ # functions @@ -40,8 +51,33 @@ check () { fi; } -stage "Launch Initializations..." +# Overloading 'Exit' builtin function to get rid of +# the running state every where in the code +exit () { + error_code=$1 + echo "Exiting '$error_code'. (Sleeping for 1h for debug purpose)" + sleep 3600 + builtin exit $error_code +} + +#--------------------------------------------------------------- +# M A I N +#--------------------------------------------------------------- +stage "Launch ENV Initializations..." cd $scripts_dir + +# All the needed variables a given by parameter passing +comment "command line is '$0 $namespace_env $DB_HOST $DB_PORT $DB_NAME $DB_USER'" + +comment "psql version..." +psql -V +check + +comment "Postgres server says : " +pg_isready -d $DB_NAME -h $DB_HOST -p $DB_PORT -U $DB_USER +check + +stage "Launch Database Initializations..." comment "Init communes" python3 main.py initCommunes comment "Init Grid" @@ -49,9 +85,7 @@ python3 main.py initGrid 5 comment "InitDatas" python3 main.py initDatas - stage "Launch Computations..." - comment "Computing factors" python3 main.py computeFactors # Possibly Multiprocessing task, Should have a list of townships @@ -61,5 +95,31 @@ python3 main.py computeIndices # Launching everything, it is possible to give a list of townships # python3 main.py computeAll -stage "Sleeping a while for debug purpose..." -sleep 3600 \ No newline at end of file +stage "Dumping result database" +# +# Option "--no-password" is set not to have to provide password by prompt. +# This requires the presnece of /root/.pgpass file (600 mode) with such a content : "hostname:port:database:username:password" +# https://stackoverflow.com/questions/50404041/pg-dumpall-without-prompting-password +# +comment "pg_dump -n base -h ${DB_HOST} -U ${DB_USER} --no-password --clean --if-exists --file=$backup_dir/$dump_name.sql ${DB_NAME}" +pg_dump -n base -h ${DB_HOST} -U ${DB_USER} --no-password --clean --if-exists --file=$backup_dir/$dump_name.sql ${DB_NAME} +check + +comment "Commpressing dump as $dump_name.tgz" +tar cvzf $backup_dir/$dump_name.tgz $backup_dir/$dump_name.sql +check + +stage "Uploading archive in repo with tag $archive_version" +comment "Upload to file server 'Geo'" + +stage "Cleanup backup dir '$backup_dir'" +comment "old sql files" +find $backup_dir -name "*.sql" -exec rm -f {} \; +check + +comment "old tgz files" +find $backup_dir -name "*.tgz" -mtime +5 -exec rm -f {} \; +check + +stage "End of script." +exit 0 diff --git a/Dockerfiles/python/scripts/main.py b/Dockerfiles/python/scripts/main.py index f18b195a035513569c27db66cc25059fdddec27a..2a21b9f01807cdf5332e5da584709376cab6d2b8 100644 --- a/Dockerfiles/python/scripts/main.py +++ b/Dockerfiles/python/scripts/main.py @@ -947,7 +947,7 @@ def main(): if argv: firstArgv = sys.argv[1:][0] - # Switch case... + # Sfs2itch case... if firstArgv == 'initCommunes': initCommunes() elif firstArgv == 'initGrid': diff --git a/Dockerfiles/python/scripts/utils.py b/Dockerfiles/python/scripts/utils.py index 1a9c71eb26dc3a9d76da1663bd4e5377193ed324..213f83c9523aeeb01f4cc2eaf7d48d198a337758 100644 --- a/Dockerfiles/python/scripts/utils.py +++ b/Dockerfiles/python/scripts/utils.py @@ -395,7 +395,7 @@ def wfs2gp_df(layer_name, url, bbox=None, wfs_version="2.0.0", outputFormat='app # Concat params params = dict(service='WFS', version=wfs_version,request='GetFeature', typeName=layer_name, outputFormat=outputFormat, crs=targetProj) # Load data in Bytes - with BytesCollection(requests.get(url,params=params).content) as f: + with BytesCollection(requests.get(url,params=params, timeout=600).content) as f: # Make GDF df = gp.GeoDataFrame.from_features(f) diff --git a/deployment/configMaps/pgdump-calqul.yml b/deployment/configMaps/pgdump-calqul.yml new file mode 100644 index 0000000000000000000000000000000000000000..b2eaf405c04ce44bc1ab00dd9010c1045aaf8385 --- /dev/null +++ b/deployment/configMaps/pgdump-calqul.yml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: calqul-pgdump-__NAMESPACE_ENV__ + namespace: __NAMESPACE__ + labels: + job-name: calqul-job-pgdump-__NAMESPACE_ENV__ +data: + psqldump.sh: | + #!/bin/bash + BACKUPDIR="/arb-data/backup" + ls -l /arb-data + while [ -f /arb-data/run/python-running ] + do + sleep 5 + echo "Waiting for calculation to finish..." + done + echo "Generating a Postgres Dump..." + pg_dump -n "__POSTGRES_SCHEMA__" __POSTGRES_DB__ > ${BACKUPDIR}/__POSTGRES_SCHEMA__-__POSTGRES_DB___-__NAMESPACE_ENV___$(date -I).sql + echo "Deleting dumps older than 30 days" + find ${BACKUPDIR} -name "*.sql" -mtime +30 -exec rm -rf {} \; + sleep 300 \ No newline at end of file diff --git a/deployment/jobs/dump-calqul.yml b/deployment/jobs/dump-calqul.yml new file mode 100644 index 0000000000000000000000000000000000000000..99a2d0671efd21eec07431d474fde1f994c46620 --- /dev/null +++ b/deployment/jobs/dump-calqul.yml @@ -0,0 +1,48 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: calqul-job-pgdump-__NAMESPACE_ENV__ +spec: + # maximum duration the job can run => 1h + activeDeadlineSeconds: 3600 + template: + metadata: + name: calqul-job-pgdump-__NAMESPACE_ENV__ + spec: + volumes: + - name: arb-data + persistentVolumeClaim: + claimName: pvc-01-__NAMESPACE__-claim + - name: psqldump-volume + configMap: + name: calqul-pgdump-__NAMESPACE_ENV__ + defaultMode: 0755 + containers: + - name: calqul-job-pgdump-__NAMESPACE_ENV__ + # image: "postgres:11" + image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-db-__NAMESPACE_ENV__ + + env: + - name: POSTGRES_DB + value: "__POSTGRES_DB__" + - name: POSTGRES_PASSWORD + value: "__POSTGRES_PASSWORD__" + - name: POSTGRES_PORT + value: "__POSTGRES_PORT__" + - name: POSTGRES_SERVER + value: "__POSTGRES_SERVER__" + - name: POSTGRES_USER + value: "__POSTGRES_USER__" + - name: POSTGRES_SCHEMA + value: "__POSTGRES_SCHEMA__" + command: ["/bin/bash", "-c", "/usr/local/bin/psqldump.sh"] + volumeMounts: + - name: psqldump-volume + mountPath: "/usr/local/bin" + - name: arb-data + mountPath: "/arb-data" + subpath: "backup" + restartPolicy: Never + + + diff --git a/deployment/jobs/master-calqul.yml b/deployment/jobs/master-calqul.yml new file mode 100644 index 0000000000000000000000000000000000000000..8497f71b3385ae17be9d9772bd953d5b472980ab --- /dev/null +++ b/deployment/jobs/master-calqul.yml @@ -0,0 +1,75 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: calqul-master-job-__NAMESPACE_ENV__ +spec: + # maximum duration the job can run => 40h + activeDeadlineSeconds: 144000 + template: + metadata: + name: calqul-master-job-__NAMESPACE_ENV__ + spec: + volumes: + - name: arb-data + persistentVolumeClaim: + claimName: pvc-01-__NAMESPACE__-claim + containers: + - name: calqul-python-__NAMESPACE_ENV__ + image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-python-__NAMESPACE_ENV__ + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 1 + memory: 1024Mi + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: database-secret-__NAMESPACE_ENV__ + key: password + - name: http_proxy + value: "proxyhttppro.grandlyon.fr:8080" + - name: https_proxy + value: "proxyhttppro.grandlyon.fr:8080" + volumeMounts: + - name: arb-data + mountPath: "/arb-data/source-files" + - name: arb-data + mountPath: "/arb-data/generated-files" + - name: arb-data + mountPath: "/arb-data/run" # storing a file telling export-job the state of python script + restartPolicy: Never + + # Clone or update sources data before calculation + initContainers: + - name: init-data-repo + image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-git-lfs-__NAMESPACE_ENV__ + env: + - name: GIT_USERNAME + valueFrom: + secretKeyRef: + name: arb-data-access-token + key: username + - name: GIT_PASSWORD + valueFrom: + secretKeyRef: + name: arb-data-access-token + key: password + - name: HTTP_PROXY + value: "http://proxyhttppro.grandlyon.fr:8080" + - name: HTTPS_PROXY + value: "http://proxyhttppro.grandlyon.fr:8080" + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - if [ -d /arb-data/source-files/data-recalcul-calque/.git ]; then cd /arb-data/source-files/data-recalcul-calque; git pull origin main; else mkdir -p /arb-data/source-files/data-recalcul-calque; cd /arb-data/source-files; git lfs clone https://${GIT_USERNAME}:${GIT_PASSWORD}@forge.grandlyon.com/erasme/sources-recalcul-calque.git ./data-recalcul-calque/; fi; echo "Data ready !"; + # - rm -rf /arb-data/source-files/data-recalcul-calque; mkdir -p /arb-data/source-files/data-recalcul-calque; cd /arb-data/source-files; git lfs clone https://arb-data-access-token:sj_-yDyc3nnvAmWmP76h@forge.grandlyon.com/erasme/sources-recalcul-calque.git ./data-recalcul-calque/; echo "Data ready !"; + volumeMounts: + - name: arb-data + mountPath: "/arb-data/source-files" + - name: arb-data + mountPath: "/arb-data" + subpath: "run" \ No newline at end of file diff --git a/deployment/db-calqul.yml b/deployment/pods/db-calqul.yml similarity index 93% rename from deployment/db-calqul.yml rename to deployment/pods/db-calqul.yml index 0e06ff3fbcf2324889d492ea5b6cc09aef57e969..cb1cd5258dd277205b7e459c9ca7f55ab758b74d 100644 --- a/deployment/db-calqul.yml +++ b/deployment/pods/db-calqul.yml @@ -22,7 +22,7 @@ spec: imagePullSecrets: - name: forge-secret volumes: - - name: db-data + - name: arb-data persistentVolumeClaim: claimName: pvc-01-__NAMESPACE__-claim containers: @@ -35,7 +35,7 @@ spec: readinessProbe: tcpSocket: port: 5432 - initialDelaySeconds: 120 + initialDelaySeconds: 15 periodSeconds: 10 livenessProbe: tcpSocket: @@ -68,7 +68,6 @@ spec: volumeMounts: # need a subpath as postgres wants an empty folder (the mounted folder `$PGDATA` has a `lost+found` directory) # see https://stackoverflow.com/questions/51168558/how-to-mount-a-postgresql-volume-using-aws-ebs-in-kubernete - - name: db-data + - name: arb-data mountPath: "/arb-data" subpath: pgdata - diff --git a/deployment/pods/git-calqul.yml b/deployment/pods/git-calqul.yml new file mode 100644 index 0000000000000000000000000000000000000000..5338e156a62e3b36837b7c3d513e1c49c8ab6c7b --- /dev/null +++ b/deployment/pods/git-calqul.yml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calqul-git-lfs-__NAMESPACE_ENV__ + namespace: __NAMESPACE__ + labels: + apps: calqul-git-lfs-__NAMESPACE_ENV__ +spec: + replicas: 1 + selector: + matchLabels: + app: calqul-git-lfs-__NAMESPACE_ENV__ + role: primary + tier: data-sources + template: + metadata: + labels: + app: calqul-git-lfs-__NAMESPACE_ENV__ + role: primary + tier: data-sources + spec: + imagePullSecrets: + - name: forge-secret + volumes: + - name: arb-data + persistentVolumeClaim: + claimName: pvc-01-__NAMESPACE__-claim + containers: + - name: calqul-git-lfs-__NAMESPACE_ENV__ + image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-git-lfs-__NAMESPACE_ENV__ + env: + - name: GIT_USERNAME + valueFrom: + secretKeyRef: + name: arb-data-access-token + key: username + - name: GIT_PASSWORD + valueFrom: + secretKeyRef: + name: arb-data-access-token + key: password + - name: HTTP_PROXY + value: "http://proxyhttppro.grandlyon.fr:8080" + - name: HTTPS_PROXY + value: "http://proxyhttppro.grandlyon.fr:8080" + command: + - /bin/sh + - -c + - sleep 3600; + # - rm -rf /arb-data/source-files/data-recalcul-calque; mkdir -p /arb-data/source-files/data-recalcul-calque; cd /arb-data/source-files/; git lfs install --force; git clone https://${GIT_USERNAME}:${GIT_PASSWORD}@forge.grandlyon.com/erasme/sources-recalcul-calque.git ./data-recalcul-calque/; cd data-recalcul-calque; git pull origin main; + volumeMounts: + - name: arb-data + mountPath: "/arb-data/source-files" + - name: arb-data + mountPath: "/arb-data/generated-files" diff --git a/deployment/pods/pvc-calqul.yml b/deployment/pods/pvc-calqul.yml new file mode 100644 index 0000000000000000000000000000000000000000..f9fea80de8f93b84cd4f515f818e1c6b375e51c2 --- /dev/null +++ b/deployment/pods/pvc-calqul.yml @@ -0,0 +1,36 @@ +# +# Pod d'accès au PVC. Utile pour remettre l'environnement de travail à blanc, y compris les fichiers Postgres. +# +kind: Deployment +apiVersion: apps/v1 +metadata: + name: calqul-pvc-__NAMESPACE_ENV__ + namespace: __NAMESPACE__ + labels: + app: calqul-pvc-__NAMESPACE_ENV__ +spec: + replicas: 1 + selector: + matchLabels: + app: calqul-pvc-__NAMESPACE_ENV__ + role: secondary + tier: exploitation + template: + metadata: + labels: + app: calqul-pvc-__NAMESPACE_ENV__ + role: secondary + tier: exploitation + spec: + containers: + - name: calqul-pvc-__NAMESPACE_ENV__ + image: registry.gitlab.grandlyon.fr/openshift-as-code/reposit-gl/busybox:1.0 + command: ["sh", "-c", "tail -f /dev/null"] + imagePullPolicy: Always + volumeMounts: + - mountPath: /mnt/data + name: data + volumes: + - name: data + persistentVolumeClaim: + claimName: pvc-01-__NAMESPACE__-claim \ No newline at end of file diff --git a/deployment/secrets/database-secret.yml b/deployment/secrets/database-secret.yml new file mode 100644 index 0000000000000000000000000000000000000000..598e57768b1ab0b54f2c2e68bf573a6d06030348 --- /dev/null +++ b/deployment/secrets/database-secret.yml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: database-secret-__NAMESPACE_ENV__ + namespace: ns-arb-d01 +type: Opaque +data: + username: UEdQQVNTV09SRA== + password: Y2FscXVs \ No newline at end of file diff --git a/deployment/service-calqul.yml b/deployment/services/db-calqul.yml similarity index 100% rename from deployment/service-calqul.yml rename to deployment/services/db-calqul.yml