Skip to content
Snippets Groups Projects
Commit 7be84c5d authored by Pierre-Gilles Levallois's avatar Pierre-Gilles Levallois
Browse files

Cleanup code

parent c8d7c7a7
No related branches found
No related tags found
1 merge request!6Fully functional version that calculate layer with all stages
Pipeline #89347 passed
......@@ -80,22 +80,4 @@ VALUES
('Déborah', 'BESSON', 'debesson@grandlyon.com', '!PlAnT4-Gl!', 'ADMIN'),
('Hind', 'NAIT-BARKA', 'hnaitbarka@grandlyon.com', '!PlAnT4-Gl!', 'ADMIN'),
('Anaïs', 'HENRY', 'anhenry@grandlyon.com', '!PlAnT4-Gl!', 'ADMIN');
-- ('prénom', 'nom', 'p.nom@example.com', 'ExAmPlE12', 'USER'),
-- ------------------------
-- DUMMY DATA
-- ------------------------
-- INSERT INTO base.history
-- (search_name, search_location, id_user)
-- VALUES
-- ('Lyon', '43.5;25.2', 1);
-- INSERT INTO base.tiles_factors
-- (id_tile, id_factor)
-- VALUES
-- (1, 1);
-- ------------------------
-- PROCESS DATA
-- ------------------------
......@@ -2,7 +2,6 @@
# Taken from https://github.com/Toblerity/Fiona/issues/1063
##################################################################################
FROM debian:10
#buster-slim
# Set the working directory to /app
WORKDIR /app
......@@ -32,8 +31,6 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip \
&& pip install --no-cache-dir manimlib \
&& pip install --no-cache-dir manimce
RUN pip install rtree \
# pygeos
# && pip install --upgrade shapely \
&& pip install --no-cache-dir --upgrade -r ./requirements.txt
# Add a non root user (OpenShift Requirements)
......@@ -56,30 +53,12 @@ COPY --chown=calquluser:calquluser .env ./
RUN chmod 755 /app/*.sh
# Arguments
# ARG NAMESPACE_ENV
# ARG DB_HOST_ENV
# ARG DB_PORT_ENV
# ARG DB_NAME_ENV
# ARG DB_USER_ENV
# Getting some Args and Setting some ENV variables
ENV CPLUS_INCLUDE_PATH="/usr/include/gdal"
ENV C_INCLUDE_PATH="/usr/include/gdal"
# ENV PROJ_LIB="/usr/local/lib/python3.7/dist-packages/pyproj/proj_dir/share/proj"
ENV PROJ_LIB="/usr/local/lib/python3.7/dist-packages/fiona/proj_data/"
ENV GDAL_DATA="/usr/local/lib/python3.7/dist-packages/fiona/gdal_data/"
# Debug verbosity
ENV PROJ_DEBUG=3
# ENV NS_ENV=$NAMESPACE_ENV
# ENV DB_HOST="$DB_HOST_ENV-$NAMESPACE_ENV"
# ENV DB_PORT=$DB_PORT_ENV
# ENV DB_NAME=$DB_NAME_ENV
# ENV DB_USER=$DB_USER_ENV
# Launch calculation
# Documentation : https://stackoverflow.com/questions/40454470/how-can-i-use-a-variable-inside-a-dockerfile-cmd
# CMD [ "/bin/sh", "-c", "/app/launch.sh $NS_ENV $DB_HOST $DB_PORT $DB_NAME $DB_USER" ]
CMD [ "/bin/bash" ]
\ No newline at end of file
......@@ -21,6 +21,7 @@ LISTE_COMMUNES=( ["ALBIGNY-SUR-SAONE"]="69003" ["BRON"]="69029" ["CAILLOUX-SUR-F
scripts_dir="/app"
data_dir="/arb-data/source-files/data-recalcul-calque"
backup_dir="/arb-data/generated-files"
cache_dir="/arb-data/geoserver_cache"
stage=1
line="\e[39m-----------------------------------------------"
today=$(date +"%Y%m%d")
......@@ -28,10 +29,6 @@ dump_name="calque-plantabilite-$namespace_env-$today"
tag="1.0" # @TODO : should be parametric from last commit on data repo.
archive_version="v$tag-$today"
################################################################################
# functions
################################################################################
#---------------------------------------------------------------
# Functions
#---------------------------------------------------------------
......@@ -98,10 +95,6 @@ case "$action" in
comment "Action is '$action'."
check
;;
#"cleanup")
# comment "Cleanup action : all the progress tables will be truncated..."
# sed -i "s/ENABLE_TRUNCATE=False/ENABLE_TRUNCATE=True/g" .env
#;;
*)
comment "Action parameter is not recognized."
usage
......@@ -111,9 +104,14 @@ esac
if [ $action == "cleanup" ] || [ $action == "all" ]; then
stage "cleanup"
comment "Cleanup progress tables"
# this cleans up the progress tables
python3 main.py cleanup
check
comment "Cleanup tiles cache"
find $cache_dir -name "*.png" -exec rm -f {} \;
check
fi
if [ $action == "init-grid" ] || [ $action == "all" ]; then
......@@ -149,9 +147,6 @@ if [ $action == "compute-indices" ] || [ $action == "all" ]; then
check
fi
# Launching everything, it is possible to give a list of townships
# python3 main.py computeAll
if [ $action == "dump-datas" ] || [ $action == "all" ]; then
stage "Dumping result database"
#
......@@ -167,9 +162,6 @@ if [ $action == "dump-datas" ] || [ $action == "all" ]; then
tar cvzf $backup_dir/$dump_name.tgz $backup_dir/$dump_name.sql
check
# stage "Uploading archive in repo with tag $archive_version"
# comment "Upload to file server 'Geo'"
stage "Cleanup backup dir '$backup_dir'"
comment "old sql files"
find $backup_dir -name "*.sql" -exec rm -f {} \;
......
......@@ -7,16 +7,4 @@
| -5 | Erreur Base de données |
| -6 | Autre erreur |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
# apiVersion: v1
# kind: ConfigMap
# metadata:
# name: calqul-pgdump-__NAMESPACE_ENV__
# namespace: __NAMESPACE__
# labels:
# job-name: calqul-job-pgdump-__NAMESPACE_ENV__
# data:
# psqldump.sh: |
# #!/bin/bash
# BACKUPDIR="/arb-data/backup"
# ls -l /arb-data
# while [ -f /arb-data/run/python-running ]
# do
# sleep 5
# echo "Waiting for calculation to finish..."
# done
# echo "Generating a Postgres Dump..."
# pg_dump -n "__POSTGRES_SCHEMA__" __POSTGRES_DB__ > ${BACKUPDIR}/__POSTGRES_SCHEMA__-__POSTGRES_DB___-__NAMESPACE_ENV___$(date -I).sql
# echo "Deleting dumps older than 30 days"
# find ${BACKUPDIR} -name "*.sql" -mtime +30 -exec rm -rf {} \;
# sleep 300
\ No newline at end of file
# apiVersion: batch/v1
# kind: Job
# metadata:
# name: calqul-job-pgdump-__NAMESPACE_ENV__
# spec:
# # maximum duration the job can run => 1h
# activeDeadlineSeconds: 3600
# template:
# metadata:
# name: calqul-job-pgdump-__NAMESPACE_ENV__
# spec:
# volumes:
# - name: arb-data
# persistentVolumeClaim:
# claimName: pvc-03-__NAMESPACE__-claim
# - name: psqldump-volume
# configMap:
# name: calqul-pgdump-__NAMESPACE_ENV__
# defaultMode: 0755
# containers:
# - name: calqul-job-pgdump-__NAMESPACE_ENV__
# # image: "postgres:11"
# image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-db-__NAMESPACE_ENV__
# env:
# - name: POSTGRES_DB
# value: "__POSTGRES_DB__"
# - name: POSTGRES_PASSWORD
# value: "__POSTGRES_PASSWORD__"
# - name: POSTGRES_PORT
# value: "__POSTGRES_PORT__"
# - name: POSTGRES_SERVER
# value: "__POSTGRES_SERVER__"
# - name: POSTGRES_USER
# value: "__POSTGRES_USER__"
# - name: POSTGRES_SCHEMA
# value: "__POSTGRES_SCHEMA__"
# command: ["/bin/bash", "-c", "/usr/local/bin/psqldump.sh"]
# volumeMounts:
# - name: psqldump-volume
# mountPath: "/usr/local/bin"
# - name: arb-data
# mountPath: "/arb-data"
# subpath: "backup"
# restartPolicy: Never
# apiVersion: batch/v1
# kind: Job
# metadata:
# name: calqul-master-job-__NAMESPACE_ENV__
# spec:
# # maximum duration the job can run => 40h
# activeDeadlineSeconds: 144000
# completions: 1
# backoffLimit: 1
# parallelism: 1
# template:
# metadata:
# name: calqul-master-job-__NAMESPACE_ENV__
# spec:
# volumes:
# - name: arb-data
# persistentVolumeClaim:
# claimName: pvc-03-__NAMESPACE__-claim
# containers:
# - name: calqul-python-__NAMESPACE_ENV__
# image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-python-__NAMESPACE_ENV__
# env:
# - name: PGPASSWORD
# valueFrom:
# secretKeyRef:
# name: database-secret-__NAMESPACE_ENV__
# key: password
# - name: HTTP_PROXY
# value: "http://proxyhttppro.grandlyon.fr:8080"
# - name: HTTPS_PROXY
# value: "http://proxyhttppro.grandlyon.fr:8080"
# resources:
# requests:
# cpu: 1
# memory: 8000Mi
# limits:
# cpu: 1
# memory: 8000Mi
# volumeMounts:
# - name: arb-data
# mountPath: "/arb-data/source-files"
# - name: arb-data
# mountPath: "/arb-data/generated-files"
# - name: arb-data
# mountPath: "/arb-data/run" # storing a file telling export-job the state of python script
# restartPolicy: Never
# # Clone or update sources data before calculation
# initContainers:
# - name: init-data-repo
# image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-git-lfs-__NAMESPACE_ENV__
# env:
# - name: GIT_USERNAME
# valueFrom:
# secretKeyRef:
# name: arb-data-access-token
# key: username
# - name: GIT_PASSWORD
# valueFrom:
# secretKeyRef:
# name: arb-data-access-token
# key: password
# - name: HTTP_PROXY
# value: "http://proxyhttppro.grandlyon.fr:8080"
# - name: HTTPS_PROXY
# value: "http://proxyhttppro.grandlyon.fr:8080"
# imagePullPolicy: IfNotPresent
# command:
# - /bin/sh
# - -c
# - if [ -d /arb-data/source-files/data-recalcul-calque/.git ]; then cd /arb-data/source-files/data-recalcul-calque; git pull origin main; else mkdir -p /arb-data/source-files/data-recalcul-calque; cd /arb-data/source-files; git lfs clone https://${GIT_USERNAME}:${GIT_PASSWORD}@forge.grandlyon.com/erasme/sources-recalcul-calque.git ./data-recalcul-calque/; fi; echo "Data ready !";
# # - rm -rf /arb-data/source-files/data-recalcul-calque; mkdir -p /arb-data/source-files/data-recalcul-calque; cd /arb-data/source-files; git lfs clone https://arb-data-access-token:sj_-yDyc3nnvAmWmP76h@forge.grandlyon.com/erasme/sources-recalcul-calque.git ./data-recalcul-calque/; echo "Data ready !";
# volumeMounts:
# - name: arb-data
# mountPath: "/arb-data/source-files"
# - name: arb-data
# mountPath: "/arb-data"
# subpath: "run"
# apiVersion: apps/v1
# kind: Deployment
# metadata:
# name: calqul-git-lfs-__NAMESPACE_ENV__
# namespace: __NAMESPACE__
# labels:
# apps: calqul-git-lfs-__NAMESPACE_ENV__
# spec:
# replicas: 1
# selector:
# matchLabels:
# app: calqul-git-lfs-__NAMESPACE_ENV__
# role: primary
# tier: data-sources
# template:
# metadata:
# labels:
# app: calqul-git-lfs-__NAMESPACE_ENV__
# role: primary
# tier: data-sources
# spec:
# imagePullSecrets:
# - name: forge-secret
# volumes:
# - name: arb-data
# persistentVolumeClaim:
# claimName: pvc-03-__NAMESPACE__-claim
# containers:
# - name: calqul-git-lfs-__NAMESPACE_ENV__
# image: registry.forge.grandlyon.com/erasme/script-recalcul-calque/calqul-git-lfs-__NAMESPACE_ENV__
# env:
# - name: GIT_USERNAME
# valueFrom:
# secretKeyRef:
# name: arb-data-access-token
# key: username
# - name: GIT_PASSWORD
# valueFrom:
# secretKeyRef:
# name: arb-data-access-token
# key: password
# - name: HTTP_PROXY
# value: "http://proxyhttppro.grandlyon.fr:8080"
# - name: HTTPS_PROXY
# value: "http://proxyhttppro.grandlyon.fr:8080"
# command:
# - /bin/sh
# - -c
# # - sleep 3600;
# - rm -rf /arb-data/source-files/data-recalcul-calque; mkdir -p /arb-data/source-files/data-recalcul-calque; cd /arb-data/source-files/; git lfs install --force; git clone https://${GIT_USERNAME}:${GIT_PASSWORD}@forge.grandlyon.com/erasme/sources-recalcul-calque.git ./data-recalcul-calque/; cd data-recalcul-calque; git pull origin main;
# volumeMounts:
# - name: arb-data
# mountPath: "/arb-data/source-files"
# - name: arb-data
# mountPath: "/arb-data/generated-files"
......@@ -12,19 +12,3 @@ spec:
targetPort: 5432
tls:
termination: edge
# apiVersion: route.openshift.io/v1
# kind: Route
# metadata:
# name: arb-database-d01-route
# namespace: ns-arb-d01
# spec:
# host: arb-database-d01-route-ns-arb-d01.apps.air.grandlyon.fr
# to:
# kind: Service
# name: calqul-db-service-d01
# port:
# targetPort: "5432"
# tls:
# termination: edge
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment