main.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Header
__author__ = "Romain MATIAS"
__copyright__ = "Copyright 2022, EXO-DEV x ERASME, Métropole de Lyon"
__description__ = "Script de création d'un calque de plantabilité pour la Métropole de Lyon"
__credits__ = ["Romain MATIAS", "Natacha SALMERON", "Anthony ANGELOT"]
__date__ = "06/07/2022"
__license__ = "MPL 2.0"
__version__ = "1.0.0"
__maintainer__ = "Romain MATIAS"
__email__ = "contact@exo-dev.fr ou info@erasme.org"
__status__ = "Expérimentation"

# Import
import os
import sys
from subprocess import call
from time import sleep
from dotenv import load_dotenv
import logging

import geopandas as gp
import pandas as pd
import numpy as np
from shapely.ops import unary_union
from owslib.wfs import WebFeatureService
import multiprocessing as mp
from pathos.multiprocessing import ProcessingPool as Pool

from utils import *

# Global variables
DB_params = None
DB_schema = None
PythonLaunch = None
ENV_targetProj = None
RemoveTempFile = None
SkipExistingData = None
EnableTruncate = None

# ------------------------
#           DOC
# ------------------------
# Target projection used : EPSG:2154
# Métropole default projection : EPSG:4326 or EPSG:4171
# Villeurbanne inseeCode : 69266

def showDoc():
    script_doc = """
    Welcome to The master script of plantability !

    Args:
        initCommunes                                        Insert Communes in database from a geoJSON file path (with geometry and insee column)
        initGrid <gridSize: int, inseeCode: int>            Generate with the size defined and insert Grid in database merged from a bounding box
                                                            Can be launch on certain "communes" with one <inseeCode> or in all territory by default (no parameter)
        initDatas                                           Make treatments on the datas from informations in metadatas table
        computeFactors <inseeCode: int>                     Compute the factors area on each tile with database informations. 
                                                            Can be launch on certain "communes" with one <inseeCode> or in all territory by default (no parameter)
        computeIndices                                      Compute the plantability indices on each tile with database informations. 
        computeAll <gridSize: int, listInseeCode: int>      Generate all the plantability layer (launch all previous steps). 
                                                            List of inseeCode must be separated with comma (,) and without space (e.g. python main.py 5 69266,69388,69256) 
                                                            but you can launch treatments for only one commune (e.g. python main.py 5 69266)
        help                                                Show this documentation
    """
    
    print('# ' + '=' * 78)
    print('Author: ' + __author__)
    print('Copyright: ' + __copyright__)
    print('Description: ' + __description__)
    print('Credits: ' + ', '.join(__credits__))
    print('Date: ' + __date__)
    print('License: ' + __license__)
    print('Version: ' + __version__)
    print('Maintainer: ' + __maintainer__)
    print('Email: ' + __email__)
    print('Status: ' + __status__)
    print('# ' + '=' * 78)
    
    print(script_doc)

# ------------------------
#           TEST
# ------------------------

def test(communesArrayInput=None):
    # Init
    retcode = None

    print('communesArrayInput')
    print(communesArrayInput)
    print(type(communesArrayInput))

    # communesArrayCasted = communesArrayInput.split(',')
    # print('\n communesArrayCasted')
    # print(communesArrayCasted)
    # print(type(communesArrayCasted))

    multiProcessFactors(communesArrayInput)

    return retcode

# ------------------------
#         COMMUNES
# ------------------------

def initCommunes():
    # Log & timer
    debugLog(style.YELLOW, "Launch initialisation script of \'Communes\'", logging.INFO)
    communesTimer = startTimerLog('Init communes')

    # Column list for GDF & DB
    columnsListToDB = ('libelle', 'insee', 'geom_poly')
    columnsArrFromGeoJSON = ['nom', 'insee', 'geometry']

    # Check if Communes already exist in db
    communesCount = getCountfromDB(DB_params, DB_schema, 'communes')

    if communesCount > 0 :
        debugLog(style.YELLOW, "/!\ Some Communes already exist in database", logging.INFO)

        if SkipExistingData == 'True':
                # Log
                debugLog(style.MAGENTA, "Init communes was skipped", logging.INFO)
                endTimerLog(communesTimer)
                return

        # Ask user to clean table ?
        if EnableTruncate:
            while True:
                removeCommunesResponse = input("Do you want to clean the Communes table ? (y/n) : ")
                if removeCommunesResponse.lower() not in ('y', 'n'):
                    print(style.RED + "Sorry, wrong response... \n", style.RESET)
                else:
                    # Good response
                    break

            if removeCommunesResponse.lower() == 'y':
                # Connect DB
                conn, cur = connectDB(DB_params)

                # Truncate COMMUNES
                resetCommunesQuery = "TRUNCATE TABLE "+ DB_schema + ".communes RESTART IDENTITY; COMMIT;"
                cur.execute(resetCommunesQuery)
                debugLog(style.GREEN, "Successfully remove all communes", logging.INFO)

                # Close DB
                closeDB(conn, cur)
            else:
                debugLog(style.YELLOW, "Init communes was skipped", logging.INFO)
                endTimerLog(communesTimer)
                return

    # Log
    debugLog(style.YELLOW, "Table " + DB_schema + ".communes is ready", logging.INFO)

    # Ask user geojson path ?
    # DEBUG LOCAL : "./file_data/communes_gl.geojson"
    while True:
        filePathResponse = input("Please enter the geoJSON file path (with geometry and insee column) : ")
        if not filePathResponse:
            print(style.RED + "Please enter an awnser... \n", style.RESET)
        else:
            # Good response
            break

    # Load geojson file in Dataframe
    communesGDF = createGDFfromGeoJSON(filePathResponse)

    if communesGDF is not None:
        # Clean useless attribute
        communesGDF = communesGDF[columnsArrFromGeoJSON]
        
        # Check input projection and reproj in 2154
        communesGDF = checkAndReproj(communesGDF, ENV_targetProj)

        # Convert to WKT
        communesGDF = convertGeomToWKT(communesGDF)

        # Insert in DB
        insertGDFintoDB(DB_params, DB_schema, communesGDF, 'communes', columnsListToDB)

    # Log end script
    debugLog(style.YELLOW, "End initialisation of \'Communes\'", logging.INFO)
    endTimerLog(communesTimer)

# ------------------------
#           GRID
# ------------------------

def initGrid(gridSize=30, inseeCode=None):
    # Log & Timer
    logInseeSuffix = ''
    if inseeCode:
        logInseeSuffix = "and with inseeCode {}".format(inseeCode)
    debugLog(style.YELLOW, "Launch initialisation script of \'Grid\' with size {}x{} {}".format(gridSize, gridSize, logInseeSuffix), logging.INFO)
    gridTimer = startTimerLog('Generate Grid')

    # Check tiles length (with insee filter)
    tilesInseeFilter = None
    if inseeCode:
        tilesInseeFilter = "insee = '" + str(inseeCode) + "'"
    tilesCount = getCountfromDB(DB_params, DB_schema, 'tiles', tilesInseeFilter)

    # Check for drop Tiles
    if tilesCount > 0:
        logInseeSuffix = ''
        if inseeCode:
            logInseeSuffix = "for inseeCode {}".format(inseeCode)
        debugLog(style.YELLOW, "/!\ Some Tiles already exist in database {}".format(logInseeSuffix), logging.INFO)

        if SkipExistingData == 'True':
            # Log
            debugLog(style.MAGENTA, "Init tiles was skipped {}".format(logInseeSuffix), logging.INFO)
            endTimerLog(gridTimer)
            return

        # Ask user to clean table ?
        if EnableTruncate:
            while True:
                removeTilesResponse = input("Do you want to clean the Tiles table ? (y/n) : ")
                if removeTilesResponse.lower() not in ('y', 'n'):
                    print(style.RED + "Sorry, wrong response... \n", style.RESET)
                else:
                    # Good response
                    break

            if removeTilesResponse.lower() == 'y':
                # Connect DB
                conn, cur = connectDB(DB_params)

                # Clean tiles
                resetTilesQuery = "TRUNCATE TABLE " + DB_schema + ".tiles RESTART IDENTITY; COMMIT;"
                cur.execute(resetTilesQuery)
                debugLog(style.GREEN, "Successfully remove all tiles", logging.INFO)

                # Close DB
                closeDB(conn, cur)

    # Log
    debugLog(style.YELLOW, "Table " + DB_schema + ".tiles is ready", logging.INFO)

    # Check communes length
    communeFilter = None
    if inseeCode:
        communeFilter = "insee = '" + str(inseeCode) + "'"
    communesCount = getCountfromDB(DB_params, DB_schema, 'communes', communeFilter)

    if communesCount > 0:
        debugLog(style.YELLOW, "Table " + DB_schema + ".communes is ready", logging.INFO)
    else:
        debugLog(style.RED, "There is no communes to merge with the Grid. Please launch initCommunes first", logging.INFO)
        return

    # Get Communes data
    query = "SELECT insee, geom_poly as geom FROM " + DB_schema + ".communes"
    if inseeCode:
        query = query + " WHERE " + communeFilter
    
    communesGDF = getGDFfromDB(DB_params, query, ENV_targetProj)

    # Create Grid
    gridGDF = createGridFromDF(communesGDF, gridSize)

    # Convert Grid into WKT
    wktGrid = convertGeomToWKT(gridGDF)
    
    # Grid column list
    columnsListToDB = ('geom_poly', 'insee')

    # Insert grid init
    insertGDFintoDB(DB_params, DB_schema, wktGrid, 'tiles', columnsListToDB)

    # End timer
    endTimerLog(gridTimer)
    
    # End Communes script
    debugLog(style.YELLOW, "End initialisation of \'Grid\'", logging.INFO)

# ------------------------
#      PROCESS DATA
# ------------------------

def initDatas():
    # Log & timer
    debugLog(style.YELLOW, "Launch initialisation script of \'Datas\'", logging.INFO)
    currInitDatasTimer = startTimerLog('InitDatas process')

    # Check length of metadatas
    metaDataCount = getCountfromDB(DB_params, DB_schema, 'metadatas')
    if metaDataCount < 1:
        debugLog(style.RED, "Not metadatas was found in database. Please relaunch this script after inserting one", logging.ERROR)
        return
    
    # Get all MetaData
    metaQuery = "SELECT * FROM " + DB_schema + ".metadatas ORDER BY id"
    metaDatas = getDatafromDB(DB_params, metaQuery)

    # Convert dict results to JSON
    jsonMD = json.loads(metaDatas)

    # Loop in result
    for currMData in jsonMD:
        # Init var
        currMDataID = currMData['id']
        currMDataName = currMData['name']
        currMDataListScript = currMData['script_path']
        currMDataListFactors = currMData['factors_list']

        # Log & timer
        debugLog(style.YELLOW, 'Begin process for current metadata \'' + currMDataName + '\'', logging.INFO)
        currMDTimer = startTimerLog(currMDataName + ' process')

        # Check existing data for this metadata
        qFilter = 'id_metadata = ' + str(currMDataID)
        currMDDataCount = getCountfromDB(DB_params, DB_schema, 'datas', qFilter)
        
        # Check count for drop Data
        if currMDDataCount > 0:
            debugLog(style.YELLOW, "/!\ Some datas already exist for this metadata in database. Analyze each factor datas...", logging.INFO)

            factorCountMissing = False
            for currMDFactor in currMDataListFactors:
                # Get count data for factor in this MD
                factorFilter = 'id_factor = ' + str(currMDFactor)
                currFactorDataCount = getCountfromDB(DB_params, DB_schema, 'datas', factorFilter)

                if currFactorDataCount == 0:
                    factorCountMissing = True

            # Skip if env variable is enable and all factors have datas
            if not factorCountMissing and SkipExistingData == 'True':
                # Log
                debugLog(style.MAGENTA, "Current metadata \'" + currMDataName + "\' was skipped", logging.INFO)
                # End timer
                endTimerLog(currMDTimer)
                # Skip this item in loop
                continue

            # Ask user to clean table ?
            if EnableTruncate:
                while True:
                    removeDataResponse = input("Do you want to clean those data ? (y/n) : ")
                    if removeDataResponse.lower() not in ('y', 'n'):
                        print(style.RED + "Sorry, wrong response... \n", style.RESET)
                    else:
                        # Good response
                        break

                if removeDataResponse.lower() == 'y':
                    # Delete DATAS for current metadata
                    deleteQFilter = "id_metadata = " + str(currMDataID)
                    deleteDataInDB(DB_params, DB_schema, 'datas', deleteQFilter)
                    # Log
                    debugLog(style.GREEN, "Successfully remove all datas for \'" + currMDataName + "\'", logging.INFO)
                else:
                    # Ask user to skip this metadata ? (if not deleted)
                    while True:
                        skipResponse = input("Do you want to skip this metadata \'" + currMDataName + "\' ? (y/n) : ")
                        if skipResponse.lower() not in ('y', 'n'):
                            print(style.RED + "Sorry, wrong response... \n", style.RESET)
                        else:
                            # Good response
                            break

                    if skipResponse.lower() == 'y':
                        # Log
                        debugLog(style.MAGENTA, "Current metadata \'" + currMDataName + "\' was skipped", logging.INFO)
                        # End timer
                        endTimerLog(currMDTimer)
                        # Skip this item in loop
                        continue

        # Init GDF
        currentGDF = None
        if currMData['temp_file_path']:
            # Load source data from file
            debugLog(style.BLUE, 'Load method for \'' + currMDataName + '\' : local geoJSON or SHP file', logging.INFO)
            currentGDF = createGDFfromGeoJSON("./file_data/" + currMData['temp_file_path'])
            
            # Check input projection and reproj in 2154
            currentGDF = checkAndReproj(currentGDF, ENV_targetProj)

        elif currMData['source_url'] and currMData['source_name']:
            #TODO: check API format is geoJSON (API geometry) ??
            # OR Load source data from API
            debugLog(style.BLUE, 'Load method for \'' + currMDataName + '\' : external API', logging.INFO)
            currentGDF = wfs2gp_df(currMData['source_name'], currMData['source_url'], reprojMetro=True, targetProj=ENV_targetProj)
        else:
            debugLog(style.RED, "Incorrect or no load method was found for this metadata \'" + currMDataName + "\'. Skipped... ", logging.ERROR)
            continue

        ### IS_EMPTY ? ###
        currentGDF = checkAndDeleteEmptyGeomFromGDF(currentGDF)

        ### IS_VALID ? ###
        currentGDF = checkAndRepairGeomFromGDF(currentGDF)

        # Get now date and format
        dt = getMinNowDate()

        # Define temp data file name
        tempFileName1 = './tmp/temp_' + text_to_id(currMDataName) + "_" + dt + ".shp"
        tempFileName2 = './tmp/temp_' + text_to_id(currMDataName) + "_" + dt + ".dbf"
        tempFileName3 = './tmp/temp_' + text_to_id(currMDataName) + "_" + dt + ".shx"
        tempFileName4 = './tmp/temp_' + text_to_id(currMDataName) + "_" + dt + ".cpg"
        tempFileName5 = './tmp/temp_' + text_to_id(currMDataName) + "_" + dt + ".prj"

        # Export timer
        exportTimer = startTimerLog('Export data')

        # Write data in geoJSON temp file
        if currMDataListScript and len(currMDataListScript) > 0:
            currentGDF.to_file(tempFileName1)

        # End export timer
        endTimerLog(exportTimer)
        
        # Init retcode
        retcode = None

        # Init factor
        currFactorId = None

        # Launch specifics scripts for this metadata
        if currMDataListScript:
            for currScript in currMDataListScript:
                # Check script if existing
                if os.path.exists(currScript):
                    try:
                        # Init var
                        currScriptShort = currScript.split('.')[0]

                        # Get currScript index
                        currScriptIndex = currMDataListScript.index(currScript)
                        
                        # Define id_factor
                        currFactorId = currMDataListFactors[currScriptIndex]

                        # Log
                        debugLog(style.YELLOW, 'Trying to launch subscript \'' + currScript + '\' for current metadata \'' + currMDataName + '\' ', logging.INFO)

                        # Define script result file name
                        tempResultFileName1 = './tmp/temp_result_' + text_to_id(currMDataName) + "_" + currScriptShort + "_" + dt + ".shp"
                        tempResultFileName2 = './tmp/temp_result_' + text_to_id(currMDataName) + "_" + currScriptShort + "_" + dt + ".dbf"
                        tempResultFileName3 = './tmp/temp_result_' + text_to_id(currMDataName) + "_" + currScriptShort + "_" + dt + ".shx"
                        tempResultFileName4 = './tmp/temp_result_' + text_to_id(currMDataName) + "_" + currScriptShort + "_" + dt + ".cpg"
                        tempResultFileName5 = './tmp/temp_result_' + text_to_id(currMDataName) + "_" + currScriptShort + "_" + dt + ".prj"

                        # Call subscript with filename in argv
                        retcode = call(PythonLaunch + " " + currScript + " " + tempFileName1 + " " + tempResultFileName1, timeout=99999999, shell=True)

                        if -retcode < 0:
                            debugLog(style.RED, "Child process was terminated by signal {}".format(-retcode), logging.ERROR)
                            debugLog(style.RED, "Temporary files was keeped", logging.ERROR)
                            debugLog(style.RED, 'Try to fix the error above and relauch this script for this metadata', logging.ERROR)
                            
                            # Skip current Metadata
                            debugLog(style.MAGENTA, "Current metadata \'" + currMDataName + "\' was skipped", logging.INFO)
                            continue
                        else:
                            debugLog(style.GREEN, "Child process returned {}".format(retcode), logging.INFO)

                            # Log success
                            debugLog(style.YELLOW, 'End of subscript \'' + currScript + '\' for \'' + currMDataName + '\' ', logging.INFO)

                            # Reload GDF from tempResultFileName1
                            resultGDF = createGDFfromGeoJSON(tempResultFileName1)

                            # Insert datas after script treatments
                            insertMDDatas(resultGDF, currMDataID, currFactorId)

                            #TODO: Refacto with array of files to delete (push)
                            # Remove temp files
                            if RemoveTempFile == 'True':
                                # Source
                                if os.path.exists(tempFileName1):
                                    os.remove(tempFileName1)
                                if os.path.exists(tempFileName2):
                                    os.remove(tempFileName2)
                                if os.path.exists(tempFileName3):
                                    os.remove(tempFileName3)
                                if os.path.exists(tempFileName4):
                                    os.remove(tempFileName4)
                                if os.path.exists(tempFileName5):
                                    os.remove(tempFileName5)
                                # Result
                                if os.path.exists(tempResultFileName1):
                                    os.remove(tempResultFileName1)
                                if os.path.exists(tempResultFileName2):
                                    os.remove(tempResultFileName2)
                                if os.path.exists(tempResultFileName3):
                                    os.remove(tempResultFileName3)
                                if os.path.exists(tempResultFileName4):
                                    os.remove(tempResultFileName4)
                                if os.path.exists(tempResultFileName5):
                                    os.remove(tempResultFileName5)
                                debugLog(style.GREEN, "Temporary files was successfully removed", logging.INFO)

                    except OSError as e:
                        debugLog(style.RED, "Execution failed : {} {}".format(e, sys.stderr), logging.ERROR)
                        debugLog(style.RED, "Temporary files was keeped", logging.ERROR)
                else:
                    debugLog(style.RED, "The sub script \'" + currScript + "\' was not found in this path", logging.ERROR)

        # End loop in Metadatas

        # Insert data if no subscript was launched
        if not currMDataListScript:
            # Define id_factor
            currFactorId = currMDataListFactors[0]

            insertMDDatas(currentGDF, currMDataID, currFactorId)

        # End timer
        endTimerLog(currMDTimer)

    # Log end script
    debugLog(style.YELLOW, "End initialisation of \'Datas\'", logging.INFO)
    endTimerLog(currInitDatasTimer)

def insertMDDatas(df, id_metadata, id_factor):
    # Init var
    multiExist = False
    
    # Check if one of geom is MULTI
    allGeomType = df.geom_type
    for geomT in allGeomType:
        if geomT == 'MultiPolygon' or geomT == 'MultiLineString':
            multiExist = True

    # Explode if multi geom
    if multiExist:
        # Clean attributes (keep only geometry)
        currentGeoSerie = df.loc[:,'geometry']
        allGeoSerie = currentGeoSerie.explode(index_parts=False)
        currGDF = gp.GeoDataFrame(allGeoSerie)
    else:
        # Clean attributes (keep only geometry)
        currentGeoSerie = df.loc[:,'geometry']
        currGDF = gp.GeoDataFrame(currentGeoSerie)

    # Add id_metadata to currentGDF
    currGDF = currGDF.assign(id_metadata=id_metadata)

    # Assign id_factor
    currGDF = currGDF.assign(id_factor=id_factor)

    # Transform to WKT
    currGDF = convertGeomToWKT(currGDF)

    # Column list for GDF & DB
    columnsListToDB = ('geom_poly', 'id_metadata', 'id_factor')

    #TODO: Check if geometry is Polygon before insert ?

    # Insert with foreign_key metadata_id
    insertGDFintoDB(DB_params, DB_schema, currGDF, 'datas', columnsListToDB)

def computeFactors(inseeCode=None):
    # Log
    logInseeSuffix = ''
    if inseeCode:
        logInseeSuffix = "for commune '{}'".format(inseeCode)
    debugLog(style.YELLOW, "Launch compute process for factors {}".format(logInseeSuffix), logging.INFO)
    computeTimer = startTimerLog('Compute factors {}'.format(logInseeSuffix))
    
    # Check if inseeCode is numeric
    if inseeCode and not inseeCode.isdigit():
        debugLog(style.RED, "The inseeCode argument is not a number. Please correct your input", logging.INFO)
        return

    # Get all tiles (filtered by insee if input)
    tilesQuery = 'SELECT id, geom_poly as geom, insee, indice FROM ' + DB_schema + '.tiles'
    if inseeCode:
        tilesQuery = tilesQuery + ' WHERE insee = ' + inseeCode
    # Get Tiles from DB
    tilesGDF = getGDFfromDB(DB_params, tilesQuery, ENV_targetProj)

    # Check empty data for all table
    if len(tilesGDF) == 0:
        # Check length depend on inseeCode
        if inseeCode:
            debugLog(style.YELLOW, "There is no tiles data for this inseeCode : {}. Please verify your input".format(inseeCode), logging.INFO)
            return
        else:
            debugLog(style.YELLOW, "There is no data in tiles table. Make sure you have launch this script with initGrid argument before", logging.INFO)
            return

    # Get all factors
    factorsQuery = "SELECT * FROM " + DB_schema + ".factors ORDER BY id"
    allFactors = getDatafromDB(DB_params, factorsQuery)
    # Convert dict results to JSON
    jsonFactors = json.loads(allFactors)

    # Get each Factor data in GDF
    for currFactor in jsonFactors:
        # Current Factor var
        currFactorID = currFactor['id']
        currFactorName = currFactor['name']

        # Check count
        currTFDataCount = 0
        if inseeCode:
            # Check TILES_FACTORS existing data (with insee)
            queryFactorAndInsee = "SELECT count(*) FROM base.tiles_factors tf INNER JOIN base.tiles t ON tf.id_tile = t.id AND t.insee = '{}' WHERE id_factor = {};".format(inseeCode, currFactorID)
            currTFDataFAI = getDatafromDB(DB_params, queryFactorAndInsee)
            currTFDataCount = json.loads(currTFDataFAI)[0]['count']
        else:
            # Check TILES_FACTORS existing data
            qFilter = 'id_factor = ' + str(currFactorID)
            currTFDataCount = getCountfromDB(DB_params, DB_schema, 'tiles_factors', qFilter)

        # Check count for tiles_factors Data
        if currTFDataCount > 0:
            debugLog(style.YELLOW, "/!\ Some datas (tiles_factors & area) already exist for the factor \'" + currFactorName + "\' in database", logging.INFO)

            # Skip factors with existing data in TILES_FACTORS
            if SkipExistingData == 'True':
                # Log
                debugLog(style.MAGENTA, "Current factor \'" + currFactorName + "\' was skipped", logging.INFO)
                continue

            # Ask user to clean table ?
            if EnableTruncate:
                while True:
                    removeDataResponse = input("Do you want to clean those datas ? (y/n) : ")
                    if removeDataResponse.lower() not in ('y', 'n'):
                        print(style.RED + "Sorry, wrong response... \n", style.RESET)
                    else:
                        # Good response
                        break

                if removeDataResponse.lower() == 'y':
                    if inseeCode:
                        # DELETE TILES_FACTORS data for current tiles linked to inseeCode AND id_factor
                        deleteTFQuery = "DELETE FROM base.tiles_factors WHERE id_tile IN ( SELECT id FROM base.tiles t WHERE t.insee = {} ) AND id_factor = {};".format(str(inseeCode), currFactorID)
                        deleteCustomDataInDB(DB_params, deleteTFQuery)
                    else:
                        # DELETE TILES_FACTORS datas with id_factor
                        deleteQFilter = "id_factor = " + str(currFactorID)
                        deleteDataInDB(DB_params, DB_schema, 'tiles_factors', deleteQFilter)
                    

                    # Log
                    debugLog(style.GREEN, "Successfully remove all TILES_FACTORS datas for \'" + currFactorName + "\' ", logging.INFO)

        # Log & timer per factor
        currFactorTimer = startTimerLog("Compute factor " + currFactorName)

        # Get data from DB
        dataFQuery = "SELECT geom_poly as geom, id_metadata, id_factor FROM " + DB_schema + ".datas WHERE id_factor = " + str(currFactorID)
        currFDataGDF = getGDFfromDB(DB_params, dataFQuery, ENV_targetProj)

        # Get commmune geom to filter data
        communesGDF = None
        if inseeCode:
            query = "SELECT insee, geom_poly as geom FROM " + DB_schema + ".communes WHERE insee = '" + inseeCode + "'"
            communesGDF = getGDFfromDB(DB_params, query, ENV_targetProj)

            # Filter data with commune geom (if insee)
            currFDataGDF = gp.overlay(communesGDF, currFDataGDF, how='intersection', keep_geom_type=False)

            # Log
            debugLog(style.MAGENTA, 'Intersect overlap end successfully with \'{}\' entities keeped'.format(len(currFDataGDF)), logging.INFO)

        if len(currFDataGDF) > 1:
            # Union timer
            unionTimer = startTimerLog('Union datas')
            
            # Regroup
            unionGeom = unary_union(currFDataGDF.geometry)
            dataUnion = [{'geometry': unionGeom}]
            unionFactorGDF = gp.GeoDataFrame(dataUnion, crs=ENV_targetProj)
            #TODO: + Clean and repair geom ??

            # End union timer
            endTimerLog(unionTimer)
        else:
            unionFactorGDF = currFDataGDF

        # Clip timer
        clipTimer = startTimerLog('Clip datas with tiles')

        # Intersect & cut data with tiles geom (clip)
        interFData = tilesGDF.clip(unionFactorGDF)
        
        # End clip timer
        endTimerLog(clipTimer)

        # Make result GDF
        interFGDF = gp.GeoDataFrame(interFData, crs=ENV_targetProj)

        # Log success
        debugLog(style.YELLOW, 'Successfully match factor \'{}\' datas with tiles. Found {} entites / {} tiles '.format(currFactorName, len(interFGDF), len(tilesGDF)), logging.INFO)

        debugLog(style.YELLOW, 'Calculating area for current factor tiles and insert in database', logging.INFO)

        # Connect to DB
        conn, cur = connectDB(DB_params, jsonEnable=True)

        # Loop in all RESULT cutFactor Geom (with tiles info)
        for index, row in interFGDF.iterrows():
            # Craft row item in dict format
            rowDict = row.to_dict()

            # Convert into GDF
            rowGDF = gp.GeoDataFrame([rowDict], geometry="geom", crs=ENV_targetProj)

            # Get current Tile id
            currTileID = rowGDF.iloc[0]['id']

            # Calculate area
            currFactorCutGeom = rowGDF.iloc[0]['geom']
            cutFactorArea = currFactorCutGeom.area
            
            # Round result
            roundCutFactorArea = round(cutFactorArea)

            # DEBUG Log
            # debugLog(style.YELLOW, "Tiles n°{} as : {} m² of \'{}\' ".format(currTileID, roundCutFactorArea, currFactorName), logging.INFO)

            # Insert area into TILES_FACTOR (with id_tile & id_factor)
            insertTileFactorQuery = "INSERT INTO " + DB_schema + ".tiles_factors (id_tile, id_factor, area) VALUES (" + str(currTileID) + "," + str(currFactorID) + "," + str(roundCutFactorArea) + "); COMMIT;"
            insertDataInDB(cur, insertTileFactorQuery)

            ##End of current cutFactor (tile) loop

        # Close cursor & DB connexion
        closeDB(conn, cur)

        # Log
        debugLog(style.GREEN, 'Successfully insert all informations and area in database', logging.INFO)
        
        # Ending log
        endTimerLog(currFactorTimer)

        ##End of current factor loop

    # Log & timer end script
    endTimerLog(computeTimer)
    debugLog(style.YELLOW, "End of computing factors process", logging.INFO)

def multiComputeFactors(communesSplitedArray):
    # Re-init env var (cause to multiprocessing)
    initEnv()

    # For each commune (in splited array)
    for currCommune in communesSplitedArray:
        # Compute current commune Factors
        computeFactors(currCommune)

    return

def multiProcessFactors(communesArrayInput=None):
    # Init var
    communesArray = []

    # Check input for Communes array
    if communesArrayInput:
        # Set real array
        communesArray = communesArrayInput
    else:
        # Get Communes data
        comQuery = "SELECT insee FROM " + DB_schema + ".communes"
        communesData = getDatafromDB(DB_params, comQuery)
        # Convert dict results to JSON
        jsonCommunes = json.loads(communesData)

        # Convert jsonArray to array
        for currCommune in jsonCommunes:
            # Init var
            currComInsee = currCommune['insee']
            # Append
            communesArray.append(currComInsee)

    # Get cores number
    cores=mp.cpu_count()

    # Split array by core
    communesSplit = splitList(communesArray, cores)
    # communesSplit = np.array_split(communesArray, cores, axis=0)

    # Create the multiprocessing pool
    pool = Pool(cores)

    # Multiprocess splited array
    df_out = np.vstack(pool.map(multiComputeFactors, communesSplit))

    # Close down the pool and join
    pool.close()
    pool.join()
    pool.clear()

    return

def computeIndices():
    # Log
    debugLog(style.YELLOW, "Start computing indice", logging.INFO)
    computeIndiceTimer = startTimerLog("Compute indice")

    # Global connect to DB & json cursor
    conn, cur = connectDB(DB_params)

    # Get TILES_FACTORS count
    tfCount = getCountfromDB(DB_params, DB_schema, 'tiles_factors', None, conn, cur)

    # Check empty data for TILES_FACTORS table
    if tfCount == 0:
        debugLog(style.RED, "There is no data in tiles_factors table. Make sure you have launch this script with computeFactors argument before", logging.ERROR)
        return

    # Get TILES count
    tCount = getCountfromDB(DB_params, DB_schema, 'tiles', None, conn, cur)

    # Check empty data for TILES table
    if tCount == 0:
        debugLog(style.RED, "There is no data in tiles table. Make sure you have launch this script with initGrid argument before", logging.ERROR)
        return

    # Launch SQL Query
    updateIndiceQuery = "UPDATE base.tiles t SET indice = sub.sum_indice FROM (SELECT id_tile, ROUND(SUM(area * f.ponderation)/100::numeric,1) AS sum_indice FROM base.tiles_factors tf JOIN base.factors f ON tf.id_factor = f.id GROUP BY id_tile) as sub WHERE t.id = sub.id_tile; COMMIT;"
    cur.execute(updateIndiceQuery)
    debugLog(style.GREEN, "Successfully update indice in all tiles", logging.INFO)

    # Global close connexion to DB
    closeDB(conn, cur)
    
    # Log
    endTimerLog(computeIndiceTimer)
    debugLog(style.YELLOW, "End computing indice", logging.INFO)

# ------------------------
#   GENERATE ALL CALQUE
# ------------------------

def computeAll(gridSize=30, communesArrayInput=None):
    # Log and timer
    debugLog(style.YELLOW, "Start computing all plantability layer", logging.INFO)
    computeLayerTimer = startTimerLog("Compute all layer")

    # Check and launch initCommunes
    initCommunes()

    jsonCommunes = None
    if communesArrayInput and len(communesArrayInput):
        # Convert communes Array to string list for SQL query
        strCommunesArray = "','".join(communesArrayInput)
        # Get communes filtered by insee
        comQuery = "SELECT insee FROM " + DB_schema + ".communes WHERE insee IN ('" + strCommunesArray + "')"
        communesData = getDatafromDB(DB_params, comQuery)
        # Convert dict results to JSON
        jsonCommunes = json.loads(communesData)
    else:
        # Get Communes data
        comQuery = "SELECT insee FROM " + DB_schema + ".communes"
        communesData = getDatafromDB(DB_params, comQuery)
        # Convert dict results to JSON
        jsonCommunes = json.loads(communesData)

    # Loop in communes
    indexCommune = 1
    lenCommunes = len(jsonCommunes)
    for currCommune in jsonCommunes:
        # Init var
        currComInsee = currCommune['insee']

        # Log
        debugLog(style.YELLOW, "Process grid for commune {} ({}/{})".format(currComInsee, indexCommune, lenCommunes), logging.INFO)

        # Check and launch initGrid
        initGrid(gridSize, currComInsee)

        # Increment index
        indexCommune = indexCommune + 1

    # Check and launch initDatas
    initDatas()

    # Launch multiprocessing to compute Factors
    if communesArrayInput and len(communesArrayInput):
        multiProcessFactors(communesArrayInput)
    else:
        multiProcessFactors()

    # Check and launch computeIndices
    computeIndices()
    
    # End timer
    endTimerLog(computeLayerTimer)
    debugLog(style.YELLOW, "End computing all plantability layer", logging.INFO)

    return

# ------------------------
#           ENV
# ------------------------

def initEnv():
    # Get db .env params & init global var
    global DB_params
    global DB_schema
    global PythonLaunch
    global ENV_targetProj
    global RemoveTempFile
    global SkipExistingData
    global EnableTruncate

    # Assign values
    DB_params = {
        "host"      : os.getenv('DB_HOST'),
        "user"      : os.getenv('DB_USER'),
        "password"  : os.getenv('DB_PWD'),
        "database"  : os.getenv('DB_NAME'),
    }
    DB_schema = os.getenv('DB_SCHEMA')
    PythonLaunch = os.getenv('PYTHON_LAUNCH')
    ENV_targetProj = os.getenv('TARGET_PROJ')
    RemoveTempFile = os.getenv('REMOVE_TEMP_FILE')
    SkipExistingData = os.getenv('SKIP_EXISTING_DATA')
    EnableTruncate = os.getenv('ENABLE_TRUNCATE')

# ------------------------
#          MAIN
# ------------------------

# @memory(percentage=0.8)
def main():
    # Launch function depend on argv
    argv = sys.argv[1:]
    firstArgv = None

    # Argv exist ?
    if argv:
        firstArgv = sys.argv[1:][0]

        # Switch case...
        if firstArgv == 'initCommunes':
            initCommunes()
        elif firstArgv == 'initGrid':
            secArgv = None
            thirdArgv = None
            # Test argv2 exist
            try:
                # Get gridSize (argv 2)
                secArgv = sys.argv[1:][1]
            except(Exception) as error:
                debugLog(style.RED, "Please input the grid size wanted as last argument", logging.ERROR)
            
            if secArgv:
                # Test argv3 exist
                try:
                    # Get inseeCode (argv 3)
                    thirdArgv = sys.argv[1:][2]
                except(Exception) as error:
                    debugLog(style.YELLOW, "/!\ This script will generate grid in all territory")

                if secArgv.isdigit():
                    # Launch function
                    initGrid(int(secArgv), thirdArgv)
                else:
                    debugLog(style.RED, "The grid size value is not a number. Please correct your input", logging.ERROR)
        elif firstArgv == 'initDatas':
            initDatas()
        elif firstArgv == 'computeFactors':
            secArgv = None
            if len(sys.argv[1:]) > 1:
                secArgv = sys.argv[1:][1]
                # Cast to list
                communesArrayCasted = secArgv.split(',')
                # One or more communes ?
                if len(communesArrayCasted) > 1:
                    multiProcessFactors(communesArrayCasted)
                else:
                    computeFactors(secArgv)
            else:
                # All communes
                debugLog(style.YELLOW, "/!\ This script will compute factors in all territory")
                multiProcessFactors()
        elif firstArgv == 'computeIndices':
            computeIndices()
        elif firstArgv == 'computeAll':
            secArgv = None
            thirdArgv = None
            # Test argv2 exist
            try:
                # Get gridSize (argv 2)
                secArgv = sys.argv[1:][1]
            except(Exception) as error:
                debugLog(style.RED, "Please input the grid size wanted as last argument")
            
            if secArgv:
                if secArgv.isdigit():
                    # Test argv3 exist
                    try:
                        # Get inseeCode (argv 3)
                        thirdArgv = sys.argv[1:][2]
                        # Cast to list
                        thirdArgv = thirdArgv.split(',')