parallelize computeFactors

Dockerfiles/python/scripts/launch.sh

@@ -13,6 +13,7 @@ DB_NAME=$5
 DB_USER=$6
 
 GRID_SIZE=5
+MAX_PARALLEL_EXEC=4
 
 declare -A LISTE_COMMUNES
 LISTE_COMMUNES=( ["ALBIGNY-SUR-SAONE"]="69003" ["BRON"]="69029" ["CAILLOUX-SUR-FONTAINES"]="69033" ["CALUIRE-ET-CUIRE"]="69034" ["CHAMPAGNE-AU-MONT-D'OR"]="69040" ["CHARBONNIERES-LES-BAINS"]="69044" ["CHARLY"]="69046" ["CHASSIEU"]="69271" ["COLLONGES-AU-MONT-D'OR"]="69063" ["CORBAS"]="69273" ["COUZON-AU-MONT-D'OR"]="69068" ["CRAPONNE"]="69069" ["CURIS-AU-MONT-D'OR"]="69071" ["DARDILLY"]="69072" ["DECINES-CHARPIEU"]="69275" ["ECULLY"]="69081" ["FEYZIN"]="69276" ["FLEURIEU-SUR-SAONE"]="69085" ["FONTAINES-SAINT-MARTIN"]="69087" ["FONTAINES-SUR-SAONE"]="69088" ["FRANCHEVILLE"]="69089" ["GENAY"]="69278" ["GIVORS"]="69091" ["GRIGNY"]="69096" ["IRIGNY"]="69100" ["JONAGE"]="69279" ["LA-MULATIERE"]="69142" ["LA-TOUR-DE-SALVAGNY"]="69250" ["LIMONEST"]="69116" ["LISSIEU"]="69117" ["LYON-1ER"]="69381" ["LYON-2EME"]="69382" ["LYON-3EME"]="69383" ["LYON-4EME"]="69384" ["LYON-5EME"]="69385" ["LYON-6EME"]="69386" ["LYON-7EME"]="69387" ["LYON-8EME"]="69388" ["LYON-9EME"]="69389" ["MARCY-L'ETOILE"]="69127" ["MEYZIEU"]="69282" ["MIONS"]="69283" ["MONTANAY"]="69284" ["NEUVILLE-SUR-SAONE"]="69143" ["OULLINS"]="69149" ["PIERRE-BENITE"]="69152" ["POLEYMIEUX-AU-MONT-D'OR"]="69153" ["QUINCIEUX"]="69163" ["RILLIEUX-LA-PAPE"]="69286" ["ROCHETAILLEE-SUR-SAONE"]="69168" ["SAINT-CYR-AU-MONT-D'OR"]="69191" ["SAINT-DIDIER-AU-MONT-D'OR"]="69194" ["SAINTE-FOY-LESLYON"]="69202" ["SAINT-FONS"]="69199" ["SAINT-GENIS-LAVAL"]="69204" ["SAINT-GENIS-LES-OLLIERES"]="69205" ["SAINT-GERMAIN-AU-MONT-D'OR"]="69207" ["SAINT-PRIEST"]="69290" ["SAINT-ROMAIN-AU-MONT-D'OR"]="69233" ["SATHONAY-CAMP"]="69292" ["SATHONAY-VILLAGE"]="69293" ["SOLAIZE"]="69296" ["TASSIN-LA-DEMI-LUNE"]="69244" ["VAULX-EN-VELIN"]="69256" ["VENISSIEUX"]="69259" ["VERNAISON"]="69260" ["VILLEURBANNE"]="69266"  )
@@ -134,13 +135,24 @@ if [ $action == "init-datas"  ] || [ $action == "all"  ]; then
 fi
 
 if [ $action == "compute-factors"  ] || [ $action == "all"  ]; then
+  # Execute several jobs in parallel
   stage "Compute Factors"
+  comment "parallelism : $MAX_PARALLEL_EXEC processes."
   for NOM_COMMUNE in $( echo "${!LISTE_COMMUNES[@]}" | tr ' ' '\n' | sort ); do
-      CODE_INSEE=${LISTE_COMMUNES[$NOM_COMMUNE]}
-      stage "Compute Factors : $NOM_COMMUNE - $CODE_INSEE"
-      python3 main.py computeFactors $CODE_INSEE
-      check
+      (
+        CODE_INSEE=${LISTE_COMMUNES[$NOM_COMMUNE]}
+        stage "Compute Factors : $NOM_COMMUNE - $CODE_INSEE"
+        python3 main.py computeFactors $CODE_INSEE
+        check
+      ) &
+      if [[ $(jobs -r -p | wc -l) -ge $MAX_PARALLEL_EXEC ]]; then
+              # now there are $N jobs already running, so wait here for any job
+              # to be finished so there is a place to start next one.
+              wait -n
+      fi;
+
   done
+
 fi
 
 if [ $action == "compute-indices"  ] || [ $action == "all"  ]; then