feat: add protections against EGL bad sorted data + clean code

manifest.konnector

 {
-  "version": "1.0.2",
+  "version": "1.0.3",
   "name": "EGL",
   "type": "konnector",
   "language": "node",

src/index.js

@@ -18,8 +18,12 @@ const manualExecution =
   process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false
 
 const startDate = manualExecution
-  ? moment().subtract(1, 'year').format('MM/DD/YYYY')
-  : moment().subtract(3, 'year').format('MM/DD/YYYY')
+  ? moment()
+      .subtract(1, 'year')
+      .format('MM/DD/YYYY')
+  : moment()
+      .subtract(3, 'year')
+      .format('MM/DD/YYYY')
 
 const endDate = moment().format('MM/DD/YYYY')
 // const timeRange = ['day', 'month', 'year']
@@ -45,8 +49,7 @@ module.exports = new BaseKonnector(start)
 // the account information come from ./konnector-dev-config.json file
 async function start(fields, cozyParameters) {
   try {
-    // resetting data for demo only
-    // await resetData()
+    // Local debug data
     // const baseUrl = fields.eglBaseURL
     // const apiAuthKey = fields.eglAPIAuthKey
     const baseUrl = cozyParameters.secret.eglBaseURL
@@ -149,30 +152,6 @@ async function buildAgregatedData(data, doctype) {
   return agregatedData
 }
 
-// async function processData(timeStep, response, baseUrl, apiAuthKey) {
-//   const doctype = rangeDate[timeStep].doctype;
-//   const loadProfile = await getData(response, baseUrl, apiAuthKey);
-//   log("info", "Saving data to Cozy");
-//   if (doctype === rangeDate.day.doctype) {
-//     await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys);
-//   } else if (doctype === rangeDate.month.doctype) {
-//     await resetInProgressAggregatedData(rangeDate.month.doctype);
-//     const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month);
-//     log("info", "Saving monthly data");
-//     await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys);
-//   } else if (doctype === rangeDate.year.doctype) {
-//     await resetInProgressAggregatedData(rangeDate.year.doctype);
-//     const yearlyData = processYearAggregation(
-//       loadProfile,
-//       rangeDate.year.doctype
-//     );
-//     log("info", "Saving yearly data");
-//     await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys);
-//   } else {
-//     throw new Error("Unkonw range type: " + doctype);
-//   }
-// }
-
 async function authenticate(login, password, baseUrl, apiAuthKey) {
   const authRequest = {
     method: 'POST',
@@ -214,16 +193,22 @@ async function getData(response, baseUrl, apiAuthKey) {
     json: true
   }
   try {
-    const responseEgl = await rp(dataRequest)
+    // Sort data by date
+    const responseEgl = await rp(dataRequest).then(eglRawData => {
+      eglRawData.resultatRetour.sort(function(a, b) {
+        return new Date(a.DateReleve) - new Date(b.DateReleve)
+      })
+      return eglRawData
+    })
     switch (responseEgl.codeRetour) {
       case 100:
         return format(responseEgl)
       case -2:
-        throw new Error(errors.LOGIN_FAILED)
+        throw errors.LOGIN_FAILED
       case -1:
-        throw new Error(errors.VENDOR_DOWN)
+        throw errors.VENDOR_DOWN
       default:
-        throw new Error(errors.UNKNOWN_ERROR)
+        throw errors.UNKNOWN_ERROR
     }
   } catch (error) {
     throw new Error(errors.VENDOR_DOWN)
@@ -232,90 +217,39 @@ async function getData(response, baseUrl, apiAuthKey) {
 
 function format(response) {
   log('info', 'origin response size is : ' + response.resultatRetour.length)
+  // Store first value as reference for index processing
+  let refValue = response.resultatRetour[0]
+  // Create copy of data without first value
   const data = response.resultatRetour
     .slice(1)
     .filter(value => value.ValeurIndex)
-  const dataLen = data.length
-  log('info', 'filtered size is : ' + dataLen)
-  const mapData = data.map((value, index) => {
+  log('info', 'filtered size is : ' + data.length)
+  return data.map(value => {
     const time = moment(value.DateReleve, moment.ISO_8601)
-    if (index !== 0 && index < dataLen) {
-      return {
-        load: value.ValeurIndex - data[index - 1].ValeurIndex,
-        year: parseInt(time.format('YYYY')),
-        month: parseInt(time.format('M')),
-        day: parseInt(time.format('D')),
-        hour: 0,
-        minute: 0,
-        type: value.TypeAgregat
-      }
-    } else {
-      return {
-        load: null,
-        year: parseInt(time.format('YYYY')),
-        month: parseInt(time.format('M')),
-        day: parseInt(time.format('D')),
-        hour: 0,
-        minute: 0,
-        type: value.TypeAgregat
-      }
+    const procesedLoad = value.ValeurIndex - refValue.ValeurIndex
+    if (procesedLoad < 0) {
+      log(
+        'error',
+        `processing load for day ${parseInt(time.format('D'))}/${parseInt(
+          time.format('M')
+        )}/${parseInt(time.format('YYYY'))}, value is : ${procesedLoad}`
+      )
+      throw errors.VENDOR_DOWN
+    }
+    // Change index ref value
+    refValue = value
+    return {
+      load: procesedLoad,
+      year: parseInt(time.format('YYYY')),
+      month: parseInt(time.format('M')),
+      day: parseInt(time.format('D')),
+      hour: 0,
+      minute: 0,
+      type: value.TypeAgregat
     }
   })
-  const res = [...mapData].filter(v => v.load !== null)
-  return res
 }
 
-// function processYearAggregation(data, doctype) {
-//   log("info", "Start aggregation for : " + doctype);
-//   const grouped = data.reduce(reduceYearFunction, {});
-//   return Object.values(grouped);
-// }
-
-// function processMonthlyAggregation(data, range) {
-//   log("info", "Start aggregation for : " + range.doctype);
-//   // Filter by year
-//   const tmpData = groupBy(data, "year");
-//   const keys = Object.keys(tmpData);
-//   var dataToStore = [];
-//   // Monthly aggregation
-//   for (const index in keys) {
-//     // Get daily data of a year
-//     var monthlyData = tmpData[keys[index]];
-//     // Monthly aggregation
-//     var aggregatedData = monthlyData.reduce(reduceMonthFunction, {});
-//     // Store it
-//     dataToStore = dataToStore.concat(Object.values(aggregatedData));
-//   }
-//   return dataToStore;
-// }
-
-// function groupBy(xs, key) {
-//   return xs.reduce(function(rv, x) {
-//     (rv[x[key]] = rv[x[key]] || []).push(x);
-//     return rv;
-//   }, {});
-// }
-
-// function reduceYearFunction(acc, x) {
-//   var id = acc[x.year];
-//   if (id) {
-//     id.load += x.load;
-//   } else {
-//     acc[x.year] = x;
-//   }
-//   return acc;
-// }
-
-// function reduceMonthFunction(acc, x) {
-//   var id = acc[x.month];
-//   if (id) {
-//     id.load += x.load;
-//   } else {
-//     acc[x.month] = x;
-//   }
-//   return acc;
-// }
-
 /**
  * Save data in the right doctype db and prevent duplicated keys
  */