feat: add data aggregation

manifest.konnector

@@ -27,8 +27,14 @@
   "data_types": [],
   "screenshots": [],
   "permissions": {
-    "load profile": {
-      "type": "egl.loadprofile"
+    "daily eau du grand lyon data": {
+      "type": "io.egl.day"
+    },
+    "monthly eau du grand lyon data": {
+      "type": "io.egl.month"
+    },
+    "yearly eau du grand lyon data": {
+      "type": "io.egl.year"
     },
     "files": {
       "type": "io.cozy.files"

src/index.js

@@ -14,9 +14,26 @@ moment.locale('fr') // set the language
 moment.tz.setDefault('Europe/Paris') // set the timezone
 
 const startDate = moment()
-  .subtract(10, 'day')
+  .startOf('year')
+  .subtract(3, 'year')
+  .subtract(1, 'day')
   .format('MM/DD/YYYY')
 const endDate = moment().format('MM/DD/YYYY')
+const timeRange = ['day', 'month', 'year']
+const rangeDate = {
+  day: {
+    doctype: 'io.egl.day',
+    keys: ['year', 'month', 'day']
+  },
+  month: {
+    doctype: 'io.egl.month',
+    keys: ['year', 'month']
+  },
+  year: {
+    doctype: 'io.egl.year',
+    keys: ['year']
+  }
+}
 
 module.exports = new BaseKonnector(start)
 
@@ -29,6 +46,7 @@ async function start(fields, cozyParameters) {
     // await resetData()
     const baseUrl = cozyParameters.secret.eglBaseURL
     const apiAuthKey = cozyParameters.secret.eglAPIAuthKey
+    log('debug', fields, 'Fields')
 
     log('info', 'Authenticating ...')
     const response = await authenticate(
@@ -38,14 +56,53 @@ async function start(fields, cozyParameters) {
       apiAuthKey
     )
     log('info', 'Successfully logged in')
-    log('info', 'Getting data')
-    const loadProfile = await getData(response, baseUrl, apiAuthKey)
-    log('info', 'Saving data to Cozy')
-    storeLoadProfile(loadProfile)
+    Promise.all(
+      timeRange.map(timeStep =>
+        processData(timeStep, response, baseUrl, apiAuthKey)
+      )
+    )
   } catch (error) {
     throw new Error(error.message)
   }
 }
+async function processData(timeStep, response, baseUrl, apiAuthKey) {
+  const doctype = rangeDate[timeStep].doctype
+  log('info', 'Getting data')
+  const loadProfile = await getData(response, baseUrl, apiAuthKey)
+  log('info', 'Saving data to Cozy')
+  if (doctype === rangeDate.day.doctype) {
+    log('info', 'Saving daily data')
+    await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys)
+  } else if (doctype === rangeDate.month.doctype) {
+    log('info', 'Saving monthly data')
+    await resetInProgressAggregatedData(rangeDate.month.doctype)
+    const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month)
+    await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys)
+  } else if (doctype === rangeDate.year.doctype) {
+    log('info', 'Saving yearly data')
+    await resetInProgressAggregatedData(rangeDate.year.doctype)
+    const yearlyData = processYearAggregation(
+      loadProfile,
+      rangeDate.year.doctype
+    )
+    await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys)
+  } else {
+    throw new Error('Unkonw range type: ' + doctype)
+  }
+
+  // .then(
+  //   () => {
+  //     processMonthlyAggregation(loadProfile, rangeDate.month)
+  //   }
+  // )
+
+  // const yearlyData = await processYearAggregation(
+  //   loadProfile,
+  //   rangeDate.year.doctype
+  // )
+  // storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.key)
+  // await processMonthlyAggregation(loadProfile, rangeDate.month)
+}
 
 async function authenticate(login, password, baseUrl, apiAuthKey) {
   const authRequest = {
@@ -74,6 +131,8 @@ async function authenticate(login, password, baseUrl, apiAuthKey) {
 }
 
 async function getData(response, baseUrl, apiAuthKey) {
+  log('debug', startDate, 'Start date')
+  log('debug', endDate, 'End date')
   const dataRequest = {
     method: 'POST',
     uri: baseUrl + '/getAllAgregatsByAbonnement.aspx',
@@ -108,39 +167,117 @@ async function getData(response, baseUrl, apiAuthKey) {
 
 function format(response) {
   const data = response.resultatRetour.slice(1).map((value, index) => {
-    
-    const time = moment(value.DateReleve, moment.ISO_8601).format('YYYY-MM-DD');
-
+    const time = moment(value.DateReleve, moment.ISO_8601)
     return {
-      
       load: value.ValeurIndex - response.resultatRetour[index].ValeurIndex,
       year: parseInt(time.format('YYYY')),
       month: parseInt(time.format('M')),
       day: parseInt(time.format('D')),
       hour: 0,
-      minute: 0
-      //,type: value.TypeAgregat
+      minute: 0,
+      type: value.TypeAgregat
     }
   })
   return data
 }
 
-function storeLoadProfile(loadProfile) {
-  return hydrateAndFilter(loadProfile, 'egl.loadprofile', {
-    keys: ['time']
+function processYearAggregation(data, doctype) {
+  log('info', 'Start aggregation for : ' + doctype)
+  const grouped = data.reduce(reduceYearFunction, {})
+  return Object.values(grouped)
+}
+
+function processMonthlyAggregation(data, range) {
+  log('info', 'Start aggregation for : ' + range.doctype)
+  // Filter by year
+  const tmpData = groupBy(data, 'year')
+  const keys = Object.keys(tmpData)
+  var dataToStore = []
+  // Monthly aggregation
+  for (const index in keys) {
+    // Get daily data of a year
+    var monthlyData = tmpData[keys[index]]
+    // Monthly aggregation
+    var aggregatedData = monthlyData.reduce(reduceMonthFunction, {})
+    // Store it
+    dataToStore = dataToStore.concat(Object.values(aggregatedData))
+  }
+  return dataToStore
+}
+
+function groupBy(xs, key) {
+  return xs.reduce(function(rv, x) {
+    ;(rv[x[key]] = rv[x[key]] || []).push(x)
+    return rv
+  }, {})
+}
+
+function reduceYearFunction(acc, x) {
+  var id = acc[x.year]
+  if (id) {
+    id.load += x.load
+  } else {
+    acc[x.year] = x
+  }
+  return acc
+}
+
+function reduceMonthFunction(acc, x) {
+  var id = acc[x.month]
+  if (id) {
+    id.load += x.load
+  } else {
+    acc[x.month] = x
+  }
+  return acc
+}
+
+async function storeData(data, doctype, keys) {
+  log('debug', 'Store into ' + doctype)
+  log('debug', keys, 'Store into keys ')
+  return hydrateAndFilter(data, doctype, {
+    keys: keys
   }).then(filteredDocuments => {
-    addData(filteredDocuments, 'egl.loadprofile')
+    addData(filteredDocuments, doctype)
   })
 }
 
-// eslint-disable-next-line no-unused-vars
-async function resetData() {
-  const result = await cozyClient.data.findAll('egl.loadprofile')
-  if (result.error) {
+/**
+ * Function handling special case.
+ * The temporary aggregated data need to be remove in order for the most recent one te be saved.
+ * ex for io.egl.month :
+ * { load: 76.712, month: 2020, ... } need to be replace by
+ * { load: 82.212, month: 2020, ... } after grdf data reprocess
+ */
+async function resetInProgressAggregatedData(doctype) {
+  // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
+  log('debug', doctype, 'Remove aggregated data for')
+  const result = await cozyClient.data.findAll(doctype)
+  if (result.error || result.length <= 0) {
     // eslint-disable-next-line no-console
-    console.error('Error while fetching loads')
-  }
-  for (const load of result) {
-    await cozyClient.data.delete('egl.loadprofile', load)
+    console.warn('Error while fetching loads, doctype not found ')
+  } else {
+    const currentDate = moment()
+    // Filter data to remove
+    var filtered = []
+    if (doctype === rangeDate.year.doctype) {
+      // Yearly case
+      filtered = result.filter(function(el) {
+        return el.year == currentDate.year()
+      })
+    } else {
+      // Monthly case
+      filtered = result.filter(function(el) {
+        return (
+          el.year == currentDate.year() &&
+          el.month == parseInt(moment().format('M'))
+        )
+      })
+    }
+    // Remove data
+    for (const doc of filtered) {
+      log('debug', doc, 'Removing this entry for ' + doctype)
+      await cozyClient.data.delete(doctype, doc)
+    }
   }
 }