feat: refactor + aggregation

.gitignore

@@ -28,7 +28,7 @@ desktop.ini
 .floo
 .flooignore
 .idea/
-
+.vscode/
 # Default
 # /!\ KEEP THIS SECTION THE LAST ONE
 !.gitkeep

src/aggregate.js

+// @ts-check
+const { log, cozyClient } = require('cozy-konnector-libs')
+
+/**
+ * Retrieve and remove old data for a specific doctype
+ * Return an Array of agregated data
+ */
+async function buildAgregatedData(data, doctype) {
+  let agregatedData = []
+  // eslint-disable-next-line no-unused-vars
+  for (let [key, value] of Object.entries(data)) {
+    const data = await buildDataFromKey(doctype, key, value)
+    const oldValue = await resetInProgressAggregatedData(data, doctype)
+    data.load += oldValue
+    agregatedData.push(data)
+  }
+  return agregatedData
+}
+
+/**
+ * Format an entry for DB storage
+ * using key and value
+ * For year doctype: key = "YYYY"
+ * For month doctype: key = "YYYY-MM"
+ */
+async function buildDataFromKey(doctype, key, value) {
+  let year, month, day, hour
+  if (doctype === 'com.grandlyon.enedis.year') {
+    year = key
+    month = 1
+    day = 0
+    hour = 0
+  } else if (doctype === 'com.grandlyon.enedis.month') {
+    const split = key.split('-')
+    year = split[0]
+    month = split[1]
+    day = 0
+    hour = 0
+  } else {
+    const split = key.split('-')
+    year = split[0]
+    month = split[1]
+    day = split[2]
+    hour = split[3]
+  }
+  return {
+    load: Math.round(value * 10000) / 10000,
+    year: parseInt(year),
+    month: parseInt(month),
+    day: parseInt(day),
+    hour: parseInt(hour),
+    minute: 0,
+  }
+}
+
+/**
+ * Function handling special case.
+ * The temporary aggregated data need to be remove in order for the most recent one te be saved.
+ * ex for com.grandlyon.enedis.year :
+ * { load: 76.712, year: 2020, ... } need to be replace by
+ * { load: 82.212, year: 2020, ... } after enedis data reprocess
+ */
+async function resetInProgressAggregatedData(data, doctype) {
+  // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
+  log('debug', doctype, 'Remove aggregated data for')
+  const result = await cozyClient.data.findAll(doctype)
+  if (result && result.length > 0) {
+    // Filter data to remove
+    var filtered = []
+    if (doctype === 'com.grandlyon.enedis.year') {
+      // Yearly case
+      filtered = result.filter(function(el) {
+        return el.year == data.year
+      })
+    } else if (doctype === 'com.grandlyon.enedis.month') {
+      // Monthly case
+      filtered = result.filter(function(el) {
+        return el.year == data.year && el.month == data.month
+      })
+    } else {
+      // Hourly case
+      filtered = result.filter(function(el) {
+        return (
+          el.year == data.year &&
+          el.month == data.month &&
+          el.day == data.day &&
+          el.hour == data.hour
+        )
+      })
+    }
+    // Remove data
+    let sum = 0.0
+    // eslint-disable-next-line no-unused-vars
+    for (const doc of filtered) {
+      sum += doc.load
+      log('debug', doc, 'Removing this entry for ' + doctype)
+      await cozyClient.data.delete(doctype, doc)
+    }
+    return sum
+  }
+  return 0.0
+}
+
+module.exports = {
+  buildAgregatedData,
+}

src/index.js

@@ -9,6 +9,8 @@ const soapRequest = require('easy-soap-request')
 const moment = require('moment')
 require('moment-timezone')
 const xml2js = require('xml2js')
+const { buildAgregatedData } = require('./aggregate')
+const { userTechnicalData, userMesureDetailles } = require('./request')
 moment.locale('fr') // set the language
 moment.tz.setDefault('Europe/Paris') // set the timezone
 
@@ -19,8 +21,7 @@ let startDailyDate = manualExecution
   ? moment().subtract(12, 'month')
   : moment().subtract(6, 'month')
 let startDailyDateString = startDailyDate.format('YYYY-MM-DD')
-// const startLoadDate = moment().subtract(7, 'day')
-// const startLoadDateString = startLoadDate.format('YYYY-MM-DD')
+const startLoadDate = moment().subtract(7, 'day')
 const endDate = moment()
 const endDateString = endDate.format('YYYY-MM-DD')
 
@@ -46,8 +47,7 @@ async function start(fields, cozyParameters) {
   //TODO: authentification ?
   log('info', 'Successfully logged in')
 
-  //TODO: get compteur start data
-
+  log('info', 'Querying data...')
   await getDataStartDate(
     `${baseUrl}/enedis_SGE_ConsultationDonneesTechniquesContractuelles/1.0`,
     apiAuthKey,
@@ -60,7 +60,13 @@ async function start(fields, cozyParameters) {
     loginUtilisateur,
     fields.pointId
   )
-  log('info', 'Konnector process end')
+  await getDataHalfHour(
+    `${baseUrl}/enedis_SGE_ConsultationMesuresDetaillees/1.0`,
+    apiAuthKey,
+    loginUtilisateur,
+    fields.pointId
+  )
+  log('info', 'Querying data: done')
 }
 /**
  *
@@ -98,7 +104,7 @@ async function getDataStartDate(url, apiAuthKey, userLogin, pointId) {
 }
 
 /**
- *
+ * Get hour data
  * @param {string} url
  * @param {string} apiAuthKey
  * @param {string} userLogin
@@ -111,6 +117,17 @@ async function getData(url, apiAuthKey, userLogin, pointId) {
     apikey: apiAuthKey,
   }
 
+  // If start date exceed the maximum amount of data we can get with one query
+  // get only 24 month
+  if (moment(endDate).diff(startDailyDate, 'months', true) > 24) {
+    log(
+      'info',
+      'Start date exceed 24 month, setting start date to current date minus 24 month'
+    )
+    startDailyDate = moment(endDate).subtract(24, 'month')
+    startDailyDateString = startDailyDate.format('YYYY-MM-DD')
+  }
+
   const { response } = await soapRequest({
     url: url,
     headers: sampleHeaders,
@@ -137,6 +154,62 @@ async function getData(url, apiAuthKey, userLogin, pointId) {
   )
 }
 
+/**
+ * Get half-hour data
+ * @param {string} url
+ * @param {string} apiAuthKey
+ * @param {string} userLogin
+ * @param {number} pointId
+ */
+async function getDataHalfHour(url, apiAuthKey, userLogin, pointId) {
+  log('info', 'Fetching data')
+  const sampleHeaders = {
+    'Content-Type': 'text/xml;charset=UTF-8',
+    apikey: apiAuthKey,
+  }
+
+  let MAX_HISTO = 4
+  // If manual execution, retrieve only 1 week
+  if (manualExecution) {
+    MAX_HISTO = 1
+  }
+  for (var i = 0; i < MAX_HISTO; i++) {
+    log('info', 'launch process with history')
+    const increamentedStartDateString = moment(startLoadDate)
+      .subtract(7 * i, 'day')
+      .format('YYYY-MM-DD')
+    const incrementedEndDateString = moment(endDate)
+      .subtract(7 * i, 'day')
+      .format('YYYY-MM-DD')
+    const { response } = await soapRequest({
+      url: url,
+      headers: sampleHeaders,
+      xml: userMesureDetailles(
+        pointId,
+        userLogin,
+        increamentedStartDateString,
+        incrementedEndDateString,
+        'COURBE',
+        'PA'
+      ),
+    }).catch(err => {
+      log('error', 'userMesureDetailles half-hour')
+      log('error', err)
+      return err
+    })
+
+    xml2js.parseString(
+      response.body,
+      {
+        tagNameProcessors: [parseTags],
+        valueProcessors: [parseValue],
+        explicitArray: false,
+      },
+      processData('com.grandlyon.enedis.minute')
+    )
+  }
+}
+
 /**
  * Format tag in order to be manipulated easly
  * @param {string} name
@@ -165,8 +238,10 @@ function parseValue(value, name) {
 
 /**
  * Parse data
+ * @param {string} doctype
+ * @returns
  */
-function processData() {
+function processData(doctype = 'com.grandlyon.enedis.day') {
   return async (err, result) => {
     if (err) {
       log('error', err)
@@ -174,11 +249,18 @@ function processData() {
     }
     // Return only needed part of info
     const data = parseSgeXmlData(result)
-    return storeData(
+    const processedDailyData = await storeData(
       await formateDataForDoctype(data),
-      'com.grandlyon.enedis.day',
-      ['year', 'month', 'day']
+      doctype,
+      ['year', 'month', 'day', 'hour', 'minute']
     )
+
+    log('info', 'Agregate enedis daily data for month and year')
+    if (doctype === 'com.grandlyon.enedis.day') {
+      console.log(processedDailyData.length)
+
+      await agregateMonthAndYearData(processedDailyData)
+    }
   }
 }
 
@@ -237,66 +319,8 @@ async function storeData(data, doctype, filterKeys) {
   const filteredDocuments = await hydrateAndFilter(data, doctype, {
     keys: filterKeys,
   })
-  return await addData(filteredDocuments, doctype)
-}
-
-/**
- * Query SGE in order to get info
- * @param {number} pointId
- * @param {string} userLogin
- * @param {string} startDt
- * @param {string} endDt
- * @returns {string}
- */
-function userMesureDetailles(pointId, userLogin, startDt, endDt) {
-  log('info', `Query data between ${startDt} and ${endDt}`)
-  return `<?xml version='1.0' encoding='utf-8'?>
-  <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
-     xmlns:v2="http://www.enedis.fr/sge/b2b/services/consultationmesuresdetaillees/v2.0"
-     xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0">
-     <soapenv:Header/>
-     <soapenv:Body>
-        <v2:consulterMesuresDetaillees>
-           <demande>
-              <initiateurLogin>${userLogin}</initiateurLogin>
-              <pointId>${pointId}</pointId>
-              <mesuresTypeCode>ENERGIE</mesuresTypeCode>
-              <grandeurPhysique>EA</grandeurPhysique>
-              <soutirage>true</soutirage>
-              <injection>false</injection>
-              <dateDebut>${startDt}</dateDebut>
-              <dateFin>${endDt}</dateFin>
-              <mesuresCorrigees>false</mesuresCorrigees>
-              <accordClient>true</accordClient>
-           </demande>
-        </v2:consulterMesuresDetaillees>
-     </soapenv:Body>
-  </soapenv:Envelope>
-  `
-}
-
-/**
- * Get user technical data
- * @param {string} pointId
- * @param {string} userLogin
- * @returns {string}
- */
-function userTechnicalData(pointId, userLogin) {
-  log('info', `Query userMesureDetailles`)
-  return `<?xml version='1.0' encoding='utf-8'?>
-  <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
-     xmlns:v2="http://www.enedis.fr/sge/b2b/services/consulterdonneestechniquescontractuelles/v1.0"
-     xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0">
-     <soapenv:Header/>
-     <soapenv:Body>
-        <v2:consulterDonneesTechniquesContractuelles>
-           <pointId>${pointId}</pointId>
-           <loginUtilisateur>${userLogin}</loginUtilisateur>
-           <autorisationClient>true</autorisationClient>
-        </v2:consulterDonneesTechniquesContractuelles>
-     </soapenv:Body>
-  </soapenv:Envelope>
-  `
+  await addData(filteredDocuments, doctype)
+  return filteredDocuments
 }
 
 /**
@@ -306,10 +330,8 @@ function userTechnicalData(pointId, userLogin) {
  */
 async function formateDataForDoctype(data) {
   log('info', 'Formating data')
-  // record
   return data.map(record => {
     let date = moment(record.d, 'YYYY/MM/DD h:mm:ss')
-
     return {
       load: record.v,
       year: parseInt(date.format('YYYY')),
@@ -320,3 +342,37 @@ async function formateDataForDoctype(data) {
     }
   })
 }
+
+/**
+ * Agregate data from daily data to monthly and yearly data
+ */
+async function agregateMonthAndYearData(data) {
+  // Sum year and month values into object with year or year-month as keys
+  if (data && data.length > 0) {
+    let monthData = {}
+    let yearData = {}
+    data.forEach(element => {
+      element.year + '-' + element.month in monthData
+        ? (monthData[element.year + '-' + element.month] += element.load)
+        : (monthData[element.year + '-' + element.month] = element.load)
+      element.year in yearData
+        ? (yearData[element.year] += element.load)
+        : (yearData[element.year] = element.load)
+    })
+    // Agregation for Month data
+    const agregatedMonthData = await buildAgregatedData(
+      monthData,
+      'com.grandlyon.enedis.month'
+    )
+    await storeData(agregatedMonthData, 'com.grandlyon.enedis.month', [
+      'year',
+      'month',
+    ])
+    // Agregation for Year data
+    const agregatedYearData = await buildAgregatedData(
+      yearData,
+      'com.grandlyon.enedis.year'
+    )
+    await storeData(agregatedYearData, 'com.grandlyon.enedis.year', ['year'])
+  }
+}

src/request.js

+// @ts-check
+const { log } = require('cozy-konnector-libs')
+
+/**
+ * Query SGE in order to get info
+ * @param {number} pointId
+ * @param {string} userLogin
+ * @param {string} startDt
+ * @param {string} endDt
+ * @returns {string}
+ */
+function userMesureDetailles(
+  pointId,
+  userLogin,
+  startDt,
+  endDt,
+  mesureType = 'ENERGIE',
+  unit = 'EA'
+) {
+  log(
+    'info',
+    `Query data ${mesureType}/${unit} between ${startDt} and ${endDt}`
+  )
+  return `<?xml version='1.0' encoding='utf-8'?>
+  <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
+     xmlns:v2="http://www.enedis.fr/sge/b2b/services/consultationmesuresdetaillees/v2.0"
+     xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0">
+     <soapenv:Header/>
+     <soapenv:Body>
+        <v2:consulterMesuresDetaillees>
+           <demande>
+              <initiateurLogin>${userLogin}</initiateurLogin>
+              <pointId>${pointId}</pointId>
+              <mesuresTypeCode>${mesureType}</mesuresTypeCode>
+              <grandeurPhysique>${unit}</grandeurPhysique>
+              <soutirage>true</soutirage>
+              <injection>false</injection>
+              <dateDebut>${startDt}</dateDebut>
+              <dateFin>${endDt}</dateFin>
+              <mesuresCorrigees>false</mesuresCorrigees>
+              <accordClient>true</accordClient>
+           </demande>
+        </v2:consulterMesuresDetaillees>
+     </soapenv:Body>
+  </soapenv:Envelope>
+  `
+}
+
+function userMesureDetaillesHalfHour(pointId, userLogin, startDt, endDt) {
+  log('info', `Query data between ${startDt} and ${endDt}`)
+  return `<?xml version='1.0' encoding='utf-8'?>
+  <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
+     xmlns:v2="http://www.enedis.fr/sge/b2b/services/consultationmesuresdetaillees/v2.0"
+     xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0">
+     <soapenv:Header/>
+     <soapenv:Body>
+        <v2:consulterMesuresDetaillees>
+           <demande>
+              <initiateurLogin>${userLogin}</initiateurLogin>
+              <pointId>${pointId}</pointId>
+              <mesuresTypeCode>COURBE</mesuresTypeCode>
+              <grandeurPhysique>PA</grandeurPhysique>
+              <soutirage>true</soutirage>
+              <injection>false</injection>
+              <dateDebut>${startDt}</dateDebut>
+              <dateFin>${endDt}</dateFin>
+              <mesuresCorrigees>false</mesuresCorrigees>
+              <accordClient>true</accordClient>
+           </demande>
+        </v2:consulterMesuresDetaillees>
+     </soapenv:Body>
+  </soapenv:Envelope>
+  `
+}
+
+/**
+ * Get user technical data
+ * @param {number} pointId
+ * @param {string} userLogin
+ * @returns {string}
+ */
+function userTechnicalData(pointId, userLogin) {
+  log('info', `Query userMesureDetailles`)
+  return `<?xml version='1.0' encoding='utf-8'?>
+  <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
+     xmlns:v2="http://www.enedis.fr/sge/b2b/services/consulterdonneestechniquescontractuelles/v1.0"
+     xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0">
+     <soapenv:Header/>
+     <soapenv:Body>
+        <v2:consulterDonneesTechniquesContractuelles>
+           <pointId>${pointId}</pointId>
+           <loginUtilisateur>${userLogin}</loginUtilisateur>
+           <autorisationClient>true</autorisationClient>
+        </v2:consulterDonneesTechniquesContractuelles>
+     </soapenv:Body>
+  </soapenv:Envelope>
+  `
+}
+
+module.exports = {
+  userTechnicalData,
+  userMesureDetaillesHalfHour,
+  userMesureDetailles,
+}