add consumption load curve data

src/index.js

@@ -13,40 +13,99 @@ moment.locale('fr') // set the language
 moment.tz.setDefault('Europe/Paris') // set the timezone
 
 /*** Connector Constants ***/
-const startDate = moment()
+const startDailyDate = moment()
   .subtract(32, 'month')
   .format('YYYY-MM-DD')
-const endDate = moment().format('YYYY-MM-DD')
+const startLoadDate = moment()
+  .subtract(8, 'day')
+  .format('YYYY-MM-DD')
+const endDate = moment()
+  .subtract(1, 'day')
+  .format('YYYY-MM-DD')
 const baseUrl = 'https://gw.hml.api.enedis.fr'
 
-// The start function is run by the BaseKonnector instance only when it got all the account
-// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
-// the account information come from ./konnector-dev-config.json file
-// cozyParameters are static parameters, independents from the account. Most often, it can be a
-// secret api key.
+/**
+ * The start function is run by the BaseKonnector instance only when it got all the account
+ * information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
+ * the account information come from ./konnector-dev-config.json file
+ * cozyParameters are static parameters, independents from the account. Most often, it can be a
+ * secret api key.
+ */
 async function start(fields) {
   try {
     const { access_token } = fields
-    const usage_point_id = 22516914714270
-    log('info', 'Fetching enedis data')
-    const fetchedData = await getDailyData(access_token, usage_point_id)
-    log('info', 'Process data')
-    await processData(fetchedData)
-    log('info', 'Saving data to Cozy')
+    const { info } = fields
+    const usage_point_id = info.usage_point_id
+
+    log('info', 'Fetching enedis daily data')
+    const fetchedDailyData = await getDailyData(access_token, usage_point_id)
+    log('info', 'Process enedis daily data')
+    const processedDailyData = await processData(
+      fetchedDailyData,
+      'io.enedis.day',
+      ['year', 'month', 'day']
+    )
+    log('info', 'Agregate enedis daily data for month and year')
+    await agregateMonthAndYearData(processedDailyData)
+
+    log('info', 'Fetching enedis load data')
+    const fetchedLoadData = await getLoadData(access_token, usage_point_id)
+    if (fetchedLoadData && fetchedLoadData.length > 0) {
+      log('info', 'Process enedis load data')
+      const processedLoadData = await processData(
+        fetchedLoadData,
+        'io.enedis.minute',
+        ['year', 'month', 'day', 'hour', 'minute']
+      )
+      log('info', 'Agregate enedis load data for hour')
+      await agregateHourlyData(processedLoadData)
+    } else {
+      log('info', 'No consent or data for load curve')
+    }
   } catch (err) {
     log('error', err.message)
   }
 }
 
-// Retrieve data from the API
-// Format: { value: "Wh", "date": "YYYY-MM-DD" }
+/**
+ * Retrieve data from the API
+ * Format: { value: "Wh", "date": "YYYY-MM-DD" }
+ */
 async function getDailyData(token, usagePointID) {
   const dataRequest = {
     method: 'GET',
     uri:
       baseUrl +
       '/v4/metering_data/daily_consumption?start=' +
-      startDate +
+      startDailyDate +
+      '&end=' +
+      endDate +
+      '&usage_point_id=' +
+      usagePointID,
+    headers: {
+      Accept: 'application/json',
+      Authorization: 'Bearer ' + token
+    }
+  }
+  try {
+    const response = await rp(dataRequest)
+    return response
+  } catch (error) {
+    throw error
+  }
+}
+
+/**
+ * Retrieve data from the API
+ * Format: { value: "W", "date": "YYYY-MM-DD hh:mm:ss" }
+ */
+async function getLoadData(token, usagePointID) {
+  const dataRequest = {
+    method: 'GET',
+    uri:
+      baseUrl +
+      '/v4/metering_data/consumption_load_curve?start=' +
+      startLoadDate +
       '&end=' +
       endDate +
       '&usage_point_id=' +
@@ -64,34 +123,87 @@ async function getDailyData(token, usagePointID) {
   }
 }
 
-async function processData(data) {
+/**
+ * Parse data
+ * Remove existing data from DB using hydrateAndFilter
+ * Store filtered data
+ * Return the list of filtered data
+ */
+async function processData(data, doctype, filterKeys) {
   const parsedData = JSON.parse(data)
   const intervalData = parsedData.meter_reading.interval_reading
   const formatedData = await formateData(intervalData)
   // Remove data for existing days into the DB
-  const filteredData = await hydrateAndFilter(formatedData, 'io.enedis.day', {
-    keys: ['year', 'month', 'day']
+  const filteredData = await hydrateAndFilter(formatedData, doctype, {
+    keys: filterKeys
   })
   // Store new day data
-  await storeData(filteredData, 'io.enedis.day', ['year', 'month', 'day'])
+  await storeData(filteredData, doctype, filterKeys)
+  return filteredData
+}
+
+/**
+ * Agregate data from daily data to monthly and yearly data
+ */
+async function agregateMonthAndYearData(data) {
   // Sum year and month values into object with year or year-month as keys
-  if (filteredData && filteredData.length > 0) {
-    let yearData = {}
+  if (data && data.length > 0) {
     let monthData = {}
-    filteredData.forEach(element => {
-      element.year in yearData
-        ? (yearData[element.year] += element.load)
-        : (yearData[element.year] = element.load)
+    let yearData = {}
+    data.forEach(element => {
       element.year + '-' + element.month in monthData
         ? (monthData[element.year + '-' + element.month] += element.load)
         : (monthData[element.year + '-' + element.month] = element.load)
+      element.year in yearData
+        ? (yearData[element.year] += element.load)
+        : (yearData[element.year] = element.load)
     })
-
-    const agregatedYearData = await agregateData('io.enedis.year', yearData)
+    // Agregation for Month data
+    const agregatedMonthData = await buildAgregatedData(
+      monthData,
+      'io.enedis.month'
+    )
+    await storeData(agregatedMonthData, 'io.enedis.month', ['year', 'month'])
+    // Agregation for Year data
+    const agregatedYearData = await buildAgregatedData(
+      yearData,
+      'io.enedis.year'
+    )
     await storeData(agregatedYearData, 'io.enedis.year', ['year'])
+  }
+}
 
-    const agregatedMonthData = await agregateData('io.enedis.month', monthData)
-    await storeData(agregatedMonthData, 'io.enedis.month', ['year', 'month'])
+/**
+ * Agregate data from load data (every 30 min) to Hourly data
+ */
+async function agregateHourlyData(data) {
+  // Sum year and month values into object with year or year-month as keys
+  if (data && data.length > 0) {
+    let hourData = {}
+    data.forEach(element => {
+      let key =
+        element.year +
+        '-' +
+        element.month +
+        '-' +
+        element.day +
+        '-' +
+        element.hour
+      key in hourData
+        ? (hourData[key] += element.load)
+        : (hourData[key] = element.load)
+    })
+    // Agregation for Month data
+    const agregatedMonthData = await buildAgregatedData(
+      hourData,
+      'io.enedis.hour'
+    )
+    await storeData(agregatedMonthData, 'io.enedis.hour', [
+      'year',
+      'month',
+      'day',
+      'hour'
+    ])
   }
 }
 
@@ -112,9 +224,8 @@ async function storeData(data, doctype, filterKeys) {
  */
 async function formateData(data) {
   log('info', 'Formating data')
-  log('debug', start, 'Start date')
   return data.map(record => {
-    const date = moment(record.date, 'YYYY/MM/DD')
+    const date = moment(record.date, 'YYYY/MM/DD h:mm:ss')
     if (record.value != -2) {
       return {
         load: parseFloat(record.value / 1000),
@@ -132,11 +243,11 @@ async function formateData(data) {
  * Retrieve and remove old data for a specific doctype
  * Return an Array of agregated data
  */
-async function agregateData(doctype, data) {
+async function buildAgregatedData(data, doctype) {
   let agregatedData = []
   for (let [key, value] of Object.entries(data)) {
     const data = await buildDataFromKey(doctype, key, value)
-    const oldValue = await resetInProgressAggregatedData(doctype, data)
+    const oldValue = await resetInProgressAggregatedData(data, doctype)
     data.load += oldValue
     agregatedData.push(data)
   }
@@ -150,21 +261,31 @@ async function agregateData(doctype, data) {
  * For month doctype: key = "YYYY-MM"
  */
 async function buildDataFromKey(doctype, key, value) {
-  let year, month
+  let year, month, day, hour
   if (doctype === 'io.enedis.year') {
     year = key
     month = 1
+    day = 0
+    hour = 0
+  } else if (doctype === 'io.enedis.month') {
+    const split = key.split('-')
+    year = split[0]
+    month = split[1]
+    day = 0
+    hour = 0
   } else {
     const split = key.split('-')
     year = split[0]
     month = split[1]
+    day = split[2]
+    hour = split[3]
   }
   return {
     load: Math.round(value * 1000) / 1000,
     year: parseInt(year),
     month: parseInt(month),
-    day: 0,
-    hour: 0,
+    day: parseInt(day),
+    hour: parseInt(hour),
     minute: 0
   }
 }
@@ -176,7 +297,7 @@ async function buildDataFromKey(doctype, key, value) {
  * { load: 76.712, year: 2020, ... } need to be replace by
  * { load: 82.212, year: 2020, ... } after enedis data reprocess
  */
-async function resetInProgressAggregatedData(doctype, data) {
+async function resetInProgressAggregatedData(data, doctype) {
   // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
   log('debug', doctype, 'Remove aggregated data for')
   const result = await cozyClient.data.findAll(doctype)
@@ -188,11 +309,21 @@ async function resetInProgressAggregatedData(doctype, data) {
       filtered = result.filter(function(el) {
         return el.year == data.year
       })
-    } else {
+    } else if (doctype === 'io.enedis.month') {
       // Monthly case
       filtered = result.filter(function(el) {
         return el.year == data.year && el.month == data.month
       })
+    } else {
+      // Hourly case
+      filtered = result.filter(function(el) {
+        return (
+          el.year == data.year &&
+          el.month == data.month &&
+          el.day == data.day &&
+          el.hour == data.hour
+        )
+      })
     }
     // Remove data
     let sum = 0.0