Merge branch 'feat/update-prices' into 'dev'

feat: update prices

See merge request !62

__tests__/helpers/aggregate.spec.js

-const { buildAggregatedData } = require('../../src/helpers/aggregate')
-const { cozyClient } = require('cozy-konnector-libs')
+const {
+  aggregateMonthlyLoad,
+  aggregateYearlyLoad,
+} = require('../../src/helpers/aggregate')
 
-describe('buildAggregatedData', () => {
-  it('should return empty', async () => {
-    const reply = await buildAggregatedData([], 'com.enedis.day')
-    expect(reply).toEqual([])
-  })
-  it('should return year value', async () => {
-    const reply = await buildAggregatedData(
-      {
-        2022: 36,
-      },
-      'com.grandlyon.enedis.year'
-    )
-    expect(reply).toEqual([
-      {
-        day: 0,
-        hour: 0,
-        load: 36,
-        minute: 0,
-        month: 1,
-        year: 2022,
-      },
-    ])
-  })
-  it('should return year value with doc existing', async () => {
-    const spy = jest.spyOn(cozyClient.data, 'findAll')
-    spy.mockResolvedValueOnce([{ year: 2022, month: 8, day: 1, load: 1 }])
-    const reply = await buildAggregatedData(
-      {
-        2022: 36,
-      },
-      'com.grandlyon.enedis.year'
-    )
-    expect(reply).toEqual([
-      {
-        day: 0,
-        hour: 0,
-        load: 37,
-        minute: 0,
-        month: 1,
-        year: 2022,
-      },
-    ])
-  })
-  it('should return month value', async () => {
-    const spy = jest.spyOn(cozyClient.data, 'findAll')
-    spy.mockResolvedValueOnce([{ year: 2022, month: 8, day: 1, load: 1 }])
-    const reply = await buildAggregatedData(
-      {
-        '2022-08': 36,
-      },
-      'com.grandlyon.enedis.month'
-    )
-    expect(reply).toEqual([
-      {
-        day: 0,
-        hour: 0,
-        load: 37,
-        minute: 0,
-        month: 8,
-        year: 2022,
-      },
+describe('aggregateMonthlyLoad', () => {
+  it('should aggregate monthly load', () => {
+    const data = [
+      { year: 2024, month: 1, day: 29, hour: 0, minute: 0, load: 1, price: 2 },
+      { year: 2024, month: 1, day: 30, hour: 0, minute: 0, load: 1, price: 2 },
+      { year: 2024, month: 1, day: 31, hour: 0, minute: 0, load: 1, price: 2 },
+      { year: 2024, month: 2, day: 1, hour: 0, minute: 0, load: 1, price: 2 },
+    ]
+
+    const result = aggregateMonthlyLoad(data)
+
+    expect(result).toStrictEqual([
+      { year: 2024, month: 1, day: 0, hour: 0, minute: 0, load: 3, price: 6 },
+      { year: 2024, month: 2, day: 0, hour: 0, minute: 0, load: 1, price: 2 },
     ])
   })
-  it('should return daily value', async () => {
-    const spy = jest.spyOn(cozyClient.data, 'findAll')
-    spy.mockResolvedValueOnce([
-      { year: 2022, month: 8, day: 1, load: 1, hour: 13 },
-    ])
-    const reply = await buildAggregatedData(
-      {
-        '2022-08-01-13:39:25+00:00': 36,
-      },
-      'com.grandlyon.enedis.minute'
-    )
-    expect(reply).toEqual([
-      {
-        day: 1,
-        hour: 13,
-        load: 37,
-        minute: 0,
-        month: 8,
-        year: 2022,
-      },
+})
+
+describe('aggregateYearlyLoad', () => {
+  it('should aggregate yearly load', () => {
+    const data = [
+      { year: 2023, month: 12, day: 0, hour: 0, minute: 0, load: 1, price: 2 },
+      { year: 2024, month: 1, day: 0, hour: 0, minute: 0, load: 1, price: 2 },
+      { year: 2024, month: 2, day: 0, hour: 0, minute: 0, load: 1, price: 2 },
+    ]
+
+    const result = aggregateYearlyLoad(data)
+
+    expect(result).toStrictEqual([
+      { year: 2023, month: 0, day: 0, hour: 0, minute: 0, load: 1, price: 2 },
+      { year: 2024, month: 0, day: 0, hour: 0, minute: 0, load: 2, price: 4 },
     ])
   })
 })

__tests__/helpers/prices.spec.js

+const axios = require('axios')
+const { getPrices, applyPrices } = require('../../src/helpers/prices')
+
+jest.mock('axios')
+jest.mock('cozy-konnector-libs', () => ({
+  log: jest.fn(),
+}))
+
+describe('getPrices', () => {
+  const boBaseUrl = 'https://example.com'
+
+  it('should return null when axios throws an error', async () => {
+    // Mock axios to throw an error
+    axios.get.mockRejectedValue(new Error('Network Error'))
+
+    const result = await getPrices(boBaseUrl)
+
+    expect(result).toBeNull()
+  })
+})
+
+describe('applyPrices', () => {
+  it('should apply prices to data', async () => {
+    const data = [
+      { year: 2024, month: 1, day: 31, hour: 0, minute: 0, load: 10 },
+      { year: 2024, month: 2, day: 1, hour: 0, minute: 0, load: 10 },
+      { year: 2024, month: 2, day: 2, hour: 0, minute: 0, load: 10 },
+    ]
+    const prices = [
+      { startDate: '2024-01-31T00:00:00Z', price: 1 },
+      { startDate: '2024-02-01T00:00:00Z', price: 2 },
+      { startDate: '2024-02-02T00:00:00Z', price: 3 },
+    ]
+
+    const result = await applyPrices(data, prices)
+
+    expect(result).toStrictEqual([
+      {
+        year: 2024,
+        month: 1,
+        day: 31,
+        hour: 0,
+        minute: 0,
+        load: 10,
+        price: 10,
+      },
+      { year: 2024, month: 2, day: 1, hour: 0, minute: 0, load: 10, price: 20 },
+      { year: 2024, month: 2, day: 2, hour: 0, minute: 0, load: 10, price: 30 },
+    ])
+  })
+
+  it('should not apply prices if data is before prices', async () => {
+    const data = [
+      { year: 2020, month: 1, day: 14, hour: 0, minute: 0, load: 10 },
+    ]
+    const prices = [{ startDate: '2024-01-14T00:00:00Z', price: 1 }]
+
+    const result = await applyPrices(data, prices)
+
+    expect(result).toStrictEqual([
+      { year: 2020, month: 1, day: 14, hour: 0, minute: 0, load: 10 },
+    ])
+  })
+
+  it('should not apply prices if prices are empty', async () => {
+    const data = [
+      { year: 2024, month: 1, day: 14, hour: 0, minute: 0, load: 10 },
+    ]
+    const prices = []
+    const result = await applyPrices(data, prices)
+
+    expect(result).toStrictEqual([
+      { year: 2024, month: 1, day: 14, hour: 0, minute: 0, load: 10 },
+    ])
+  })
+})

src/constants.js

+const rangeDate = {
+  minute: {
+    doctype: 'com.grandlyon.enedis.minute',
+    keys: ['year', 'month', 'day', 'hour', 'minute'],
+  },
+  day: {
+    doctype: 'com.grandlyon.enedis.day',
+    keys: ['year', 'month', 'day'],
+  },
+  month: {
+    doctype: 'com.grandlyon.enedis.month',
+    keys: ['year', 'month'],
+  },
+  year: {
+    doctype: 'com.grandlyon.enedis.year',
+    keys: ['year'],
+  },
+  maxPower: {
+    doctype: 'com.grandlyon.enedis.maxpower',
+    keys: ['year', 'month', 'day', 'hour', 'minute'],
+  },
+}
+
+module.exports = { rangeDate }

src/core/types/types.js

@@ -6,6 +6,8 @@
  * @property {number} day
  * @property {number} hour
  * @property {number} minute
+ * @property {number} load
+ * @property {number} price
  */
 
 /**
@@ -107,3 +109,12 @@
  * @property {{code: string}} $
  * @property {string} libelle
  */
+
+/**
+ * Price definition
+ * @typedef {object} Price
+ * @property {number} fluidtype
+ * @property {number} price
+ * @property {string} startDate
+ * @property {string} endDate
+ */

src/helpers/aggregate.js

-// @ts-check
-const { log, cozyClient } = require('cozy-konnector-libs')
+const { cozyClient } = require('cozy-konnector-libs')
+const { rangeDate } = require('../constants')
 
 /**
- * Retrieve and remove old data for a specific doctype
- * Return an Array of aggregated data
+ * Aggregates the load data by month, summing the load of each day for each month.
+ *
+ * @param {EnedisKonnectorData[]} data - The data to aggregate.
+ *
+ * @returns {EnedisKonnectorData[]} - An array of aggregated data by month.
  */
-async function buildAggregatedData(data, doctype) {
-  let aggregatedData = []
-  for (let [key, value] of Object.entries(data)) {
-    const data = buildDataFromKey(doctype, key, value)
-    const oldValue = await resetInProgressAggregatedData(data, doctype)
-    data.load += oldValue
-    aggregatedData.push(data)
+function aggregateMonthlyLoad(data) {
+  const monthlyLoad = {}
+
+  for (const entry of data) {
+    const { year, month, load, price } = entry
+    const monthKey = `${year}-${month}`
+
+    if (!monthlyLoad[monthKey]) {
+      monthlyLoad[monthKey] = {
+        year: year,
+        month: month,
+        day: 0,
+        hour: 0,
+        minute: 0,
+        load: load,
+        price: price,
+      }
+    } else {
+      monthlyLoad[monthKey].load += load
+      monthlyLoad[monthKey].price += price
+    }
   }
-  return aggregatedData
+
+  return Object.values(monthlyLoad)
 }
 
 /**
- * Format an entry for DB storage
- * using key and value
- * For year doctype: key = "YYYY"
- * For month doctype: key = "YYYY-MM"
+ * Aggregates the load data by year, summing the load of each month for each year.
+ *
+ * @param {EnedisKonnectorData[]} data - The data to aggregate.
+ *
+ * @returns {EnedisKonnectorData[]} - An array of aggregated data by year.
  */
-function buildDataFromKey(doctype, key, value) {
-  let year, month, day, hour
-  if (doctype === 'com.grandlyon.enedis.year') {
-    year = key
-    month = 1
-    day = 0
-    hour = 0
-  } else if (doctype === 'com.grandlyon.enedis.month') {
-    const split = key.split('-')
-    year = split[0]
-    month = split[1]
-    day = 0
-    hour = 0
-  } else {
-    const split = key.split('-')
-    year = split[0]
-    month = split[1]
-    day = split[2]
-    hour = split[3]
+function aggregateYearlyLoad(data) {
+  const yearlyLoad = {}
+
+  for (const entry of data) {
+    const { year, load, price } = entry
+
+    if (!yearlyLoad[year]) {
+      yearlyLoad[year] = {
+        year: year,
+        month: 0,
+        day: 0,
+        hour: 0,
+        minute: 0,
+        load: load,
+        price: price,
+      }
+    } else {
+      yearlyLoad[year].load += load
+      yearlyLoad[year].price += price
+    }
   }
-  return {
-    load: Math.round(value * 10000) / 10000,
-    year: parseInt(year),
-    month: parseInt(month),
-    day: parseInt(day),
-    hour: parseInt(hour),
-    minute: 0,
+
+  return Object.values(yearlyLoad)
+}
+
+/**
+ * Removes aggregates matching the same year and month as the first data retrieved from enedis if it is already in database.
+ *
+ * This step is needed to avoid updating the 3 years old month aggregate with incomplete data
+ *
+ * @param {number} firstMonth
+ * @param {number} firstYear
+ * @param {EnedisKonnectorData[]} yearlyLoads
+ * @returns {Promise<EnedisKonnectorData[]>}
+ */
+async function filterFirstMonthlyLoad(firstMonth, firstYear, monthlyLoads) {
+  const monthlyLoadRef = await cozyClient.data.defineIndex(
+    rangeDate.month.doctype,
+    rangeDate.month.keys
+  )
+  const startDateMonthlyLoad = await cozyClient.data.query(monthlyLoadRef, {
+    selector: { year: firstYear, month: firstMonth },
+    limit: 1,
+  })
+
+  if (!startDateMonthlyLoad.length) {
+    return monthlyLoads
   }
+  return monthlyLoads.filter(
+    monthlyLoad =>
+      monthlyLoad.year !== startDateMonthlyLoad[0].year ||
+      monthlyLoad.month !== startDateMonthlyLoad[0].month
+  )
 }
 
 /**
- * Function handling special case.
- * The temporary aggregated data need to be remove in order for the most recent one te be saved.
- * ex for com.grandlyon.enedis.year :
- * { load: 76.712, year: 2020, ... } need to be replace by
- * { load: 82.212, year: 2020, ... } after enedis data reprocess
+ * Removes aggregates matching the same year as the first data retrieved from enedis if it is already in database.
+ *
+ * This step is needed to avoid updating the 3 years old year aggregate with incomplete data
+ *
+ * @param {number} firstYear
+ * @param {EnedisKonnectorData[]} yearlyLoads
+ * @returns {Promise<EnedisKonnectorData[]>}
  */
-async function resetInProgressAggregatedData(data, doctype) {
-  // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
-  log('debug', doctype, 'Remove aggregated data for')
-  const result = await cozyClient.data.findAll(doctype)
-  if (result && result.length > 0) {
-    // Filter data to remove
-    var filtered = []
-    if (doctype === 'com.grandlyon.enedis.year') {
-      // Yearly case
-      filtered = result.filter(function (el) {
-        return el.year == data.year
-      })
-    } else if (doctype === 'com.grandlyon.enedis.month') {
-      // Monthly case
-      filtered = result.filter(function (el) {
-        return el.year == data.year && el.month == data.month
-      })
-    } else {
-      // Hourly case
-      filtered = result.filter(function (el) {
-        return (
-          el.year == data.year &&
-          el.month == data.month &&
-          el.day == data.day &&
-          el.hour == data.hour
-        )
-      })
-    }
-    // Remove data
-    let sum = 0.0
-    for (const doc of filtered) {
-      sum += doc.load
-      log('debug', doc, 'Removing this entry for ' + doctype)
-      await cozyClient.data.delete(doctype, doc)
-    }
-    return sum
+async function filterFirstYearlyLoad(firstYear, yearlyLoads) {
+  const yearlyLoadRef = await cozyClient.data.defineIndex(
+    rangeDate.year.doctype,
+    rangeDate.year.keys
+  )
+  const startDateYearlyLoad = await cozyClient.data.query(yearlyLoadRef, {
+    selector: { year: firstYear },
+    limit: 1,
+  })
+  if (!startDateYearlyLoad.length) {
+    return yearlyLoads
   }
-  return 0.0
+  return yearlyLoads.filter(
+    yearlyLoad => yearlyLoad.year !== startDateYearlyLoad[0].year
+  )
 }
 
 module.exports = {
-  buildAggregatedData,
+  aggregateMonthlyLoad,
+  aggregateYearlyLoad,
+  filterFirstMonthlyLoad,
+  filterFirstYearlyLoad,
 }

src/helpers/prices.js

+const { log } = require('cozy-konnector-libs')
+const axios = require('axios').default
+const Sentry = require('@sentry/node')
+const moment = require('moment-timezone')
+
+/**
+ * @param {string} boBaseUrl
+ */
+async function getPrices(boBaseUrl) {
+  const boElecPricesUrl = new URL('/api/common/prices/0', boBaseUrl).href
+
+  try {
+    /** @type {Price[]} */
+    const prices = (await axios.get(boElecPricesUrl)).data
+    return prices
+  } catch (error) {
+    log('error', 'Could not fetch BO prices')
+    Sentry.captureException(error, {
+      tags: {
+        section: 'getPrices',
+      },
+    })
+    return null
+  }
+}
+
+/**
+ * Apply the given prices to the given data array and return the result.
+ * @param {EnedisKonnectorData[]} data
+ * @param {Price[]} fluidPrices
+ */
+async function applyPrices(data, fluidPrices) {
+  // Sort prices by descending start date
+  fluidPrices.sort((a, b) =>
+    moment(b.startDate).tz('UTC').diff(moment(a.startDate).tz('UTC'))
+  )
+
+  return data.map(load => {
+    // Select the first price that is before the load date
+    const loadDate = moment.tz(
+      { year: load.year, month: load.month - 1, day: load.day },
+      'UTC'
+    )
+    const fluidPrice = fluidPrices.find(p => {
+      const startDate = moment.tz(p.startDate, 'UTC')
+      return loadDate.isSameOrAfter(startDate)
+    })
+    if (!fluidPrice) return load
+    return { ...load, price: fluidPrice.price * load.load }
+  })
+}
+module.exports = { getPrices, applyPrices }

src/index.js

@@ -4,14 +4,13 @@ const {
   BaseKonnector,
   log,
   hydrateAndFilter,
-  addData,
   errors,
+  updateOrCreate,
 } = require('cozy-konnector-libs')
 const soapRequest = require('easy-soap-request')
 const moment = require('moment')
 require('moment-timezone')
 const xml2js = require('xml2js')
-const { buildAggregatedData } = require('./helpers/aggregate')
 const {
   parseSgeXmlData,
   formateDataForDoctype,
@@ -42,6 +41,14 @@ const { getAccount, saveAccountData } = require('./requests/cozy')
 const { isLocal } = require('./helpers/env')
 const Sentry = require('@sentry/node')
 const { catchRequestReject } = require('./helpers/catch')
+const { applyPrices, getPrices } = require('./helpers/prices.js')
+const { rangeDate } = require('./constants.js')
+const {
+  aggregateMonthlyLoad,
+  filterFirstMonthlyLoad,
+  aggregateYearlyLoad,
+  filterFirstYearlyLoad,
+} = require('./helpers/aggregate.js')
 
 moment.locale('fr') // set the language
 moment.tz.setDefault('Europe/Paris') // set the timezone
@@ -242,7 +249,7 @@ async function start(fields, cozyParameters) {
       }
     }
     log('info', 'Successfully logged in')
-    await gatherData(baseUrl, apiAuthKey, sgeLogin, pointId)
+    await gatherData(baseUrl, apiAuthKey, sgeLogin, pointId, boBaseUrl)
 
     log('info', 'Konnector success')
   } catch (error) {
@@ -317,8 +324,9 @@ async function deleteConsent(
  * @param {string} apiAuthKey
  * @param {string} sgeLogin
  * @param {string} pointId
+ * @param {string} boBaseUrl
  */
-async function gatherData(baseUrl, apiAuthKey, sgeLogin, pointId) {
+async function gatherData(baseUrl, apiAuthKey, sgeLogin, pointId, boBaseUrl) {
   log('info', 'Querying data...')
   const measuresUrl = new URL(
     '/enedis_SGE_ConsultationMesuresDetaillees_v3/1.0',
@@ -328,9 +336,13 @@ async function gatherData(baseUrl, apiAuthKey, sgeLogin, pointId) {
     '/enedis_SGE_ConsultationDonneesTechniquesContractuelles/1.0',
     baseUrl
   ).href
-  await getData(measuresUrl, apiAuthKey, sgeLogin, pointId)
+
+  log('info', 'Fetching BO prices')
+  const prices = await getPrices(boBaseUrl)
+
+  await getDailyData(measuresUrl, apiAuthKey, sgeLogin, pointId, prices)
   await getMaxPowerData(measuresUrl, apiAuthKey, sgeLogin, pointId)
-  await getDataHalfHour(measuresUrl, apiAuthKey, sgeLogin, pointId)
+  await getDataHalfHour(measuresUrl, apiAuthKey, sgeLogin, pointId, prices)
   await getOffPeakHours(contractUrl, apiAuthKey, sgeLogin, pointId)
   log('info', 'Querying data: done')
 }
@@ -404,7 +416,7 @@ async function getOffPeakHours(url, apiAuthKey, userLogin, pointId) {
  * @param {string} userLogin
  * @param {string} pointId
  */
-async function getData(url, apiAuthKey, userLogin, pointId) {
+async function getDailyData(url, apiAuthKey, userLogin, pointId, prices) {
   log('info', 'Fetching daily data')
   const sgeHeaders = {
     'Content-Type': 'text/xml;charset=UTF-8',
@@ -440,7 +452,7 @@ async function getData(url, apiAuthKey, userLogin, pointId) {
       valueProcessors: [parseValue],
       explicitArray: false,
     },
-    processData('com.grandlyon.enedis.day')
+    processDailyData(prices)
   )
 }
 
@@ -485,7 +497,7 @@ async function getMaxPowerData(url, apiAuthKey, userLogin, pointId) {
       valueProcessors: [parseValue],
       explicitArray: false,
     },
-    processData('com.grandlyon.enedis.maxpower')
+    processMaxPowerData()
   )
 }
 
@@ -496,7 +508,7 @@ async function getMaxPowerData(url, apiAuthKey, userLogin, pointId) {
  * @param {string} userLogin
  * @param {string} pointId
  */
-async function getDataHalfHour(url, apiAuthKey, userLogin, pointId) {
+async function getDataHalfHour(url, apiAuthKey, userLogin, pointId, prices) {
   log('info', 'Fetching half-hour data')
   const sgeHeaders = {
     'Content-Type': 'text/xml;charset=UTF-8',
@@ -547,17 +559,16 @@ async function getDataHalfHour(url, apiAuthKey, userLogin, pointId) {
         valueProcessors: [parseValueHalfHour],
         explicitArray: false,
       },
-      processData('com.grandlyon.enedis.minute')
+      processHalfHourData(prices)
     )
   }
 }
 
 /**
- * Parse data
- * @param {string} doctype
+ * @param {Price[] | null} prices
  * @returns
  */
-function processData(doctype) {
+function processDailyData(prices) {
   return async (err, result) => {
     if (err) {
       log('error', err)
@@ -565,79 +576,164 @@ function processData(doctype) {
       throw err
     }
     // Return only needed part of info
-    log('info', `Processing ${doctype} data`)
+    log('info', `Processing ${rangeDate.day.doctype} data`)
     try {
       const data = parseSgeXmlData(result)
-      const processedDailyData = await storeData(
-        await formateDataForDoctype(data),
-        doctype,
-        ['year', 'month', 'day', 'hour', 'minute']
-      )
-      if (doctype === 'com.grandlyon.enedis.day') {
-        log('info', 'Aggregating...')
-        await aggregateMonthAndYearData(processedDailyData)
+      let dailyData = await formateDataForDoctype(data)
+
+      if (prices && prices.length > 0) {
+        log('info', 'Found BO prices, applying them to enedis data')
+        dailyData = await applyPrices(dailyData, prices)
       }
+
+      const filterDayKeys = [...rangeDate.day.keys, 'load']
+      if (prices) filterDayKeys.push('price')
+      const daysToUpdate = await hydrateAndFilter(
+        dailyData,
+        rangeDate.day.doctype,
+        { keys: filterDayKeys }
+      )
+      log('debug', 'Store enedis daily load data')
+      await updateOrCreate(
+        daysToUpdate,
+        rangeDate.day.doctype,
+        rangeDate.day.keys
+      )
+
+      const { year: firstYear, month: firstMonth } = dailyData[0]
+
+      log('debug', 'Aggregate enedis monthly load data')
+      const monthlyLoads = aggregateMonthlyLoad(dailyData)
+
+      log('debug', 'Filter first month aggregate if already in database')
+      const filteredMonthlyLoads = await filterFirstMonthlyLoad(
+        firstMonth,
+        firstYear,
+        monthlyLoads
+      )
+
+      const filterMonthKeys = [...rangeDate.month.keys, 'load']
+      if (prices) filterMonthKeys.push('price')
+      const monthsToUpdate = await hydrateAndFilter(
+        filteredMonthlyLoads,
+        rangeDate.month.doctype,
+        { keys: filterMonthKeys }
+      )
+
+      log('debug', 'Store aggregated enedis monthly load data')
+      await updateOrCreate(
+        monthsToUpdate,
+        rangeDate.month.doctype,
+        rangeDate.month.keys
+      )
+
+      log('debug', 'Aggregate enedis yearly load data')
+      const yearlyLoads = aggregateYearlyLoad(monthlyLoads)
+
+      log('debug', 'Filter first year aggregate if already in database')
+      const filteredYearlyLoads = await filterFirstYearlyLoad(
+        firstYear,
+        yearlyLoads
+      )
+
+      const filterYearKeys = [...rangeDate.year.keys, 'load']
+      if (prices) filterYearKeys.push('price')
+      const yearsToUpdate = await hydrateAndFilter(
+        filteredYearlyLoads,
+        rangeDate.year.doctype,
+        { keys: filterYearKeys }
+      )
+
+      log('debug', 'Store aggregated enedis yearly load data')
+      await updateOrCreate(
+        yearsToUpdate,
+        rangeDate.year.doctype,
+        rangeDate.year.keys
+      )
     } catch (e) {
-      if (doctype === 'com.grandlyon.enedis.minute') {
-        const errorMessage = `No half-hour activated. Issue: ${result.Envelope.Body.Fault.faultstring}`
-        Sentry.captureMessage(errorMessage, {
-          tags: { section: 'processData' },
-        })
-        log('warn', errorMessage)
-      } else {
-        log('warn', `Unknown error ${e}`)
-      }
+      log('warn', `Unknown error ${e}`)
     }
   }
 }
 
-/**
- * Save data in the right doctype db and prevent duplicated keys
- * @param {EnedisKonnectorData[]} data
- * @param {string} doctype
- * @param {string[]} filterKeys
- * @returns {Promise<*>}
- */
-async function storeData(data, doctype, filterKeys) {
-  log('debug', doctype, 'Store into')
-  const filteredDocuments = await hydrateAndFilter(data, doctype, {
-    keys: filterKeys,
-  })
-  await addData(filteredDocuments, doctype)
-  return filteredDocuments
+function processMaxPowerData() {
+  return async (err, result) => {
+    if (err) {
+      log('error', err)
+      Sentry.captureException('error while processing daily data')
+      throw err
+    }
+    // Return only needed part of info
+    log('info', `Processing ${rangeDate.maxPower.doctype} data`)
+    try {
+      const data = parseSgeXmlData(result)
+      const maxPowerData = await formateDataForDoctype(data)
+
+      const filterMaxPowerKeys = [...rangeDate.maxPower.keys, 'load']
+      const maxPowerToUpdate = await hydrateAndFilter(
+        maxPowerData,
+        rangeDate.maxPower.doctype,
+        { keys: filterMaxPowerKeys }
+      )
+      log('debug', 'Store Enedis max power load data')
+      await updateOrCreate(
+        maxPowerToUpdate,
+        rangeDate.maxPower.doctype,
+        rangeDate.maxPower.keys
+      )
+    } catch (e) {
+      log('warn', `Unknown error ${e}`)
+    }
+  }
 }
 
 /**
- * Aggregate data from daily data to monthly and yearly data
+ * @param {Price[] | null} prices
+ * @returns
  */
-async function aggregateMonthAndYearData(data) {
-  // Sum year and month values into object with year or year-month as keys
-  if (data && data.length > 0) {
-    let monthData = {}
-    let yearData = {}
-    data.forEach(element => {
-      element.year + '-' + element.month in monthData
-        ? (monthData[element.year + '-' + element.month] += element.load)
-        : (monthData[element.year + '-' + element.month] = element.load)
-      element.year in yearData
-        ? (yearData[element.year] += element.load)
-        : (yearData[element.year] = element.load)
-    })
-    // Aggregation for Month data
-    const aggregatedMonthData = await buildAggregatedData(
-      monthData,
-      'com.grandlyon.enedis.month'
-    )
-    await storeData(aggregatedMonthData, 'com.grandlyon.enedis.month', [
-      'year',
-      'month',
-    ])
-    // Aggregation for Year data
-    const aggregatedYearData = await buildAggregatedData(
-      yearData,
-      'com.grandlyon.enedis.year'
-    )
-    await storeData(aggregatedYearData, 'com.grandlyon.enedis.year', ['year'])
+function processHalfHourData(prices) {
+  return async (err, result) => {
+    if (err) {
+      log('error', err)
+      Sentry.captureException('error while processing half-hour data')
+      throw err
+    }
+    // Return only needed part of info
+    log('info', `Processing ${rangeDate.minute.doctype} data`)
+    try {
+      const data = parseSgeXmlData(result)
+      let minuteData = await formateDataForDoctype(data)
+
+      if (
+        (rangeDate.minute.doctype === 'com.grandlyon.enedis.day' ||
+          rangeDate.minute.doctype === 'com.grandlyon.enedis.minute') &&
+        prices &&
+        prices.length > 0
+      ) {
+        log('info', 'Found BO prices, applying them to enedis data')
+        minuteData = await applyPrices(minuteData, prices)
+      }
+
+      const filterMinuteKeys = [...rangeDate.minute.keys, 'load']
+      if (prices) filterMinuteKeys.push('price')
+      const minutesToUpdate = await hydrateAndFilter(
+        minuteData,
+        rangeDate.minute.doctype,
+        { keys: filterMinuteKeys }
+      )
+      log('debug', 'Store Enedis minute load data')
+      await updateOrCreate(
+        minutesToUpdate,
+        rangeDate.minute.doctype,
+        rangeDate.minute.keys
+      )
+    } catch (e) {
+      const errorMessage = `No half-hour activated. Issue: ${result.Envelope.Body.Fault.faultstring}`
+      Sentry.captureMessage(errorMessage, {
+        tags: { section: 'processData' },
+      })
+      log('warn', errorMessage)
+    }
   }
 }