Skip to content
Snippets Groups Projects
Commit f86e9743 authored by Yoan VALLET's avatar Yoan VALLET
Browse files

add consumption load curve data

parent 2b8cec72
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 170 additions and 39 deletions
src/index.js
+ 170
− 39
View file @ f86e9743
......@@ -13,40 +13,99 @@ moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone
/*** Connector Constants ***/
const startDate = moment()
const startDailyDate = moment()
.subtract(32, 'month')
.format('YYYY-MM-DD')
const endDate = moment().format('YYYY-MM-DD')
const startLoadDate = moment()
.subtract(8, 'day')
.format('YYYY-MM-DD')
const endDate = moment()
.subtract(1, 'day')
.format('YYYY-MM-DD')
const baseUrl = 'https://gw.hml.api.enedis.fr'
// The start function is run by the BaseKonnector instance only when it got all the account
// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
// the account information come from ./konnector-dev-config.json file
// cozyParameters are static parameters, independents from the account. Most often, it can be a
// secret api key.
/**
* The start function is run by the BaseKonnector instance only when it got all the account
* information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
* the account information come from ./konnector-dev-config.json file
* cozyParameters are static parameters, independents from the account. Most often, it can be a
* secret api key.
*/
async function start(fields) {
try {
const { access_token } = fields
const usage_point_id = 22516914714270
log('info', 'Fetching enedis data')
const fetchedData = await getDailyData(access_token, usage_point_id)
log('info', 'Process data')
await processData(fetchedData)
log('info', 'Saving data to Cozy')
const { info } = fields
const usage_point_id = info.usage_point_id
log('info', 'Fetching enedis daily data')
const fetchedDailyData = await getDailyData(access_token, usage_point_id)
log('info', 'Process enedis daily data')
const processedDailyData = await processData(
fetchedDailyData,
'io.enedis.day',
['year', 'month', 'day']
)
log('info', 'Agregate enedis daily data for month and year')
await agregateMonthAndYearData(processedDailyData)
log('info', 'Fetching enedis load data')
const fetchedLoadData = await getLoadData(access_token, usage_point_id)
if (fetchedLoadData && fetchedLoadData.length > 0) {
log('info', 'Process enedis load data')
const processedLoadData = await processData(
fetchedLoadData,
'io.enedis.minute',
['year', 'month', 'day', 'hour', 'minute']
)
log('info', 'Agregate enedis load data for hour')
await agregateHourlyData(processedLoadData)
} else {
log('info', 'No consent or data for load curve')
}
} catch (err) {
log('error', err.message)
}
}
// Retrieve data from the API
// Format: { value: "Wh", "date": "YYYY-MM-DD" }
/**
* Retrieve data from the API
* Format: { value: "Wh", "date": "YYYY-MM-DD" }
*/
async function getDailyData(token, usagePointID) {
const dataRequest = {
method: 'GET',
uri:
baseUrl +
'/v4/metering_data/daily_consumption?start=' +
startDate +
startDailyDate +
'&end=' +
endDate +
'&usage_point_id=' +
usagePointID,
headers: {
Accept: 'application/json',
Authorization: 'Bearer ' + token
}
}
try {
const response = await rp(dataRequest)
return response
} catch (error) {
throw error
}
}
/**
* Retrieve data from the API
* Format: { value: "W", "date": "YYYY-MM-DD hh:mm:ss" }
*/
async function getLoadData(token, usagePointID) {
const dataRequest = {
method: 'GET',
uri:
baseUrl +
'/v4/metering_data/consumption_load_curve?start=' +
startLoadDate +
'&end=' +
endDate +
'&usage_point_id=' +
......@@ -64,34 +123,87 @@ async function getDailyData(token, usagePointID) {
}
}
async function processData(data) {
/**
* Parse data
* Remove existing data from DB using hydrateAndFilter
* Store filtered data
* Return the list of filtered data
*/
async function processData(data, doctype, filterKeys) {
const parsedData = JSON.parse(data)
const intervalData = parsedData.meter_reading.interval_reading
const formatedData = await formateData(intervalData)
// Remove data for existing days into the DB
const filteredData = await hydrateAndFilter(formatedData, 'io.enedis.day', {
keys: ['year', 'month', 'day']
const filteredData = await hydrateAndFilter(formatedData, doctype, {
keys: filterKeys
})
// Store new day data
await storeData(filteredData, 'io.enedis.day', ['year', 'month', 'day'])
await storeData(filteredData, doctype, filterKeys)
return filteredData
}
/**
* Agregate data from daily data to monthly and yearly data
*/
async function agregateMonthAndYearData(data) {
// Sum year and month values into object with year or year-month as keys
if (filteredData && filteredData.length > 0) {
let yearData = {}
if (data && data.length > 0) {
let monthData = {}
filteredData.forEach(element => {
element.year in yearData
? (yearData[element.year] += element.load)
: (yearData[element.year] = element.load)
let yearData = {}
data.forEach(element => {
element.year + '-' + element.month in monthData
? (monthData[element.year + '-' + element.month] += element.load)
: (monthData[element.year + '-' + element.month] = element.load)
element.year in yearData
? (yearData[element.year] += element.load)
: (yearData[element.year] = element.load)
})
const agregatedYearData = await agregateData('io.enedis.year', yearData)
// Agregation for Month data
const agregatedMonthData = await buildAgregatedData(
monthData,
'io.enedis.month'
)
await storeData(agregatedMonthData, 'io.enedis.month', ['year', 'month'])
// Agregation for Year data
const agregatedYearData = await buildAgregatedData(
yearData,
'io.enedis.year'
)
await storeData(agregatedYearData, 'io.enedis.year', ['year'])
}
}
const agregatedMonthData = await agregateData('io.enedis.month', monthData)
await storeData(agregatedMonthData, 'io.enedis.month', ['year', 'month'])
/**
* Agregate data from load data (every 30 min) to Hourly data
*/
async function agregateHourlyData(data) {
// Sum year and month values into object with year or year-month as keys
if (data && data.length > 0) {
let hourData = {}
data.forEach(element => {
let key =
element.year +
'-' +
element.month +
'-' +
element.day +
'-' +
element.hour
key in hourData
? (hourData[key] += element.load)
: (hourData[key] = element.load)
})
// Agregation for Month data
const agregatedMonthData = await buildAgregatedData(
hourData,
'io.enedis.hour'
)
await storeData(agregatedMonthData, 'io.enedis.hour', [
'year',
'month',
'day',
'hour'
])
}
}
......@@ -112,9 +224,8 @@ async function storeData(data, doctype, filterKeys) {
*/
async function formateData(data) {
log('info', 'Formating data')
log('debug', start, 'Start date')
return data.map(record => {
const date = moment(record.date, 'YYYY/MM/DD')
const date = moment(record.date, 'YYYY/MM/DD h:mm:ss')
if (record.value != -2) {
return {
load: parseFloat(record.value / 1000),
......@@ -132,11 +243,11 @@ async function formateData(data) {
* Retrieve and remove old data for a specific doctype
* Return an Array of agregated data
*/
async function agregateData(doctype, data) {
async function buildAgregatedData(data, doctype) {
let agregatedData = []
for (let [key, value] of Object.entries(data)) {
const data = await buildDataFromKey(doctype, key, value)
const oldValue = await resetInProgressAggregatedData(doctype, data)
const oldValue = await resetInProgressAggregatedData(data, doctype)
data.load += oldValue
agregatedData.push(data)
}
......@@ -150,21 +261,31 @@ async function agregateData(doctype, data) {
* For month doctype: key = "YYYY-MM"
*/
async function buildDataFromKey(doctype, key, value) {
let year, month
let year, month, day, hour
if (doctype === 'io.enedis.year') {
year = key
month = 1
day = 0
hour = 0
} else if (doctype === 'io.enedis.month') {
const split = key.split('-')
year = split[0]
month = split[1]
day = 0
hour = 0
} else {
const split = key.split('-')
year = split[0]
month = split[1]
day = split[2]
hour = split[3]
}
return {
load: Math.round(value * 1000) / 1000,
year: parseInt(year),
month: parseInt(month),
day: 0,
hour: 0,
day: parseInt(day),
hour: parseInt(hour),
minute: 0
}
}
......@@ -176,7 +297,7 @@ async function buildDataFromKey(doctype, key, value) {
* { load: 76.712, year: 2020, ... } need to be replace by
* { load: 82.212, year: 2020, ... } after enedis data reprocess
*/
async function resetInProgressAggregatedData(doctype, data) {
async function resetInProgressAggregatedData(data, doctype) {
// /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
log('debug', doctype, 'Remove aggregated data for')
const result = await cozyClient.data.findAll(doctype)
......@@ -188,11 +309,21 @@ async function resetInProgressAggregatedData(doctype, data) {
filtered = result.filter(function(el) {
return el.year == data.year
})
} else {
} else if (doctype === 'io.enedis.month') {
// Monthly case
filtered = result.filter(function(el) {
return el.year == data.year && el.month == data.month
})
} else {
// Hourly case
filtered = result.filter(function(el) {
return (
el.year == data.year &&
el.month == data.month &&
el.day == data.day &&
el.hour == data.hour
)
})
}
// Remove data
let sum = 0.0
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment