From ae9d1bd9efb20c61dfdb4fa73ecc6605d642e530 Mon Sep 17 00:00:00 2001
From: Hugo Subtil <ext.sopra.husubtil@grandlyon.com>
Date: Tue, 2 Nov 2021 11:29:07 +0100
Subject: [PATCH] feat: add protections against EGL bad sorted data + clean
code
---
manifest.konnector | 2 +-
src/index.js | 150 +++++++++++++--------------------------------
2 files changed, 43 insertions(+), 109 deletions(-)
diff --git a/manifest.konnector b/manifest.konnector
index 2aea462..57c9815 100644
--- a/manifest.konnector
+++ b/manifest.konnector
@@ -1,5 +1,5 @@
{
- "version": "1.0.2",
+ "version": "1.0.3",
"name": "EGL",
"type": "konnector",
"language": "node",
diff --git a/src/index.js b/src/index.js
index ad07a7d..903cc75 100644
--- a/src/index.js
+++ b/src/index.js
@@ -18,8 +18,12 @@ const manualExecution =
process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false
const startDate = manualExecution
- ? moment().subtract(1, 'year').format('MM/DD/YYYY')
- : moment().subtract(3, 'year').format('MM/DD/YYYY')
+ ? moment()
+ .subtract(1, 'year')
+ .format('MM/DD/YYYY')
+ : moment()
+ .subtract(3, 'year')
+ .format('MM/DD/YYYY')
const endDate = moment().format('MM/DD/YYYY')
// const timeRange = ['day', 'month', 'year']
@@ -45,8 +49,7 @@ module.exports = new BaseKonnector(start)
// the account information come from ./konnector-dev-config.json file
async function start(fields, cozyParameters) {
try {
- // resetting data for demo only
- // await resetData()
+ // Local debug data
// const baseUrl = fields.eglBaseURL
// const apiAuthKey = fields.eglAPIAuthKey
const baseUrl = cozyParameters.secret.eglBaseURL
@@ -149,30 +152,6 @@ async function buildAgregatedData(data, doctype) {
return agregatedData
}
-// async function processData(timeStep, response, baseUrl, apiAuthKey) {
-// const doctype = rangeDate[timeStep].doctype;
-// const loadProfile = await getData(response, baseUrl, apiAuthKey);
-// log("info", "Saving data to Cozy");
-// if (doctype === rangeDate.day.doctype) {
-// await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys);
-// } else if (doctype === rangeDate.month.doctype) {
-// await resetInProgressAggregatedData(rangeDate.month.doctype);
-// const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month);
-// log("info", "Saving monthly data");
-// await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys);
-// } else if (doctype === rangeDate.year.doctype) {
-// await resetInProgressAggregatedData(rangeDate.year.doctype);
-// const yearlyData = processYearAggregation(
-// loadProfile,
-// rangeDate.year.doctype
-// );
-// log("info", "Saving yearly data");
-// await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys);
-// } else {
-// throw new Error("Unkonw range type: " + doctype);
-// }
-// }
-
async function authenticate(login, password, baseUrl, apiAuthKey) {
const authRequest = {
method: 'POST',
@@ -214,16 +193,22 @@ async function getData(response, baseUrl, apiAuthKey) {
json: true
}
try {
- const responseEgl = await rp(dataRequest)
+ // Sort data by date
+ const responseEgl = await rp(dataRequest).then(eglRawData => {
+ eglRawData.resultatRetour.sort(function(a, b) {
+ return new Date(a.DateReleve) - new Date(b.DateReleve)
+ })
+ return eglRawData
+ })
switch (responseEgl.codeRetour) {
case 100:
return format(responseEgl)
case -2:
- throw new Error(errors.LOGIN_FAILED)
+ throw errors.LOGIN_FAILED
case -1:
- throw new Error(errors.VENDOR_DOWN)
+ throw errors.VENDOR_DOWN
default:
- throw new Error(errors.UNKNOWN_ERROR)
+ throw errors.UNKNOWN_ERROR
}
} catch (error) {
throw new Error(errors.VENDOR_DOWN)
@@ -232,90 +217,39 @@ async function getData(response, baseUrl, apiAuthKey) {
function format(response) {
log('info', 'origin response size is : ' + response.resultatRetour.length)
+ // Store first value as reference for index processing
+ let refValue = response.resultatRetour[0]
+ // Create copy of data without first value
const data = response.resultatRetour
.slice(1)
.filter(value => value.ValeurIndex)
- const dataLen = data.length
- log('info', 'filtered size is : ' + dataLen)
- const mapData = data.map((value, index) => {
+ log('info', 'filtered size is : ' + data.length)
+ return data.map(value => {
const time = moment(value.DateReleve, moment.ISO_8601)
- if (index !== 0 && index < dataLen) {
- return {
- load: value.ValeurIndex - data[index - 1].ValeurIndex,
- year: parseInt(time.format('YYYY')),
- month: parseInt(time.format('M')),
- day: parseInt(time.format('D')),
- hour: 0,
- minute: 0,
- type: value.TypeAgregat
- }
- } else {
- return {
- load: null,
- year: parseInt(time.format('YYYY')),
- month: parseInt(time.format('M')),
- day: parseInt(time.format('D')),
- hour: 0,
- minute: 0,
- type: value.TypeAgregat
- }
+ const procesedLoad = value.ValeurIndex - refValue.ValeurIndex
+ if (procesedLoad < 0) {
+ log(
+ 'error',
+ `processing load for day ${parseInt(time.format('D'))}/${parseInt(
+ time.format('M')
+ )}/${parseInt(time.format('YYYY'))}, value is : ${procesedLoad}`
+ )
+ throw errors.VENDOR_DOWN
+ }
+ // Change index ref value
+ refValue = value
+ return {
+ load: procesedLoad,
+ year: parseInt(time.format('YYYY')),
+ month: parseInt(time.format('M')),
+ day: parseInt(time.format('D')),
+ hour: 0,
+ minute: 0,
+ type: value.TypeAgregat
}
})
- const res = [...mapData].filter(v => v.load !== null)
- return res
}
-// function processYearAggregation(data, doctype) {
-// log("info", "Start aggregation for : " + doctype);
-// const grouped = data.reduce(reduceYearFunction, {});
-// return Object.values(grouped);
-// }
-
-// function processMonthlyAggregation(data, range) {
-// log("info", "Start aggregation for : " + range.doctype);
-// // Filter by year
-// const tmpData = groupBy(data, "year");
-// const keys = Object.keys(tmpData);
-// var dataToStore = [];
-// // Monthly aggregation
-// for (const index in keys) {
-// // Get daily data of a year
-// var monthlyData = tmpData[keys[index]];
-// // Monthly aggregation
-// var aggregatedData = monthlyData.reduce(reduceMonthFunction, {});
-// // Store it
-// dataToStore = dataToStore.concat(Object.values(aggregatedData));
-// }
-// return dataToStore;
-// }
-
-// function groupBy(xs, key) {
-// return xs.reduce(function(rv, x) {
-// (rv[x[key]] = rv[x[key]] || []).push(x);
-// return rv;
-// }, {});
-// }
-
-// function reduceYearFunction(acc, x) {
-// var id = acc[x.year];
-// if (id) {
-// id.load += x.load;
-// } else {
-// acc[x.year] = x;
-// }
-// return acc;
-// }
-
-// function reduceMonthFunction(acc, x) {
-// var id = acc[x.month];
-// if (id) {
-// id.load += x.load;
-// } else {
-// acc[x.month] = x;
-// }
-// return acc;
-// }
-
/**
* Save data in the right doctype db and prevent duplicated keys
*/
--
GitLab