From cba9031db83d3b9f778e41c4b0981b886cba3d00 Mon Sep 17 00:00:00 2001
From: git-directory-deploy <>
Date: Thu, 25 Feb 2021 16:40:21 +0100
Subject: [PATCH] publish: update egl
generated from commit 55779530e839cf138a02ceb037380da37dc2d9a1
---
index.js | 453 +++++++++++++++++++++++++++++++++++--------------------
1 file changed, 291 insertions(+), 162 deletions(-)
diff --git a/index.js b/index.js
index 506592c..aead142 100644
--- a/index.js
+++ b/index.js
@@ -95,39 +95,46 @@ const {
addData,
hydrateAndFilter,
cozyClient
-} = __webpack_require__(1);
+} = __webpack_require__(1)
-const rp = __webpack_require__(24);
-const moment = __webpack_require__(1492);
-__webpack_require__(1630);
+const rp = __webpack_require__(24)
+const moment = __webpack_require__(1492)
+__webpack_require__(1630)
-moment.locale("fr"); // set the language
-moment.tz.setDefault("Europe/Paris"); // set the timezone
+moment.locale('fr') // set the language
+moment.tz.setDefault('Europe/Paris') // set the timezone
-const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === "true" ? true : false
+const manualExecution =
+ process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false
const startDate = manualExecution
- ? moment().startOf("year").subtract(1, "year").format("MM/DD/YYYY")
- : moment().startOf("year").subtract(3, "year").format("MM/DD/YYYY")
-
-const endDate = moment().format("MM/DD/YYYY");
-const timeRange = ["day", "month", "year"];
+ ? moment()
+ .startOf('year')
+ .subtract(1, 'year')
+ .format('MM/DD/YYYY')
+ : moment()
+ .startOf('year')
+ .subtract(3, 'year')
+ .format('MM/DD/YYYY')
+
+const endDate = moment().format('MM/DD/YYYY')
+// const timeRange = ['day', 'month', 'year']
const rangeDate = {
day: {
- doctype: "com.grandlyon.egl.day",
- keys: ["year", "month", "day"]
+ doctype: 'com.grandlyon.egl.day',
+ keys: ['year', 'month', 'day']
},
month: {
- doctype: "com.grandlyon.egl.month",
- keys: ["year", "month"]
+ doctype: 'com.grandlyon.egl.month',
+ keys: ['year', 'month']
},
year: {
- doctype: "com.grandlyon.egl.year",
- keys: ["year"]
+ doctype: 'com.grandlyon.egl.year',
+ keys: ['year']
}
-};
+}
-module.exports = new BaseKonnector(start);
+module.exports = new BaseKonnector(start)
// The start function is run by the BaseKonnector instance only when it got all the account
// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
@@ -136,80 +143,159 @@ async function start(fields, cozyParameters) {
try {
// resetting data for demo only
// await resetData()
- const baseUrl = cozyParameters.secret.eglBaseURL;
- const apiAuthKey = cozyParameters.secret.eglAPIAuthKey;
- log("info", "Authenticating ...");
+ const baseUrl = cozyParameters.secret.eglBaseURL
+ const apiAuthKey = cozyParameters.secret.eglAPIAuthKey
+ log('info', 'Authenticating ...')
const response = await authenticate(
fields.login,
fields.password,
baseUrl,
apiAuthKey
- );
- log("info", "Successfully logged in");
- Promise.all(
- timeRange.map(timeStep =>
- processData(timeStep, response, baseUrl, apiAuthKey)
+ )
+ log('info', 'Successfully logged in')
+
+ const eglData = await getData(response, baseUrl, apiAuthKey)
+ if (eglData) {
+ log('debug', 'Process egl daily data')
+ const processedLoadData = await processData(
+ eglData,
+ rangeDate.day.doctype,
+ rangeDate.day.keys
)
- );
+ log('debug', 'Agregate egl load data for month and year')
+ await agregateMonthAndYearData(processedLoadData)
+ } else {
+ log('debug', 'No data found')
+ }
} catch (error) {
- throw new Error(error.message);
- }
-}
-async function processData(timeStep, response, baseUrl, apiAuthKey) {
- const doctype = rangeDate[timeStep].doctype;
- const loadProfile = await getData(response, baseUrl, apiAuthKey);
- log("info", "Saving data to Cozy");
- if (doctype === rangeDate.day.doctype) {
- await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys);
- } else if (doctype === rangeDate.month.doctype) {
- await resetInProgressAggregatedData(rangeDate.month.doctype);
- const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month);
- log("info", "Saving monthly data");
- await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys);
- } else if (doctype === rangeDate.year.doctype) {
- await resetInProgressAggregatedData(rangeDate.year.doctype);
- const yearlyData = processYearAggregation(
- loadProfile,
- rangeDate.year.doctype
- );
- log("info", "Saving yearly data");
- await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys);
- } else {
- throw new Error("Unkonw range type: " + doctype);
+ throw new Error(error.message)
}
}
+/**
+ * Parse data
+ * Remove existing data from DB using hydrateAndFilter
+ * Store filtered data
+ * Return the list of filtered data
+ */
+async function processData(data, doctype, filterKeys) {
+ // const formatedData = await formateData(data)
+ log('debug', 'processData - data formated')
+ // Remove data for existing days into the DB
+ const filteredData = await hydrateAndFilter(data, doctype, {
+ keys: filterKeys
+ })
+ log('debug', 'processData - data filtered')
+ // Store new day data
+ await storeData(filteredData, doctype, filterKeys)
+ return filteredData
+}
+
+/**
+ * Agregate data from daily data to monthly and yearly data
+ */
+async function agregateMonthAndYearData(data) {
+ // Sum year and month values into object with year or year-month as keys
+ if (data && data.length !== 0) {
+ let monthData = {}
+ let yearData = {}
+
+ data.forEach(element => {
+ element.year + '-' + element.month in monthData
+ ? (monthData[element.year + '-' + element.month] += element.load)
+ : (monthData[element.year + '-' + element.month] = element.load)
+ element.year in yearData
+ ? (yearData[element.year] += element.load)
+ : (yearData[element.year] = element.load)
+ })
+ // Agregation for Month data
+ const agregatedMonthData = await buildAgregatedData(
+ monthData,
+ 'com.grandlyon.egl.month'
+ )
+ await storeData(agregatedMonthData, 'com.grandlyon.egl.month', [
+ 'year',
+ 'month'
+ ])
+ // Agregation for Year data
+ const agregatedYearData = await buildAgregatedData(
+ yearData,
+ 'com.grandlyon.egl.year'
+ )
+ await storeData(agregatedYearData, 'com.grandlyon.egl.year', ['year'])
+ }
+}
+
+/**
+ * Retrieve and remove old data for a specific doctype
+ * Return an Array of agregated data
+ */
+async function buildAgregatedData(data, doctype) {
+ let agregatedData = []
+ for (let [key, value] of Object.entries(data)) {
+ const data = await buildDataFromKey(doctype, key, value)
+ const oldValue = await resetInProgressAggregatedData(data, doctype)
+ data.load += oldValue
+ agregatedData.push(data)
+ }
+ return agregatedData
+}
+
+// async function processData(timeStep, response, baseUrl, apiAuthKey) {
+// const doctype = rangeDate[timeStep].doctype;
+// const loadProfile = await getData(response, baseUrl, apiAuthKey);
+// log("info", "Saving data to Cozy");
+// if (doctype === rangeDate.day.doctype) {
+// await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys);
+// } else if (doctype === rangeDate.month.doctype) {
+// await resetInProgressAggregatedData(rangeDate.month.doctype);
+// const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month);
+// log("info", "Saving monthly data");
+// await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys);
+// } else if (doctype === rangeDate.year.doctype) {
+// await resetInProgressAggregatedData(rangeDate.year.doctype);
+// const yearlyData = processYearAggregation(
+// loadProfile,
+// rangeDate.year.doctype
+// );
+// log("info", "Saving yearly data");
+// await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys);
+// } else {
+// throw new Error("Unkonw range type: " + doctype);
+// }
+// }
+
async function authenticate(login, password, baseUrl, apiAuthKey) {
const authRequest = {
- method: "POST",
- uri: baseUrl + "/connect.aspx",
+ method: 'POST',
+ uri: baseUrl + '/connect.aspx',
headers: {
AuthKey: apiAuthKey,
- "Content-Type": "application/x-www-form-urlencoded"
+ 'Content-Type': 'application/x-www-form-urlencoded'
},
form: {
login: login,
pass: password
},
json: true
- };
- const response = await rp(authRequest);
+ }
+ const response = await rp(authRequest)
if (response.codeRetour === 100) {
- return response;
+ return response
} else {
- throw new Error(errors.LOGIN_FAILED);
+ throw new Error(errors.LOGIN_FAILED)
}
}
async function getData(response, baseUrl, apiAuthKey) {
- log("debug", "Start date : " + startDate);
- log("debug", "End date : " + endDate);
+ log('debug', 'Start date : ' + startDate)
+ log('debug', 'End date : ' + endDate)
const dataRequest = {
- method: "POST",
- uri: baseUrl + "/getAllAgregatsByAbonnement.aspx",
+ method: 'POST',
+ uri: baseUrl + '/getAllAgregatsByAbonnement.aspx',
headers: {
AuthKey: apiAuthKey,
- "Content-Type": "application/x-www-form-urlencoded"
+ 'Content-Type': 'application/x-www-form-urlencoded'
},
form: {
token: response.resultatRetour.token,
@@ -218,116 +304,116 @@ async function getData(response, baseUrl, apiAuthKey) {
date_fin: endDate
},
json: true
- };
+ }
try {
- const responseEgl = await rp(dataRequest);
+ const responseEgl = await rp(dataRequest)
switch (responseEgl.codeRetour) {
case 100:
- return format(responseEgl);
+ return format(responseEgl)
case -2:
- throw new Error(errors.LOGIN_FAILED);
+ throw new Error(errors.LOGIN_FAILED)
case -1:
- throw new Error(errors.VENDOR_DOWN);
+ throw new Error(errors.VENDOR_DOWN)
default:
- throw new Error(errors.UNKNOWN_ERROR);
+ throw new Error(errors.UNKNOWN_ERROR)
}
} catch (error) {
- throw new Error(errors.VENDOR_DOWN);
+ throw new Error(errors.VENDOR_DOWN)
}
}
function format(response) {
- log("info", "origin response size is : " + response.resultatRetour.length)
+ log('info', 'origin response size is : ' + response.resultatRetour.length)
const data = response.resultatRetour
.slice(1)
- .filter(value => value.ValeurIndex);
- const dataLen = data.length;
- log("info", "filtered size is : " + dataLen);
+ .filter(value => value.ValeurIndex)
+ const dataLen = data.length
+ log('info', 'filtered size is : ' + dataLen)
const mapData = data.map((value, index) => {
- const time = moment(value.DateReleve, moment.ISO_8601);
+ const time = moment(value.DateReleve, moment.ISO_8601)
if (index + 1 < dataLen) {
return {
load: data[index + 1].ValeurIndex - value.ValeurIndex,
- year: parseInt(time.format("YYYY")),
- month: parseInt(time.format("M")),
- day: parseInt(time.format("D")),
+ year: parseInt(time.format('YYYY')),
+ month: parseInt(time.format('M')),
+ day: parseInt(time.format('D')),
hour: 0,
minute: 0,
type: value.TypeAgregat
- };
+ }
} else {
- log("info", "end of data - date is : " + value.DateReleve);
+ log('info', 'end of data - date is : ' + value.DateReleve)
return {
load: null,
- year: parseInt(time.format("YYYY")),
- month: parseInt(time.format("M")),
- day: parseInt(time.format("D")),
+ year: parseInt(time.format('YYYY')),
+ month: parseInt(time.format('M')),
+ day: parseInt(time.format('D')),
hour: 0,
minute: 0,
type: value.TypeAgregat
- };
+ }
}
- });
- return mapData;
-}
-
-function processYearAggregation(data, doctype) {
- log("info", "Start aggregation for : " + doctype);
- const grouped = data.reduce(reduceYearFunction, {});
- return Object.values(grouped);
-}
-
-function processMonthlyAggregation(data, range) {
- log("info", "Start aggregation for : " + range.doctype);
- // Filter by year
- const tmpData = groupBy(data, "year");
- const keys = Object.keys(tmpData);
- var dataToStore = [];
- // Monthly aggregation
- for (const index in keys) {
- // Get daily data of a year
- var monthlyData = tmpData[keys[index]];
- // Monthly aggregation
- var aggregatedData = monthlyData.reduce(reduceMonthFunction, {});
- // Store it
- dataToStore = dataToStore.concat(Object.values(aggregatedData));
- }
- return dataToStore;
-}
-
-function groupBy(xs, key) {
- return xs.reduce(function(rv, x) {
- (rv[x[key]] = rv[x[key]] || []).push(x);
- return rv;
- }, {});
-}
-
-function reduceYearFunction(acc, x) {
- var id = acc[x.year];
- if (id) {
- id.load += x.load;
- } else {
- acc[x.year] = x;
- }
- return acc;
-}
-
-function reduceMonthFunction(acc, x) {
- var id = acc[x.month];
- if (id) {
- id.load += x.load;
- } else {
- acc[x.month] = x;
- }
- return acc;
-}
+ })
+ return mapData
+}
+
+// function processYearAggregation(data, doctype) {
+// log("info", "Start aggregation for : " + doctype);
+// const grouped = data.reduce(reduceYearFunction, {});
+// return Object.values(grouped);
+// }
+
+// function processMonthlyAggregation(data, range) {
+// log("info", "Start aggregation for : " + range.doctype);
+// // Filter by year
+// const tmpData = groupBy(data, "year");
+// const keys = Object.keys(tmpData);
+// var dataToStore = [];
+// // Monthly aggregation
+// for (const index in keys) {
+// // Get daily data of a year
+// var monthlyData = tmpData[keys[index]];
+// // Monthly aggregation
+// var aggregatedData = monthlyData.reduce(reduceMonthFunction, {});
+// // Store it
+// dataToStore = dataToStore.concat(Object.values(aggregatedData));
+// }
+// return dataToStore;
+// }
+
+// function groupBy(xs, key) {
+// return xs.reduce(function(rv, x) {
+// (rv[x[key]] = rv[x[key]] || []).push(x);
+// return rv;
+// }, {});
+// }
+
+// function reduceYearFunction(acc, x) {
+// var id = acc[x.year];
+// if (id) {
+// id.load += x.load;
+// } else {
+// acc[x.year] = x;
+// }
+// return acc;
+// }
+
+// function reduceMonthFunction(acc, x) {
+// var id = acc[x.month];
+// if (id) {
+// id.load += x.load;
+// } else {
+// acc[x.month] = x;
+// }
+// return acc;
+// }
/**
* Save data in the right doctype db and prevent duplicated keys
*/
async function storeData(data, doctype, filterKeys) {
- log("debug", "Store into " + doctype);
- log("debug", "Store into keys : " + filterKeys);
+ log('debug', 'Store into ' + doctype)
+ log('debug', 'Store into keys : ' + filterKeys)
// data.map(v => {
// log("info", "Saving data " + v.load + " for " + v.day + "/" + v.month + "/" + v.year);
// });
@@ -337,44 +423,87 @@ async function storeData(data, doctype, filterKeys) {
return await addData(filteredDocuments, doctype)
}
+/**
+ * Format an entry for DB storage
+ * using key and value
+ * For year doctype: key = "YYYY"
+ * For month doctype: key = "YYYY-MM"
+ */
+async function buildDataFromKey(doctype, key, value) {
+ let year, month, day, hour
+ if (doctype === 'com.grandlyon.egl.year') {
+ year = key
+ month = 1
+ day = 0
+ hour = 0
+ } else if (doctype === 'com.grandlyon.egl.month') {
+ const split = key.split('-')
+ year = split[0]
+ month = split[1]
+ day = 0
+ hour = 0
+ } else {
+ const split = key.split('-')
+ year = split[0]
+ month = split[1]
+ day = split[2]
+ hour = split[3]
+ }
+ return {
+ load: Math.round(value * 10000) / 10000,
+ year: parseInt(year),
+ month: parseInt(month),
+ day: parseInt(day),
+ hour: parseInt(hour),
+ minute: 0
+ }
+}
+
/**
* Function handling special case.
* The temporary aggregated data need to be remove in order for the most recent one te be saved.
* ex for com.grandlyon.egl.month :
* { load: 76.712, month: 2020, ... } need to be replace by
- * { load: 82.212, month: 2020, ... } after grdf data reprocess
+ * { load: 82.212, month: 2020, ... } after egl data reprocess
*/
-async function resetInProgressAggregatedData(doctype) {
+async function resetInProgressAggregatedData(data, doctype) {
// /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
- log("debug", "Remove aggregated data for " + doctype);
- const result = await cozyClient.data.findAll(doctype);
- if (result.error || result.length <= 0) {
- // eslint-disable-next-line no-console
- console.warn("Error while fetching loads, doctype not found ");
- } else {
- const currentDate = moment();
+ log('debug', 'Remove aggregated data for ' + doctype)
+ const result = await cozyClient.data.findAll(doctype)
+ if (result && result.length > 0) {
// Filter data to remove
- var filtered = [];
- if (doctype === rangeDate.year.doctype) {
+ var filtered = []
+ if (doctype === 'com.grandlyon.egl.year') {
// Yearly case
filtered = result.filter(function(el) {
- return el.year == currentDate.year();
- });
- } else {
+ return el.year == data.year
+ })
+ } else if (doctype === 'com.grandlyon.egl.month') {
// Monthly case
+ filtered = result.filter(function(el) {
+ return el.year == data.year && el.month == data.month
+ })
+ } else {
+ // Hourly case
filtered = result.filter(function(el) {
return (
- el.year == currentDate.year() &&
- el.month == parseInt(moment().format("M"))
- );
- });
+ el.year == data.year &&
+ el.month == data.month &&
+ el.day == data.day &&
+ el.hour == data.hour
+ )
+ })
}
// Remove data
+ let sum = 0.0
for (const doc of filtered) {
- log("debug", "Removing this entry for " + doc);
- await cozyClient.data.delete(doctype, doc);
+ sum += doc.load
+ log('debug', 'Removing this entry for ' + doc.load)
+ await cozyClient.data.delete(doctype, doc)
}
+ return sum
}
+ return 0.0
}
--
GitLab