From 380c7792f73c9258b7dd501591d52136c3a39721 Mon Sep 17 00:00:00 2001
From: Yoan VALLET <ext.sopra.yvallet@grandlyon.com>
Date: Mon, 25 May 2020 18:02:41 +0200
Subject: [PATCH] change time format and add data aggregation
---
manifest.konnector | 10 ++-
src/index.js | 168 ++++++++++++++++++++++++++++++++++++++++-----
2 files changed, 158 insertions(+), 20 deletions(-)
diff --git a/manifest.konnector b/manifest.konnector
index ef0316e..4c16279 100644
--- a/manifest.konnector
+++ b/manifest.konnector
@@ -27,8 +27,14 @@
"data_types": [],
"screenshots": [],
"permissions": {
- "load profile": {
- "type": "egl.loadprofile"
+ "daily eau du grand lyon data": {
+ "type": "io.egl.day"
+ },
+ "monthly eau du grand lyon data": {
+ "type": "io.egl.month"
+ },
+ "yearly eau du grand lyon data": {
+ "type": "io.egl.year"
},
"files": {
"type": "io.cozy.files"
diff --git a/src/index.js b/src/index.js
index 3a4cfff..1bad305 100644
--- a/src/index.js
+++ b/src/index.js
@@ -14,9 +14,26 @@ moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone
const startDate = moment()
- .subtract(10, 'day')
+ .startOf('year')
+ .subtract(3, 'year')
+ .subtract(1, 'day')
.format('MM/DD/YYYY')
const endDate = moment().format('MM/DD/YYYY')
+const timeRange = ['day', 'month', 'year']
+const rangeDate = {
+ day: {
+ doctype: 'io.egl.day',
+ keys: ['year', 'month', 'day']
+ },
+ month: {
+ doctype: 'io.egl.month',
+ keys: ['year', 'month']
+ },
+ year: {
+ doctype: 'io.egl.year',
+ keys: ['year']
+ }
+}
module.exports = new BaseKonnector(start)
@@ -29,6 +46,7 @@ async function start(fields, cozyParameters) {
// await resetData()
const baseUrl = cozyParameters.secret.eglBaseURL
const apiAuthKey = cozyParameters.secret.eglAPIAuthKey
+ log('debug', fields, 'Fields')
log('info', 'Authenticating ...')
const response = await authenticate(
@@ -38,14 +56,40 @@ async function start(fields, cozyParameters) {
apiAuthKey
)
log('info', 'Successfully logged in')
- log('info', 'Getting data')
- const loadProfile = await getData(response, baseUrl, apiAuthKey)
- log('info', 'Saving data to Cozy')
- storeLoadProfile(loadProfile)
+ Promise.all(
+ timeRange.map(timeStep =>
+ processData(timeStep, response, baseUrl, apiAuthKey)
+ )
+ )
} catch (error) {
throw new Error(error.message)
}
}
+async function processData(timeStep, response, baseUrl, apiAuthKey) {
+ const doctype = rangeDate[timeStep].doctype
+ log('info', 'Getting data')
+ const loadProfile = await getData(response, baseUrl, apiAuthKey)
+ log('info', 'Saving data to Cozy')
+ if (doctype === rangeDate.day.doctype) {
+ log('info', 'Saving daily data')
+ await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys)
+ } else if (doctype === rangeDate.month.doctype) {
+ log('info', 'Saving monthly data')
+ await resetInProgressAggregatedData(rangeDate.month.doctype)
+ const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month)
+ await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys)
+ } else if (doctype === rangeDate.year.doctype) {
+ log('info', 'Saving yearly data')
+ await resetInProgressAggregatedData(rangeDate.year.doctype)
+ const yearlyData = processYearAggregation(
+ loadProfile,
+ rangeDate.year.doctype
+ )
+ await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys)
+ } else {
+ throw new Error('Unkonw range type: ' + doctype)
+ }
+}
async function authenticate(login, password, baseUrl, apiAuthKey) {
const authRequest = {
@@ -74,6 +118,8 @@ async function authenticate(login, password, baseUrl, apiAuthKey) {
}
async function getData(response, baseUrl, apiAuthKey) {
+ log('debug', startDate, 'Start date')
+ log('debug', endDate, 'End date')
const dataRequest = {
method: 'POST',
uri: baseUrl + '/getAllAgregatsByAbonnement.aspx',
@@ -108,31 +154,117 @@ async function getData(response, baseUrl, apiAuthKey) {
function format(response) {
const data = response.resultatRetour.slice(1).map((value, index) => {
+ const time = moment(value.DateReleve, moment.ISO_8601)
return {
- time: moment(value.DateReleve, moment.ISO_8601).format('YYYY-MM-DD'),
load: value.ValeurIndex - response.resultatRetour[index].ValeurIndex,
+ year: parseInt(time.format('YYYY')),
+ month: parseInt(time.format('M')),
+ day: parseInt(time.format('D')),
+ hour: 0,
+ minute: 0,
type: value.TypeAgregat
}
})
return data
}
-function storeLoadProfile(loadProfile) {
- return hydrateAndFilter(loadProfile, 'egl.loadprofile', {
- keys: ['time']
+function processYearAggregation(data, doctype) {
+ log('info', 'Start aggregation for : ' + doctype)
+ const grouped = data.reduce(reduceYearFunction, {})
+ return Object.values(grouped)
+}
+
+function processMonthlyAggregation(data, range) {
+ log('info', 'Start aggregation for : ' + range.doctype)
+ // Filter by year
+ const tmpData = groupBy(data, 'year')
+ const keys = Object.keys(tmpData)
+ var dataToStore = []
+ // Monthly aggregation
+ for (const index in keys) {
+ // Get daily data of a year
+ var monthlyData = tmpData[keys[index]]
+ // Monthly aggregation
+ var aggregatedData = monthlyData.reduce(reduceMonthFunction, {})
+ // Store it
+ dataToStore = dataToStore.concat(Object.values(aggregatedData))
+ }
+ return dataToStore
+}
+
+function groupBy(xs, key) {
+ return xs.reduce(function(rv, x) {
+ ;(rv[x[key]] = rv[x[key]] || []).push(x)
+ return rv
+ }, {})
+}
+
+function reduceYearFunction(acc, x) {
+ var id = acc[x.year]
+ if (id) {
+ id.load += x.load
+ } else {
+ acc[x.year] = x
+ }
+ return acc
+}
+
+function reduceMonthFunction(acc, x) {
+ var id = acc[x.month]
+ if (id) {
+ id.load += x.load
+ } else {
+ acc[x.month] = x
+ }
+ return acc
+}
+
+async function storeData(data, doctype, keys) {
+ log('debug', 'Store into ' + doctype)
+ log('debug', keys, 'Store into keys ')
+ return hydrateAndFilter(data, doctype, {
+ keys: keys
}).then(filteredDocuments => {
- addData(filteredDocuments, 'egl.loadprofile')
+ addData(filteredDocuments, doctype)
})
}
-// eslint-disable-next-line no-unused-vars
-async function resetData() {
- const result = await cozyClient.data.findAll('egl.loadprofile')
- if (result.error) {
+/**
+ * Function handling special case.
+ * The temporary aggregated data need to be remove in order for the most recent one te be saved.
+ * ex for io.egl.month :
+ * { load: 76.712, month: 2020, ... } need to be replace by
+ * { load: 82.212, month: 2020, ... } after grdf data reprocess
+ */
+async function resetInProgressAggregatedData(doctype) {
+ // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
+ log('debug', doctype, 'Remove aggregated data for')
+ const result = await cozyClient.data.findAll(doctype)
+ if (result.error || result.length <= 0) {
// eslint-disable-next-line no-console
- console.error('Error while fetching loads')
- }
- for (const load of result) {
- await cozyClient.data.delete('egl.loadprofile', load)
+ console.warn('Error while fetching loads, doctype not found ')
+ } else {
+ const currentDate = moment()
+ // Filter data to remove
+ var filtered = []
+ if (doctype === rangeDate.year.doctype) {
+ // Yearly case
+ filtered = result.filter(function(el) {
+ return el.year == currentDate.year()
+ })
+ } else {
+ // Monthly case
+ filtered = result.filter(function(el) {
+ return (
+ el.year == currentDate.year() &&
+ el.month == parseInt(moment().format('M'))
+ )
+ })
+ }
+ // Remove data
+ for (const doc of filtered) {
+ log('debug', doc, 'Removing this entry for ' + doctype)
+ await cozyClient.data.delete(doctype, doc)
+ }
}
}
--
GitLab