diff --git a/index.js b/index.js index e357be37ea15c3951d71e5c628e63f46bd283547..6ec7db53e238e647aa6154eb0c040730f44c9a32 100644 --- a/index.js +++ b/index.js @@ -137,9 +137,26 @@ moment.locale('fr') // set the language moment.tz.setDefault('Europe/Paris') // set the timezone const startDate = moment() - .subtract(10, 'day') + .startOf('year') + .subtract(3, 'year') + .subtract(1, 'day') .format('MM/DD/YYYY') const endDate = moment().format('MM/DD/YYYY') +const timeRange = ['day', 'month', 'year'] +const rangeDate = { + day: { + doctype: 'io.egl.day', + keys: ['year', 'month', 'day'] + }, + month: { + doctype: 'io.egl.month', + keys: ['year', 'month'] + }, + year: { + doctype: 'io.egl.year', + keys: ['year'] + } +} module.exports = new BaseKonnector(start) @@ -152,6 +169,7 @@ async function start(fields, cozyParameters) { // await resetData() const baseUrl = cozyParameters.secret.eglBaseURL const apiAuthKey = cozyParameters.secret.eglAPIAuthKey + log('debug', fields, 'Fields') log('info', 'Authenticating ...') const response = await authenticate( @@ -161,14 +179,40 @@ async function start(fields, cozyParameters) { apiAuthKey ) log('info', 'Successfully logged in') - log('info', 'Getting data') - const loadProfile = await getData(response, baseUrl, apiAuthKey) - log('info', 'Saving data to Cozy') - storeLoadProfile(loadProfile) + Promise.all( + timeRange.map(timeStep => + processData(timeStep, response, baseUrl, apiAuthKey) + ) + ) } catch (error) { throw new Error(error.message) } } +async function processData(timeStep, response, baseUrl, apiAuthKey) { + const doctype = rangeDate[timeStep].doctype + log('info', 'Getting data') + const loadProfile = await getData(response, baseUrl, apiAuthKey) + log('info', 'Saving data to Cozy') + if (doctype === rangeDate.day.doctype) { + log('info', 'Saving daily data') + await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys) + } else if (doctype === rangeDate.month.doctype) { + log('info', 'Saving monthly data') + await resetInProgressAggregatedData(rangeDate.month.doctype) + const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month) + await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys) + } else if (doctype === rangeDate.year.doctype) { + log('info', 'Saving yearly data') + await resetInProgressAggregatedData(rangeDate.year.doctype) + const yearlyData = processYearAggregation( + loadProfile, + rangeDate.year.doctype + ) + await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys) + } else { + throw new Error('Unkonw range type: ' + doctype) + } +} async function authenticate(login, password, baseUrl, apiAuthKey) { const authRequest = { @@ -197,6 +241,8 @@ async function authenticate(login, password, baseUrl, apiAuthKey) { } async function getData(response, baseUrl, apiAuthKey) { + log('debug', startDate, 'Start date') + log('debug', endDate, 'End date') const dataRequest = { method: 'POST', uri: baseUrl + '/getAllAgregatsByAbonnement.aspx', @@ -231,32 +277,118 @@ async function getData(response, baseUrl, apiAuthKey) { function format(response) { const data = response.resultatRetour.slice(1).map((value, index) => { + const time = moment(value.DateReleve, moment.ISO_8601) return { - time: moment(value.DateReleve, moment.ISO_8601).format('YYYY-MM-DD'), load: value.ValeurIndex - response.resultatRetour[index].ValeurIndex, + year: parseInt(time.format('YYYY')), + month: parseInt(time.format('M')), + day: parseInt(time.format('D')), + hour: 0, + minute: 0, type: value.TypeAgregat } }) return data } -function storeLoadProfile(loadProfile) { - return hydrateAndFilter(loadProfile, 'egl.loadprofile', { - keys: ['time'] +function processYearAggregation(data, doctype) { + log('info', 'Start aggregation for : ' + doctype) + const grouped = data.reduce(reduceYearFunction, {}) + return Object.values(grouped) +} + +function processMonthlyAggregation(data, range) { + log('info', 'Start aggregation for : ' + range.doctype) + // Filter by year + const tmpData = groupBy(data, 'year') + const keys = Object.keys(tmpData) + var dataToStore = [] + // Monthly aggregation + for (const index in keys) { + // Get daily data of a year + var monthlyData = tmpData[keys[index]] + // Monthly aggregation + var aggregatedData = monthlyData.reduce(reduceMonthFunction, {}) + // Store it + dataToStore = dataToStore.concat(Object.values(aggregatedData)) + } + return dataToStore +} + +function groupBy(xs, key) { + return xs.reduce(function(rv, x) { + ;(rv[x[key]] = rv[x[key]] || []).push(x) + return rv + }, {}) +} + +function reduceYearFunction(acc, x) { + var id = acc[x.year] + if (id) { + id.load += x.load + } else { + acc[x.year] = x + } + return acc +} + +function reduceMonthFunction(acc, x) { + var id = acc[x.month] + if (id) { + id.load += x.load + } else { + acc[x.month] = x + } + return acc +} + +async function storeData(data, doctype, keys) { + log('debug', 'Store into ' + doctype) + log('debug', keys, 'Store into keys ') + return hydrateAndFilter(data, doctype, { + keys: keys }).then(filteredDocuments => { - addData(filteredDocuments, 'egl.loadprofile') + addData(filteredDocuments, doctype) }) } -// eslint-disable-next-line no-unused-vars -async function resetData() { - const result = await cozyClient.data.findAll('egl.loadprofile') - if (result.error) { +/** + * Function handling special case. + * The temporary aggregated data need to be remove in order for the most recent one te be saved. + * ex for io.egl.month : + * { load: 76.712, month: 2020, ... } need to be replace by + * { load: 82.212, month: 2020, ... } after grdf data reprocess + */ +async function resetInProgressAggregatedData(doctype) { + // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs + log('debug', doctype, 'Remove aggregated data for') + const result = await cozyClient.data.findAll(doctype) + if (result.error || result.length <= 0) { // eslint-disable-next-line no-console - console.error('Error while fetching loads') - } - for (const load of result) { - await cozyClient.data.delete('egl.loadprofile', load) + console.warn('Error while fetching loads, doctype not found ') + } else { + const currentDate = moment() + // Filter data to remove + var filtered = [] + if (doctype === rangeDate.year.doctype) { + // Yearly case + filtered = result.filter(function(el) { + return el.year == currentDate.year() + }) + } else { + // Monthly case + filtered = result.filter(function(el) { + return ( + el.year == currentDate.year() && + el.month == parseInt(moment().format('M')) + ) + }) + } + // Remove data + for (const doc of filtered) { + log('debug', doc, 'Removing this entry for ' + doctype) + await cozyClient.data.delete(doctype, doc) + } } } @@ -160046,6 +160178,7 @@ const log = __webpack_require__(2).namespace('addData'); module.exports = (entries, doctype) => { const cozy = __webpack_require__(628); return bluebird.mapSeries(entries, async entry => { + log('debug', doctype, 'Adding entry for doctype'); log('debug', entry, 'Adding this entry'); const dbEntry = await (entry._id ? cozy.data.update(doctype, entry, omit(entry, '_rev')) : cozy.data.create(doctype, entry)); // Also update the original entry _id to allow saveBills' @@ -160055,6 +160188,7 @@ module.exports = (entries, doctype) => { }); }; + /***/ }), /* 903 */ /***/ (function(module, exports, __webpack_require__) { @@ -188164,7 +188298,7 @@ async function solveWithAntiCaptcha(taskParams, timeout = DEFAULT_TIMEOUT, secre const fs = __webpack_require__(165); const path = __webpack_require__(157); -let manifest = typeof {"version":"0.1.0","name":"EGL","type":"konnector","language":"node","icon":"icon.png","slug":"egl","source":"https://forge.grandlyon.com/web-et-numerique/llle_project/egl-konnector.git","editor":"Grand Lyon","vendor_link":"www.grandlyon.com","frequency":"daily","categories":["other"],"fields":{"login":{"type":"text"},"password":{"type":"password"},"advancedFields":{"folderPath":{"advanced":true,"isRequired":false}}},"data_types":[],"screenshots":[],"permissions":{"load profile":{"type":"egl.loadprofile"},"files":{"type":"io.cozy.files"},"accounts":{"type":"io.cozy.accounts","verbs":["GET"]}},"developer":{"name":"Métropole de Lyon","url":"https://grandlyon.com"},"langs":["fr","en"],"locales":{"fr":{"short_description":"Courbe de charge depuis l'API eau du Grand Lyon","long_description":"ce connecteur récupère la courbe de charge enregistrée par le compteur Téléo","permissions":{"load profile":{"description":"La courbe de charge enregistrée par le compteur téléo"},"files":{"description":"Utilisé pour ?"},"accounts":{"description":"Utilisé pour obtenir les données du compte"}}},"en":{"short_description":"Load profile from Eau du Grand Lyon API","long_description":"This connector fetches the load profile recorded by the Teleo water meter","permissions":{"load profile":{"description":"The load profile recorded by Téléo"},"files":{"description":"Required to ?"},"accounts":{"description":"Required to get the account's data"}}}},"manifest_version":"2"} !== 'undefined' ? {"version":"0.1.0","name":"EGL","type":"konnector","language":"node","icon":"icon.png","slug":"egl","source":"https://forge.grandlyon.com/web-et-numerique/llle_project/egl-konnector.git","editor":"Grand Lyon","vendor_link":"www.grandlyon.com","frequency":"daily","categories":["other"],"fields":{"login":{"type":"text"},"password":{"type":"password"},"advancedFields":{"folderPath":{"advanced":true,"isRequired":false}}},"data_types":[],"screenshots":[],"permissions":{"load profile":{"type":"egl.loadprofile"},"files":{"type":"io.cozy.files"},"accounts":{"type":"io.cozy.accounts","verbs":["GET"]}},"developer":{"name":"Métropole de Lyon","url":"https://grandlyon.com"},"langs":["fr","en"],"locales":{"fr":{"short_description":"Courbe de charge depuis l'API eau du Grand Lyon","long_description":"ce connecteur récupère la courbe de charge enregistrée par le compteur Téléo","permissions":{"load profile":{"description":"La courbe de charge enregistrée par le compteur téléo"},"files":{"description":"Utilisé pour ?"},"accounts":{"description":"Utilisé pour obtenir les données du compte"}}},"en":{"short_description":"Load profile from Eau du Grand Lyon API","long_description":"This connector fetches the load profile recorded by the Teleo water meter","permissions":{"load profile":{"description":"The load profile recorded by Téléo"},"files":{"description":"Required to ?"},"accounts":{"description":"Required to get the account's data"}}}},"manifest_version":"2"} : {}; +let manifest = typeof {"version":"1.0.0","name":"EGL API Connector","type":"konnector","language":"node","icon":"icon.png","slug":"egl-api-connector","source":"git://gitlab.alpha.grandlyon.com/cozy/egl-api-connector.git","editor":"Grand Lyon","vendor_link":"www.grandlyon.com","frequency":"daily","categories":["other"],"fields":{"login":{"type":"text"},"password":{"type":"password"},"advancedFields":{"folderPath":{"advanced":true,"isRequired":false}}},"data_types":[],"screenshots":[],"permissions":{"daily eau du grand lyon data":{"type":"io.egl.day"},"monthly eau du grand lyon data":{"type":"io.egl.month"},"yearly eau du grand lyon data":{"type":"io.egl.year"},"files":{"type":"io.cozy.files"},"accounts":{"type":"io.cozy.accounts","verbs":["GET"]}},"developer":{"name":"Métropole de Lyon","url":"https://grandlyon.com"},"langs":["fr","en"],"locales":{"fr":{"short_description":"Courbe de charge depuis l'API eau du Grand Lyon","long_description":"ce connecteur récupère la courbe de charge enregistrée par le compteur Téléo","permissions":{"load profile":{"description":"La courbe de charge enregistrée par le compteur téléo"},"files":{"description":"Utilisé pour ?"},"accounts":{"description":"Utilisé pour obtenir les données du compte"}}},"en":{"short_description":"Load profile from Eau du Grand Lyon API","long_description":"This connector fetches the load profile recorded by the Teleo water meter","permissions":{"load profile":{"description":"The load profile recorded by Téléo"},"files":{"description":"Required to ?"},"accounts":{"description":"Required to get the account's data"}}}},"manifest_version":"2"} !== 'undefined' ? {"version":"1.0.0","name":"EGL API Connector","type":"konnector","language":"node","icon":"icon.png","slug":"egl-api-connector","source":"git://gitlab.alpha.grandlyon.com/cozy/egl-api-connector.git","editor":"Grand Lyon","vendor_link":"www.grandlyon.com","frequency":"daily","categories":["other"],"fields":{"login":{"type":"text"},"password":{"type":"password"},"advancedFields":{"folderPath":{"advanced":true,"isRequired":false}}},"data_types":[],"screenshots":[],"permissions":{"daily eau du grand lyon data":{"type":"io.egl.day"},"monthly eau du grand lyon data":{"type":"io.egl.month"},"yearly eau du grand lyon data":{"type":"io.egl.year"},"files":{"type":"io.cozy.files"},"accounts":{"type":"io.cozy.accounts","verbs":["GET"]}},"developer":{"name":"Métropole de Lyon","url":"https://grandlyon.com"},"langs":["fr","en"],"locales":{"fr":{"short_description":"Courbe de charge depuis l'API eau du Grand Lyon","long_description":"ce connecteur récupère la courbe de charge enregistrée par le compteur Téléo","permissions":{"load profile":{"description":"La courbe de charge enregistrée par le compteur téléo"},"files":{"description":"Utilisé pour ?"},"accounts":{"description":"Utilisé pour obtenir les données du compte"}}},"en":{"short_description":"Load profile from Eau du Grand Lyon API","long_description":"This connector fetches the load profile recorded by the Teleo water meter","permissions":{"load profile":{"description":"The load profile recorded by Téléo"},"files":{"description":"Required to ?"},"accounts":{"description":"Required to get the account's data"}}}},"manifest_version":"2"} : {}; if (false) {}