const { BaseKonnector, log, errors, addData, hydrateAndFilter, cozyClient } = require('cozy-konnector-libs') const fetch = require('node-fetch') const moment = require('moment') require('moment-timezone') moment.locale('fr') // set the language moment.tz.setDefault('Europe/Paris') // set the timezone const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false const startDate = manualExecution ? moment() .subtract(1, 'year') .format('MM/DD/YYYY') : moment() .subtract(3, 'year') .format('MM/DD/YYYY') const endDate = moment().format('MM/DD/YYYY') // const timeRange = ['day', 'month', 'year'] const rangeDate = { day: { doctype: 'com.grandlyon.egl.day', keys: ['year', 'month', 'day'] }, month: { doctype: 'com.grandlyon.egl.month', keys: ['year', 'month'] }, year: { doctype: 'com.grandlyon.egl.year', keys: ['year'] } } module.exports = new BaseKonnector(start) // The start function is run by the BaseKonnector instance only when it got all the account // information (fields). When you run this connector yourself in "standalone" mode or "dev" mode, // the account information come from ./konnector-dev-config.json file async function start(fields, cozyParameters) { try { // Local debug data // const baseUrl = fields.eglBaseURL // const apiAuthKey = fields.eglAPIAuthKey const baseUrl = cozyParameters.secret.eglBaseURL const apiAuthKey = cozyParameters.secret.eglAPIAuthKey log('info', 'Authenticating ...') const response = await authenticate( fields.login, fields.password, baseUrl, apiAuthKey ) log('info', 'Successfully logged in') const eglData = await getData(response, baseUrl, apiAuthKey) if (eglData) { log('debug', 'Process egl daily data') const processedLoadData = await processData( eglData, rangeDate.day.doctype, rangeDate.day.keys ) log('debug', 'Agregate egl load data for month and year') await agregateMonthAndYearData(processedLoadData) } else { log('debug', 'No data found') } } catch (error) { throw new Error(error.message) } } /** * Parse data * Remove existing data from DB using hydrateAndFilter * Store filtered data * Return the list of filtered data */ async function processData(data, doctype, filterKeys) { // const formatedData = await formateData(data) log('debug', 'processData - data formated') // Remove data for existing days into the DB const filteredData = await hydrateAndFilter(data, doctype, { keys: filterKeys }) log('debug', 'processData - data filtered') // Store new day data await storeData(filteredData, doctype, filterKeys) return filteredData } /** * Agregate data from daily data to monthly and yearly data */ async function agregateMonthAndYearData(data) { // Sum year and month values into object with year or year-month as keys if (data && data.length !== 0) { let monthData = {} let yearData = {} data.forEach(element => { element.year + '-' + element.month in monthData ? (monthData[element.year + '-' + element.month] += element.load) : (monthData[element.year + '-' + element.month] = element.load) element.year in yearData ? (yearData[element.year] += element.load) : (yearData[element.year] = element.load) }) // Agregation for Month data const agregatedMonthData = await buildAgregatedData( monthData, 'com.grandlyon.egl.month' ) await storeData(agregatedMonthData, 'com.grandlyon.egl.month', [ 'year', 'month' ]) // Agregation for Year data const agregatedYearData = await buildAgregatedData( yearData, 'com.grandlyon.egl.year' ) await storeData(agregatedYearData, 'com.grandlyon.egl.year', ['year']) } } /** * Retrieve and remove old data for a specific doctype * Return an Array of agregated data */ async function buildAgregatedData(data, doctype) { log('info', 'entering buildAgregatedData') let agregatedData = [] for (let [key, value] of Object.entries(data)) { const data = await buildDataFromKey(doctype, key, value) const oldValue = await resetInProgressAggregatedData(data, doctype) log('info', 'Dataload + oldvalue is ' + data.load + ' + ' + oldValue) data.load += oldValue agregatedData.push(data) } return agregatedData } async function authenticate(login, password, baseUrl, apiAuthKey) { const authRequest = { method: "POST", headers: { AuthKey: apiAuthKey, "Content-Type": "application/x-www-form-urlencoded" }, body: new URLSearchParams({ login: login, pass: password }) } const data = await fetch(baseUrl + "/connect.aspx", authRequest) const response = await data.json() if (response.codeRetour === 100) { return response } else { throw new Error(errors.LOGIN_FAILED) } } async function getData(response, baseUrl, apiAuthKey) { log('debug', 'Start date : ' + startDate) log('debug', 'End date : ' + endDate) const dataRequest = { method: "POST", headers: { AuthKey: apiAuthKey, 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ token: response.resultatRetour.token, num_abt: response.resultatRetour.num_abt, date_debut: startDate, date_fin: endDate }) } try { // Sort data by date const data = await fetch( baseUrl + "/getAllAgregatsByAbonnement.aspx", dataRequest ) const responseEgl = await data.json() responseEgl.resultatRetour.sort(function(a, b) { return new Date(a.DateReleve) - new Date(b.DateReleve); }) switch (responseEgl.codeRetour) { case 100: return format(responseEgl) case -2: throw errors.LOGIN_FAILED case -1: throw errors.VENDOR_DOWN default: throw errors.UNKNOWN_ERROR } } catch (error) { log('debug', 'Error from getAllAgregatsByAbonnement') throw new Error(errors.VENDOR_DOWN) } } function format(response) { log('info', 'origin response size is : ' + response.resultatRetour.length) // Store first value as reference for index processing let refValue = response.resultatRetour[0] // Create copy of data without first value const data = response.resultatRetour .slice(1) .filter(value => value.ValeurIndex) log('info', 'filtered size is : ' + data.length) return data.map(value => { const time = moment(value.DateReleve, moment.ISO_8601) const procesedLoad = value.ValeurIndex - refValue.ValeurIndex if (procesedLoad < 0) { log( 'error', `processing load for day ${parseInt(time.format('D'))}/${parseInt( time.format('M') )}/${parseInt(time.format('YYYY'))}, value is : ${procesedLoad}` ) throw errors.VENDOR_DOWN } // Change index ref value refValue = value return { load: procesedLoad, year: parseInt(time.format('YYYY')), month: parseInt(time.format('M')), day: parseInt(time.format('D')), hour: 0, minute: 0, type: value.TypeAgregat } }) } /** * Save data in the right doctype db and prevent duplicated keys */ async function storeData(data, doctype, filterKeys) { log('debug', 'Store into ' + doctype) log('debug', 'Store into keys : ' + filterKeys) // data.map(v => { // log("info", "Saving data " + v.load + " for " + v.day + "/" + v.month + "/" + v.year); // }); const filteredDocuments = await hydrateAndFilter(data, doctype, { keys: filterKeys }) return await addData(filteredDocuments, doctype) } /** * Format an entry for DB storage * using key and value * For year doctype: key = "YYYY" * For month doctype: key = "YYYY-MM" */ async function buildDataFromKey(doctype, key, value) { let year, month, day, hour if (doctype === 'com.grandlyon.egl.year') { year = key month = 1 day = 0 hour = 0 } else if (doctype === 'com.grandlyon.egl.month') { const split = key.split('-') year = split[0] month = split[1] day = 0 hour = 0 } else { const split = key.split('-') year = split[0] month = split[1] day = split[2] hour = split[3] } return { load: Math.round(value * 10000) / 10000, year: parseInt(year), month: parseInt(month), day: parseInt(day), hour: parseInt(hour), minute: 0 } } /** * Function handling special case. * The temporary aggregated data need to be remove in order for the most recent one te be saved. * ex for com.grandlyon.egl.month : * { load: 76.712, month: 2020, ... } need to be replace by * { load: 82.212, month: 2020, ... } after egl data reprocess */ async function resetInProgressAggregatedData(data, doctype) { // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs log('debug', 'Remove aggregated data for ' + doctype) const result = await cozyClient.data.findAll(doctype) if (result && result.length > 0) { // Filter data to remove var filtered = [] if (doctype === 'com.grandlyon.egl.year') { // Yearly case filtered = result.filter(function(el) { return el.year == data.year }) } else if (doctype === 'com.grandlyon.egl.month') { // Monthly case filtered = result.filter(function(el) { return el.year == data.year && el.month == data.month }) } else { // Hourly case filtered = result.filter(function(el) { return ( el.year == data.year && el.month == data.month && el.day == data.day && el.hour == data.hour ) }) } // Remove data let sum = 0.0 for (const doc of filtered) { sum += doc.load log('debug', 'Removing this entry for ' + doc.load) await cozyClient.data.delete(doctype, doc) } return sum } return 0.0 }