const { BaseKonnector, log, errors, addData, hydrateAndFilter, cozyClient } = require('cozy-konnector-libs') const rp = require('request-promise') const moment = require('moment') require('moment-timezone') moment.locale('fr') // set the language moment.tz.setDefault('Europe/Paris') // set the timezone const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false const startDate = manualExecution ? moment().subtract(1, 'year').format('YYYY-MM-DD') : moment().subtract(3, 'year').format('YYYY-MM-DD') const endDate = moment().format('MM/DD/YYYY') // const timeRange = ['day', 'month', 'year'] const rangeDate = { day: { doctype: 'com.grandlyon.egl.day', keys: ['year', 'month', 'day'] }, month: { doctype: 'com.grandlyon.egl.month', keys: ['year', 'month'] }, year: { doctype: 'com.grandlyon.egl.year', keys: ['year'] } } module.exports = new BaseKonnector(start) // The start function is run by the BaseKonnector instance only when it got all the account // information (fields). When you run this connector yourself in "standalone" mode or "dev" mode, // the account information come from ./konnector-dev-config.json file async function start(fields, cozyParameters) { try { // resetting data for demo only // await resetData() // const baseUrl = fields.eglBaseURL // const apiAuthKey = fields.eglAPIAuthKey const baseUrl = cozyParameters.secret.eglBaseURL const apiAuthKey = cozyParameters.secret.eglAPIAuthKey log('info', 'Authenticating ...') const response = await authenticate( fields.login, fields.password, baseUrl, apiAuthKey ) log('info', 'Successfully logged in') const eglData = await getData(response, baseUrl, apiAuthKey) if (eglData) { log('debug', 'Process egl daily data') const processedLoadData = await processData( eglData, rangeDate.day.doctype, rangeDate.day.keys ) log('debug', 'Agregate egl load data for month and year') await agregateMonthAndYearData(processedLoadData) } else { log('debug', 'No data found') } } catch (error) { throw new Error(error.message) } } /** * Parse data * Remove existing data from DB using hydrateAndFilter * Store filtered data * Return the list of filtered data */ async function processData(data, doctype, filterKeys) { // const formatedData = await formateData(data) log('debug', 'processData - data formated') // Remove data for existing days into the DB const filteredData = await hydrateAndFilter(data, doctype, { keys: filterKeys }) log('debug', 'processData - data filtered') // Store new day data await storeData(filteredData, doctype, filterKeys) return filteredData } /** * Agregate data from daily data to monthly and yearly data */ async function agregateMonthAndYearData(data) { // Sum year and month values into object with year or year-month as keys if (data && data.length !== 0) { let monthData = {} let yearData = {} data.forEach(element => { element.year + '-' + element.month in monthData ? (monthData[element.year + '-' + element.month] += element.load) : (monthData[element.year + '-' + element.month] = element.load) element.year in yearData ? (yearData[element.year] += element.load) : (yearData[element.year] = element.load) }) // Agregation for Month data const agregatedMonthData = await buildAgregatedData( monthData, 'com.grandlyon.egl.month' ) await storeData(agregatedMonthData, 'com.grandlyon.egl.month', [ 'year', 'month' ]) // Agregation for Year data const agregatedYearData = await buildAgregatedData( yearData, 'com.grandlyon.egl.year' ) await storeData(agregatedYearData, 'com.grandlyon.egl.year', ['year']) } } /** * Retrieve and remove old data for a specific doctype * Return an Array of agregated data */ async function buildAgregatedData(data, doctype) { log('info', 'entering buildAgregatedData') let agregatedData = [] for (let [key, value] of Object.entries(data)) { const data = await buildDataFromKey(doctype, key, value) const oldValue = await resetInProgressAggregatedData(data, doctype) log('info', 'Dataload + oldvalue is ' + data.load + ' + ' + oldValue) data.load += oldValue agregatedData.push(data) } return agregatedData } // async function processData(timeStep, response, baseUrl, apiAuthKey) { // const doctype = rangeDate[timeStep].doctype; // const loadProfile = await getData(response, baseUrl, apiAuthKey); // log("info", "Saving data to Cozy"); // if (doctype === rangeDate.day.doctype) { // await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys); // } else if (doctype === rangeDate.month.doctype) { // await resetInProgressAggregatedData(rangeDate.month.doctype); // const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month); // log("info", "Saving monthly data"); // await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys); // } else if (doctype === rangeDate.year.doctype) { // await resetInProgressAggregatedData(rangeDate.year.doctype); // const yearlyData = processYearAggregation( // loadProfile, // rangeDate.year.doctype // ); // log("info", "Saving yearly data"); // await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys); // } else { // throw new Error("Unkonw range type: " + doctype); // } // } async function authenticate(login, password, baseUrl, apiAuthKey) { const authRequest = { method: 'POST', uri: baseUrl + '/connect.aspx', headers: { AuthKey: apiAuthKey, 'Content-Type': 'application/x-www-form-urlencoded' }, form: { login: login, pass: password }, json: true } const response = await rp(authRequest) if (response.codeRetour === 100) { return response } else { throw new Error(errors.LOGIN_FAILED) } } async function getData(response, baseUrl, apiAuthKey) { log('debug', 'Start date : ' + startDate) log('debug', 'End date : ' + endDate) const dataRequest = { method: 'POST', uri: baseUrl + '/getAllAgregatsByAbonnement.aspx', headers: { AuthKey: apiAuthKey, 'Content-Type': 'application/x-www-form-urlencoded' }, form: { token: response.resultatRetour.token, num_abt: response.resultatRetour.num_abt, date_debut: startDate, date_fin: endDate }, json: true } try { const responseEgl = await rp(dataRequest) switch (responseEgl.codeRetour) { case 100: return format(responseEgl) case -2: throw new Error(errors.LOGIN_FAILED) case -1: throw new Error(errors.VENDOR_DOWN) default: throw new Error(errors.UNKNOWN_ERROR) } } catch (error) { throw new Error(errors.VENDOR_DOWN) } } function format(response) { log('info', 'origin response size is : ' + response.resultatRetour.length) const data = response.resultatRetour .slice(1) .filter(value => value.ValeurIndex) const dataLen = data.length log('info', 'filtered size is : ' + dataLen) const mapData = data.map((value, index) => { const time = moment(value.DateReleve, moment.ISO_8601) if (index + 1 < dataLen) { return { load: data[index + 1].ValeurIndex - value.ValeurIndex, year: parseInt(time.format('YYYY')), month: parseInt(time.format('M')), day: parseInt(time.format('D')), hour: 0, minute: 0, type: value.TypeAgregat } } else { return { load: null, year: parseInt(time.format('YYYY')), month: parseInt(time.format('M')), day: parseInt(time.format('D')), hour: 0, minute: 0, type: value.TypeAgregat } } }) const res = [...mapData].filter(v => v.load !== null) return res } // function processYearAggregation(data, doctype) { // log("info", "Start aggregation for : " + doctype); // const grouped = data.reduce(reduceYearFunction, {}); // return Object.values(grouped); // } // function processMonthlyAggregation(data, range) { // log("info", "Start aggregation for : " + range.doctype); // // Filter by year // const tmpData = groupBy(data, "year"); // const keys = Object.keys(tmpData); // var dataToStore = []; // // Monthly aggregation // for (const index in keys) { // // Get daily data of a year // var monthlyData = tmpData[keys[index]]; // // Monthly aggregation // var aggregatedData = monthlyData.reduce(reduceMonthFunction, {}); // // Store it // dataToStore = dataToStore.concat(Object.values(aggregatedData)); // } // return dataToStore; // } // function groupBy(xs, key) { // return xs.reduce(function(rv, x) { // (rv[x[key]] = rv[x[key]] || []).push(x); // return rv; // }, {}); // } // function reduceYearFunction(acc, x) { // var id = acc[x.year]; // if (id) { // id.load += x.load; // } else { // acc[x.year] = x; // } // return acc; // } // function reduceMonthFunction(acc, x) { // var id = acc[x.month]; // if (id) { // id.load += x.load; // } else { // acc[x.month] = x; // } // return acc; // } /** * Save data in the right doctype db and prevent duplicated keys */ async function storeData(data, doctype, filterKeys) { log('debug', 'Store into ' + doctype) log('debug', 'Store into keys : ' + filterKeys) // data.map(v => { // log("info", "Saving data " + v.load + " for " + v.day + "/" + v.month + "/" + v.year); // }); const filteredDocuments = await hydrateAndFilter(data, doctype, { keys: filterKeys }) return await addData(filteredDocuments, doctype) } /** * Format an entry for DB storage * using key and value * For year doctype: key = "YYYY" * For month doctype: key = "YYYY-MM" */ async function buildDataFromKey(doctype, key, value) { let year, month, day, hour if (doctype === 'com.grandlyon.egl.year') { year = key month = 1 day = 0 hour = 0 } else if (doctype === 'com.grandlyon.egl.month') { const split = key.split('-') year = split[0] month = split[1] day = 0 hour = 0 } else { const split = key.split('-') year = split[0] month = split[1] day = split[2] hour = split[3] } return { load: Math.round(value * 10000) / 10000, year: parseInt(year), month: parseInt(month), day: parseInt(day), hour: parseInt(hour), minute: 0 } } /** * Function handling special case. * The temporary aggregated data need to be remove in order for the most recent one te be saved. * ex for com.grandlyon.egl.month : * { load: 76.712, month: 2020, ... } need to be replace by * { load: 82.212, month: 2020, ... } after egl data reprocess */ async function resetInProgressAggregatedData(data, doctype) { // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs log('debug', 'Remove aggregated data for ' + doctype) const result = await cozyClient.data.findAll(doctype) if (result && result.length > 0) { // Filter data to remove var filtered = [] if (doctype === 'com.grandlyon.egl.year') { // Yearly case filtered = result.filter(function(el) { return el.year == data.year }) } else if (doctype === 'com.grandlyon.egl.month') { // Monthly case filtered = result.filter(function(el) { return el.year == data.year && el.month == data.month }) } else { // Hourly case filtered = result.filter(function(el) { return ( el.year == data.year && el.month == data.month && el.day == data.day && el.hour == data.hour ) }) } // Remove data let sum = 0.0 for (const doc of filtered) { sum += doc.load log('debug', 'Removing this entry for ' + doc.load) await cozyClient.data.delete(doctype, doc) } return sum } return 0.0 }