const { BaseKonnector, log, errors, addData, hydrateAndFilter, cozyClient } = require("cozy-konnector-libs"); // const fetch = require('node-fetch') const rp = require("request-promise"); const moment = require("moment"); require("moment-timezone"); moment.locale("fr"); // set the language moment.tz.setDefault("Europe/Paris"); // set the timezone const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === "true" ? true : false; const startDate = manualExecution ? moment() .subtract(1, "year") .format("MM/DD/YYYY") : moment() .subtract(3, "year") .format("MM/DD/YYYY"); const endDate = moment().format("MM/DD/YYYY"); // const timeRange = ['day', 'month', 'year'] const rangeDate = { day: { doctype: "com.grandlyon.egl.day", keys: ["year", "month", "day"] }, month: { doctype: "com.grandlyon.egl.month", keys: ["year", "month"] }, year: { doctype: "com.grandlyon.egl.year", keys: ["year"] } }; module.exports = new BaseKonnector(start); // The start function is run by the BaseKonnector instance only when it got all the account // information (fields). When you run this connector yourself in "standalone" mode or "dev" mode, // the account information come from ./konnector-dev-config.json file async function start(fields, cozyParameters) { try { // Local debug data // const baseUrl = fields.eglBaseURL // const apiAuthKey = fields.eglAPIAuthKey const baseUrl = cozyParameters.secret.eglBaseURL; const apiAuthKey = cozyParameters.secret.eglAPIAuthKey; log("info", "Authenticating ..."); const response = await authenticate( fields.login, fields.password, baseUrl, apiAuthKey ); log("info", "Successfully logged in"); const eglData = await getData(response, baseUrl, apiAuthKey); if (eglData) { log("debug", "Process egl daily data"); const processedLoadData = await processData( eglData, rangeDate.day.doctype, rangeDate.day.keys ); log("debug", "Agregate egl load data for month and year"); await agregateMonthAndYearData(processedLoadData); } else { log("debug", "No data found"); } } catch (error) { throw new Error(error.message); } } /** * Parse data * Remove existing data from DB using hydrateAndFilter * Store filtered data * Return the list of filtered data */ async function processData(data, doctype, filterKeys) { // const formatedData = await formateData(data) log("debug", "processData - data formated"); // Remove data for existing days into the DB const filteredData = await hydrateAndFilter(data, doctype, { keys: filterKeys }); log("debug", "processData - data filtered"); // Store new day data await storeData(filteredData, doctype, filterKeys); return filteredData; } /** * Agregate data from daily data to monthly and yearly data */ async function agregateMonthAndYearData(data) { // Sum year and month values into object with year or year-month as keys if (data && data.length !== 0) { let monthData = {}; let yearData = {}; data.forEach(element => { element.year + "-" + element.month in monthData ? (monthData[element.year + "-" + element.month] += element.load) : (monthData[element.year + "-" + element.month] = element.load); element.year in yearData ? (yearData[element.year] += element.load) : (yearData[element.year] = element.load); }); // Agregation for Month data const agregatedMonthData = await buildAgregatedData( monthData, "com.grandlyon.egl.month" ); await storeData(agregatedMonthData, "com.grandlyon.egl.month", [ "year", "month" ]); // Agregation for Year data const agregatedYearData = await buildAgregatedData( yearData, "com.grandlyon.egl.year" ); await storeData(agregatedYearData, "com.grandlyon.egl.year", ["year"]); } } /** * Retrieve and remove old data for a specific doctype * Return an Array of agregated data */ async function buildAgregatedData(data, doctype) { log("info", "entering buildAgregatedData"); let agregatedData = []; for (let [key, value] of Object.entries(data)) { const data = await buildDataFromKey(doctype, key, value); const oldValue = await resetInProgressAggregatedData(data, doctype); log("info", "Dataload + oldvalue is " + data.load + " + " + oldValue); data.load += oldValue; agregatedData.push(data); } return agregatedData; } async function authenticate(login, password, baseUrl, apiAuthKey) { const authRequest = { method: "POST", uri: baseUrl + "/connect.aspx", headers: { AuthKey: apiAuthKey, "Content-Type": "application/x-www-form-urlencoded" }, formData: { login: login, pass: password }, json: true }; const response = await rp(authRequest); if (response.codeRetour === 100) { return response; } else { throw new Error(errors.LOGIN_FAILED); } } async function getData(response, baseUrl, apiAuthKey) { log("debug", "Start date : " + startDate); log("debug", "End date : " + endDate); const dataRequest = { method: "POST", uri: baseUrl + "/getAllAgregatsByAbonnement.aspx", headers: { AuthKey: apiAuthKey, "Content-Type": "application/x-www-form-urlencoded" }, form: { token: response.resultatRetour.token, num_abt: response.resultatRetour.num_abt, date_debut: startDate, date_fin: endDate }, json: true }; try { // Sort data by date const responseEgl = await rp(dataRequest).then(eglRawData => { eglRawData.resultatRetour.sort(function(a, b) { return new Date(a.DateReleve) - new Date(b.DateReleve); }); return eglRawData; }); switch (responseEgl.codeRetour) { case 100: return format(responseEgl); case -2: throw errors.LOGIN_FAILED; case -1: throw errors.VENDOR_DOWN; default: throw errors.UNKNOWN_ERROR; } } catch (error) { log("debug", "Error from getAllAgregatsByAbonnement"); throw new Error(errors.VENDOR_DOWN); } } function format(response) { log("info", "origin response size is : " + response.resultatRetour.length); // Store first value as reference for index processing let refValue = response.resultatRetour[0]; // Create copy of data without first value const data = response.resultatRetour .slice(1) .filter(value => value.ValeurIndex); log("info", "filtered size is : " + data.length); return data.map(value => { const time = moment(value.DateReleve, moment.ISO_8601); const procesedLoad = value.ValeurIndex - refValue.ValeurIndex; if (procesedLoad < 0) { log( "error", `processing load for day ${parseInt(time.format("D"))}/${parseInt( time.format("M") )}/${parseInt(time.format("YYYY"))}, value is : ${procesedLoad}` ); throw errors.VENDOR_DOWN; } // Change index ref value refValue = value; return { load: procesedLoad, year: parseInt(time.format("YYYY")), month: parseInt(time.format("M")), day: parseInt(time.format("D")), hour: 0, minute: 0, type: value.TypeAgregat }; }); } /** * Save data in the right doctype db and prevent duplicated keys */ async function storeData(data, doctype, filterKeys) { log("debug", "Store into " + doctype); log("debug", "Store into keys : " + filterKeys); // data.map(v => { // log("info", "Saving data " + v.load + " for " + v.day + "/" + v.month + "/" + v.year); // }); const filteredDocuments = await hydrateAndFilter(data, doctype, { keys: filterKeys }); return await addData(filteredDocuments, doctype); } /** * Format an entry for DB storage * using key and value * For year doctype: key = "YYYY" * For month doctype: key = "YYYY-MM" */ async function buildDataFromKey(doctype, key, value) { let year, month, day, hour; if (doctype === "com.grandlyon.egl.year") { year = key; month = 1; day = 0; hour = 0; } else if (doctype === "com.grandlyon.egl.month") { const split = key.split("-"); year = split[0]; month = split[1]; day = 0; hour = 0; } else { const split = key.split("-"); year = split[0]; month = split[1]; day = split[2]; hour = split[3]; } return { load: Math.round(value * 10000) / 10000, year: parseInt(year), month: parseInt(month), day: parseInt(day), hour: parseInt(hour), minute: 0 }; } /** * Function handling special case. * The temporary aggregated data need to be remove in order for the most recent one te be saved. * ex for com.grandlyon.egl.month : * { load: 76.712, month: 2020, ... } need to be replace by * { load: 82.212, month: 2020, ... } after egl data reprocess */ async function resetInProgressAggregatedData(data, doctype) { // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs log("debug", "Remove aggregated data for " + doctype); const result = await cozyClient.data.findAll(doctype); if (result && result.length > 0) { // Filter data to remove var filtered = []; if (doctype === "com.grandlyon.egl.year") { // Yearly case filtered = result.filter(function(el) { return el.year == data.year; }); } else if (doctype === "com.grandlyon.egl.month") { // Monthly case filtered = result.filter(function(el) { return el.year == data.year && el.month == data.month; }); } else { // Hourly case filtered = result.filter(function(el) { return ( el.year == data.year && el.month == data.month && el.day == data.day && el.hour == data.hour ); }); } // Remove data let sum = 0.0; for (const doc of filtered) { sum += doc.load; log("debug", "Removing this entry for " + doc.load); await cozyClient.data.delete(doctype, doc); } return sum; } return 0.0; }