const { BaseKonnector, log, errors, addData, hydrateAndFilter, cozyClient, } = require('cozy-konnector-libs') const axios = require('axios').default const moment = require('moment') require('moment-timezone') moment.locale('fr') // set the language moment.tz.setDefault('Europe/Paris') // set the timezone const Sentry = require('@sentry/node') // eslint-disable-next-line const Tracing = require('@sentry/tracing') // Needed for tracking performance in Sentry const { version } = require('../package.json') const { isDev } = require('./helpers/env') const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false const startDate = manualExecution ? moment() .subtract(1, 'year') .format('MM/DD/YYYY') : moment() .subtract(3, 'year') .format('MM/DD/YYYY') const endDate = moment().format('MM/DD/YYYY') const rangeDate = { day: { doctype: 'com.grandlyon.egl.day', keys: ['year', 'month', 'day'], }, month: { doctype: 'com.grandlyon.egl.month', keys: ['year', 'month'], }, year: { doctype: 'com.grandlyon.egl.year', keys: ['year'], }, } module.exports = new BaseKonnector(start) /** * Sentry */ Sentry.init({ dsn: 'https://3f97baf46c2b44c2bd9e0c371abe3e05@grandlyon.errors.cozycloud.cc/2', // Set tracesSampleRate to 1.0 to capture 100% // of transactions for performance monitoring. // We recommend adjusting this value in production tracesSampleRate: 1.0, release: version, environment: isDev() ? 'development' : 'production', debug: isDev(), integrations: [ // enable HTTP calls tracing new Sentry.Integrations.Http({ tracing: true }), ], }) // The start function is run by the BaseKonnector instance only when it got all the account // information (fields). When you run this connector yourself in 'standalone' mode or 'dev' mode, // the account information come from ./konnector-dev-config.json file async function start(fields, cozyParameters) { const transaction = Sentry.startTransaction({ op: 'konnector', name: 'EGL Konnector', }) transaction.startChild({ op: 'Konnector starting' }) try { // Local debug data // const baseUrl = fields.eglBaseURL // const apiAuthKey = fields.eglAPIAuthKey const baseUrl = cozyParameters.secret.eglBaseURL const apiAuthKey = cozyParameters.secret.eglAPIAuthKey log('info', 'Authenticating ...') const response = await authenticate( fields.login, fields.password, baseUrl, apiAuthKey ) log('info', 'Successfully logged in') const eglData = await getData(response, baseUrl, apiAuthKey) if (eglData) { log('debug', 'Process egl daily data') const processedLoadData = await processData( eglData, rangeDate.day.doctype, rangeDate.day.keys ) log('debug', 'Aggregate egl load data for month and year') await aggregateMonthAndYearData(processedLoadData) } else { log('debug', 'No data found') } transaction.setStatus(Tracing.SpanStatus.Ok) transaction.finish() } catch (error) { const errorMessage = `EGL konnector encountered an error. Response data: ${JSON.stringify( error.message )}` Sentry.captureMessage(errorMessage, { tags: { section: 'start', }, }) transaction.setStatus(Tracing.SpanStatus.Aborted) transaction.finish() await Sentry.flush() throw error } } /** * Parse data * Remove existing data from DB using hydrateAndFilter * Store filtered data * Return the list of filtered data */ async function processData(data, doctype, filterKeys) { log('debug', 'processData - data formatted') // Remove data for existing days into the DB const filteredData = await hydrateAndFilter(data, doctype, { keys: filterKeys, }) log('debug', 'processData - data filtered') // Store new day data await storeData(filteredData, doctype, filterKeys) return filteredData } /** * Aggregate data from daily data to monthly and yearly data */ async function aggregateMonthAndYearData(data) { // Sum year and month values into object with year or year-month as keys if (data && data.length !== 0) { let monthData = {} let yearData = {} data.forEach(element => { const monthDataKey = element.year + '-' + element.month if (monthDataKey in monthData) { monthData[monthDataKey] += element.load } else { monthData[monthDataKey] = element.load } const yearDataKey = element.year if (yearDataKey in yearData) { yearData[yearDataKey] += element.load } else { yearData[yearDataKey] = element.load } }) // Aggregation for Month data const aggregatedMonthData = await buildAggregatedData( monthData, 'com.grandlyon.egl.month' ) await storeData(aggregatedMonthData, 'com.grandlyon.egl.month', [ 'year', 'month', ]) // Aggregation for Year data const aggregatedYearData = await buildAggregatedData( yearData, 'com.grandlyon.egl.year' ) await storeData(aggregatedYearData, 'com.grandlyon.egl.year', ['year']) } } /** * Retrieve and remove old data for a specific doctype * Return an Array of aggregated data */ async function buildAggregatedData(data, doctype) { log('info', 'entering buildAggregatedData') let aggregatedData = [] for (let [key, value] of Object.entries(data)) { const data = await buildDataFromKey(doctype, key, value) const oldValue = await resetInProgressAggregatedData(data, doctype) log('info', 'Data load + old value is ' + data.load + ' + ' + oldValue) data.load += oldValue aggregatedData.push(data) } return aggregatedData } async function authenticate(login, password, baseUrl, apiAuthKey) { log('info', 'Authenticating ...') const authRequest = { method: 'post', url: baseUrl + '/connect.aspx', headers: { AuthKey: apiAuthKey, 'Content-Type': 'application/x-www-form-urlencoded', }, data: { login: login, pass: password, }, } try { const resp = await axios(authRequest) if (resp.data.codeRetour === 100) { return resp.data } else if (resp.data.codeRetour === -4) { throw new Error(errors.LOGIN_FAILED) } const errorMessage = `Authentication failed. Response data: ${resp?.data?.libelleRetour}` log('error', errorMessage) throw new Error(errors.VENDOR_DOWN) } catch (error) { log('error', error.response?.data) Sentry.captureException(error, { tags: { section: 'authenticate', }, extra: { compte: login, }, }) throw error } } async function getData(response, baseUrl, apiAuthKey) { const dataRequest = { method: 'post', url: baseUrl + '/getAllAgregatsByAbonnement.aspx', headers: { AuthKey: apiAuthKey, 'Content-Type': 'application/x-www-form-urlencoded', }, data: { token: response.resultatRetour.token, num_abt: response.resultatRetour.num_abt, date_debut: startDate, date_fin: endDate, }, } try { // Sort data by date const resp = await axios(dataRequest) resp.data.resultatRetour.sort(function(a, b) { return new Date(a.DateReleve) - new Date(b.DateReleve) }) switch (resp.data.codeRetour) { case 100: return format(resp.data) case -2: log( 'error', `Get data failed. codeRetour -2. ${resp.data.libelleRetour}` ) throw errors.LOGIN_FAILED case -1: log( 'error', `Get data failed. codeRetour -1. ${resp.data.libelleRetour}` ) throw errors.VENDOR_DOWN default: log( 'error', `Get data failed. ${resp.data.codeRetour}. ${resp.data.libelleRetour}` ) throw errors.UNKNOWN_ERROR } } catch (error) { log('debug', error.message) Sentry.captureException(error, { tags: { section: 'getData', }, extra: { start: startDate, end: endDate, }, }) if (axios.isAxiosError(error)) { throw new Error(errors.VENDOR_DOWN) } throw error } } function format(response) { log('info', 'origin response size is: ' + response.resultatRetour.length) // Store first value as reference for index processing let refValue = response.resultatRetour[0] // Create copy of data without first value const data = response.resultatRetour .slice(1) .filter(value => value.ValeurIndex) log('info', 'filtered size is: ' + data.length) try { return data.map(value => { const time = moment(value.DateReleve, moment.ISO_8601) const processedLoad = value.ValeurIndex - refValue.ValeurIndex if (processedLoad < 0) { const errorMessage = `Processing load error for day ${parseInt( time.format('D') )}/${parseInt(time.format('M'))}/${parseInt( time.format('YYYY') )}, value is: ${processedLoad}` log('debug', errorMessage) throw errors.VENDOR_DOWN } // Change index ref value refValue = value return { load: processedLoad, year: parseInt(time.format('YYYY')), month: parseInt(time.format('M')), day: parseInt(time.format('D')), hour: 0, minute: 0, type: value.TypeAgregat, } }) } catch (error) { log('debug', error.message) Sentry.captureException(error, { tags: { section: 'format', }, }) throw error } } /** * Save data in the right doctype db and prevent duplicated keys */ async function storeData(data, doctype, filterKeys) { log('debug', 'Store into ' + doctype) log('debug', 'Store into keys : ' + filterKeys) const filteredDocuments = await hydrateAndFilter(data, doctype, { keys: filterKeys, }) return await addData(filteredDocuments, doctype) } /** * Format an entry for DB storage * using key and value * For year doctype: key = 'YYYY' * For month doctype: key = 'YYYY-MM' */ async function buildDataFromKey(doctype, key, value) { let year, month, day, hour if (doctype === 'com.grandlyon.egl.year') { year = key month = 1 day = 0 hour = 0 } else if (doctype === 'com.grandlyon.egl.month') { const split = key.split('-') year = split[0] month = split[1] day = 0 hour = 0 } else { const split = key.split('-') year = split[0] month = split[1] day = split[2] hour = split[3] } return { load: Math.round(value * 10000) / 10000, year: parseInt(year), month: parseInt(month), day: parseInt(day), hour: parseInt(hour), minute: 0, } } /** * Function handling special case. * The temporary aggregated data need to be remove in order for the most recent one te be saved. * ex for com.grandlyon.egl.month : * { load: 76.712, month: 2020, ... } need to be replace by * { load: 82.212, month: 2020, ... } after egl data reprocess */ async function resetInProgressAggregatedData(data, doctype) { // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs log('debug', 'Remove aggregated data for ' + doctype) const result = await cozyClient.data.findAll(doctype) if (result && result.length > 0) { // Filter data to remove let filtered = [] if (doctype === 'com.grandlyon.egl.year') { // Yearly case filtered = result.filter(function(el) { return el.year == data.year }) } else if (doctype === 'com.grandlyon.egl.month') { // Monthly case filtered = result.filter(function(el) { return el.year == data.year && el.month == data.month }) } else { // Hourly case filtered = result.filter(function(el) { return ( el.year == data.year && el.month == data.month && el.day == data.day && el.hour == data.hour ) }) } // Remove data let sum = 0.0 for (const doc of filtered) { sum += doc.load log('debug', 'Removing this entry for ' + doc.load) await cozyClient.data.delete(doctype, doc) } return sum } return 0.0 }