// @ts-check const { BaseKonnector, log, hydrateAndFilter, addData, } = require('cozy-konnector-libs') const soapRequest = require('easy-soap-request') const moment = require('moment') require('moment-timezone') const xml2js = require('xml2js') const { buildAgregatedData } = require('./aggregate') const { parseSgeXmlData, parseSgeXmlTechnicalData } = require('./parsing') const { userTechnicalData, userMesureDetailles } = require('./request') moment.locale('fr') // set the language moment.tz.setDefault('Europe/Paris') // set the timezone /*** Connector Constants ***/ const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false let startDailyDate = manualExecution ? moment().subtract(12, 'month') : moment().subtract(6, 'month') let startDailyDateString = startDailyDate.format('YYYY-MM-DD') const startLoadDate = moment().subtract(7, 'day') const endDate = moment() const endDateString = endDate.format('YYYY-MM-DD') module.exports = new BaseKonnector(start) // The start function is run by the BaseKonnector instance only when it got all the account // information (fields). When you run this connector yourself in "standalone" mode or "dev" mode, // the account information come from ./konnector-dev-config.json file // cozyParameters are static parameters, independents from the account. Most often, it can be a // secret api key. async function start(fields, cozyParameters) { log('info', 'Gathering data ...') let baseUrl = fields.wso2BaseUrl let apiAuthKey = fields.apiToken let loginUtilisateur = fields.loginUtilisateur log('info', 'Authenticating ...') if (cozyParameters && Object.keys(cozyParameters).length !== 0) { log('debug', 'Found COZY_PARAMETERS') baseUrl = cozyParameters.secret.wso2BaseUrl apiAuthKey = cozyParameters.secret.apiToken loginUtilisateur = cozyParameters.secret.loginUtilisateur } //TODO: authentification ? log('info', 'Successfully logged in') log('info', 'Querying data...') await getDataStartDate( `${baseUrl}/enedis_SGE_ConsultationDonneesTechniquesContractuelles/1.0`, apiAuthKey, loginUtilisateur, fields.pointId ) await getData( `${baseUrl}/enedis_SGE_ConsultationMesuresDetaillees/1.0`, apiAuthKey, loginUtilisateur, fields.pointId ) await getDataHalfHour( `${baseUrl}/enedis_SGE_ConsultationMesuresDetaillees/1.0`, apiAuthKey, loginUtilisateur, fields.pointId ) log('info', 'Querying data: done') } /** * * @param {string} url * @param {string} apiAuthKey * @param {string} userLogin * @param {number} pointId */ async function getDataStartDate(url, apiAuthKey, userLogin, pointId) { log('info', 'Fetching data start date') const sampleHeaders = { 'Content-Type': 'text/xml;charset=UTF-8', apikey: apiAuthKey, } const { response } = await soapRequest({ url: url, headers: sampleHeaders, xml: userTechnicalData(pointId, userLogin), }).catch(err => { log('error', 'technicalDataResponse') log('error', err) return err }) xml2js.parseString( response.body, { tagNameProcessors: [parseTags], valueProcessors: [parseValue], explicitArray: false, }, processStartDate() ) } /** * Get hour data * @param {string} url * @param {string} apiAuthKey * @param {string} userLogin * @param {number} pointId */ async function getData(url, apiAuthKey, userLogin, pointId) { log('info', 'Fetching data') const sampleHeaders = { 'Content-Type': 'text/xml;charset=UTF-8', apikey: apiAuthKey, } // If start date exceed the maximum amount of data we can get with one query // get only 24 month if (moment(endDate).diff(startDailyDate, 'months', true) > 24) { log( 'info', 'Start date exceed 24 month, setting start date to current date minus 24 month' ) startDailyDate = moment(endDate).subtract(24, 'month') startDailyDateString = startDailyDate.format('YYYY-MM-DD') } const { response } = await soapRequest({ url: url, headers: sampleHeaders, xml: userMesureDetailles( pointId, userLogin, startDailyDateString, endDateString ), }).catch(err => { log('error', 'userMesureDetailles') log('error', err) return err }) xml2js.parseString( response.body, { tagNameProcessors: [parseTags], valueProcessors: [parseValue], explicitArray: false, }, processData() ) } /** * Get half-hour data * @param {string} url * @param {string} apiAuthKey * @param {string} userLogin * @param {number} pointId */ async function getDataHalfHour(url, apiAuthKey, userLogin, pointId) { log('info', 'Fetching data') const sampleHeaders = { 'Content-Type': 'text/xml;charset=UTF-8', apikey: apiAuthKey, } let MAX_HISTO = 4 // If manual execution, retrieve only 1 week if (manualExecution) { MAX_HISTO = 1 } for (var i = 0; i < MAX_HISTO; i++) { log('info', 'launch process with history') const increamentedStartDateString = moment(startLoadDate) .subtract(7 * i, 'day') .format('YYYY-MM-DD') const incrementedEndDateString = moment(endDate) .subtract(7 * i, 'day') .format('YYYY-MM-DD') const { response } = await soapRequest({ url: url, headers: sampleHeaders, xml: userMesureDetailles( pointId, userLogin, increamentedStartDateString, incrementedEndDateString, 'COURBE', 'PA' ), }).catch(err => { log('error', 'userMesureDetailles half-hour') log('error', err) return err }) xml2js.parseString( response.body, { tagNameProcessors: [parseTags], valueProcessors: [parseValue], explicitArray: false, }, processData('com.grandlyon.enedis.minute') ) } } /** * Format tag in order to be manipulated easly * @param {string} name * @returns {string} name */ function parseTags(name) { if (name.split(':')[1] !== undefined) { return name.split(':')[1] } return name } /** * * @param {string} value * @param {string} name * @returns {string|number} value */ function parseValue(value, name) { // Wh => KWh if (name === 'v') { return parseFloat((parseInt(value) / 1000).toFixed(2)) } return value } /** * Parse data * @param {string} doctype * @returns */ function processData(doctype = 'com.grandlyon.enedis.day') { return async (err, result) => { if (err) { log('error', err) throw err } // Return only needed part of info const data = parseSgeXmlData(result) const processedDailyData = await storeData( await formateDataForDoctype(data), doctype, ['year', 'month', 'day', 'hour', 'minute'] ) log('info', 'Agregate enedis daily data for month and year') if (doctype === 'com.grandlyon.enedis.day') { console.log(processedDailyData.length) await agregateMonthAndYearData(processedDailyData) } } } /** * Parse data */ function processStartDate() { return async (err, result) => { if (err) { log('error', err) throw err } // update start Date with contract openning date startDailyDate = moment(parseSgeXmlTechnicalData(result), 'YYYY-MM-DD') startDailyDateString = startDailyDate.format('YYYY-MM-DD') } } /** * Save data in the right doctype db and prevent duplicated keys * @param {EnedisKonnectorData[]} data * @param {string} doctype * @param {string[]} filterKeys * @returns {Promise<*>} */ async function storeData(data, doctype, filterKeys) { log('debug', doctype, 'Store into') const filteredDocuments = await hydrateAndFilter(data, doctype, { keys: filterKeys, }) await addData(filteredDocuments, doctype) return filteredDocuments } /** * Format data for DB storage * @param {SGEData[]} data * @returns {Promise<EnedisKonnectorData[]>} Parsed timestamp array */ async function formateDataForDoctype(data) { log('info', 'Formating data') return data.map(record => { let date = moment(record.d, 'YYYY/MM/DD h:mm:ss') return { load: record.v, year: parseInt(date.format('YYYY')), month: parseInt(date.format('M')), day: parseInt(date.format('D')), hour: parseInt(date.format('H')), minute: parseInt(date.format('m')), } }) } /** * Agregate data from daily data to monthly and yearly data */ async function agregateMonthAndYearData(data) { // Sum year and month values into object with year or year-month as keys if (data && data.length > 0) { let monthData = {} let yearData = {} data.forEach(element => { element.year + '-' + element.month in monthData ? (monthData[element.year + '-' + element.month] += element.load) : (monthData[element.year + '-' + element.month] = element.load) element.year in yearData ? (yearData[element.year] += element.load) : (yearData[element.year] = element.load) }) // Agregation for Month data const agregatedMonthData = await buildAgregatedData( monthData, 'com.grandlyon.enedis.month' ) await storeData(agregatedMonthData, 'com.grandlyon.enedis.month', [ 'year', 'month', ]) // Agregation for Year data const agregatedYearData = await buildAgregatedData( yearData, 'com.grandlyon.enedis.year' ) await storeData(agregatedYearData, 'com.grandlyon.enedis.year', ['year']) } }