index.js

// @ts-check
const {
  BaseKonnector,
  log,
  hydrateAndFilter,
  addData,
} = require('cozy-konnector-libs')
const soapRequest = require('easy-soap-request')
const moment = require('moment')
require('moment-timezone')
const xml2js = require('xml2js')
const { buildAgregatedData } = require('./aggregate')
const { parseSgeXmlData, parseSgeXmlTechnicalData } = require('./parsing')
const { userTechnicalData, userMesureDetailles } = require('./request')
moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone

/*** Connector Constants ***/
const manualExecution =
  process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false
let startDailyDate = manualExecution
  ? moment().subtract(12, 'month')
  : moment().subtract(6, 'month')
let startDailyDateString = startDailyDate.format('YYYY-MM-DD')
const startLoadDate = moment().subtract(7, 'day')
const endDate = moment()
const endDateString = endDate.format('YYYY-MM-DD')

module.exports = new BaseKonnector(start)

// The start function is run by the BaseKonnector instance only when it got all the account
// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
// the account information come from ./konnector-dev-config.json file
// cozyParameters are static parameters, independents from the account. Most often, it can be a
// secret api key.
async function start(fields, cozyParameters) {
  log('info', 'Gathering data ...')
  let baseUrl = fields.wso2BaseUrl
  let apiAuthKey = fields.apiToken
  let loginUtilisateur = fields.loginUtilisateur
  log('info', 'Authenticating ...')
  if (cozyParameters && Object.keys(cozyParameters).length !== 0) {
    log('debug', 'Found COZY_PARAMETERS')
    baseUrl = cozyParameters.secret.wso2BaseUrl
    apiAuthKey = cozyParameters.secret.apiToken
    loginUtilisateur = cozyParameters.secret.loginUtilisateur
  }
  //TODO: authentification ?
  log('info', 'Successfully logged in')

  log('info', 'Querying data...')
  await getDataStartDate(
    `${baseUrl}/enedis_SGE_ConsultationDonneesTechniquesContractuelles/1.0`,
    apiAuthKey,
    loginUtilisateur,
    fields.pointId
  )
  await getData(
    `${baseUrl}/enedis_SGE_ConsultationMesuresDetaillees/1.0`,
    apiAuthKey,
    loginUtilisateur,
    fields.pointId
  )
  await getDataHalfHour(
    `${baseUrl}/enedis_SGE_ConsultationMesuresDetaillees/1.0`,
    apiAuthKey,
    loginUtilisateur,
    fields.pointId
  )
  log('info', 'Querying data: done')
}
/**
 *
 * @param {string} url
 * @param {string} apiAuthKey
 * @param {string} userLogin
 * @param {number} pointId
 */
async function getDataStartDate(url, apiAuthKey, userLogin, pointId) {
  log('info', 'Fetching data start date')
  const sampleHeaders = {
    'Content-Type': 'text/xml;charset=UTF-8',
    apikey: apiAuthKey,
  }

  const { response } = await soapRequest({
    url: url,
    headers: sampleHeaders,
    xml: userTechnicalData(pointId, userLogin),
  }).catch(err => {
    log('error', 'technicalDataResponse')
    log('error', err)
    return err
  })

  xml2js.parseString(
    response.body,
    {
      tagNameProcessors: [parseTags],
      valueProcessors: [parseValue],
      explicitArray: false,
    },
    processStartDate()
  )
}

/**
 * Get hour data
 * @param {string} url
 * @param {string} apiAuthKey
 * @param {string} userLogin
 * @param {number} pointId
 */
async function getData(url, apiAuthKey, userLogin, pointId) {
  log('info', 'Fetching data')
  const sampleHeaders = {
    'Content-Type': 'text/xml;charset=UTF-8',
    apikey: apiAuthKey,
  }

  // If start date exceed the maximum amount of data we can get with one query
  // get only 24 month
  if (moment(endDate).diff(startDailyDate, 'months', true) > 24) {
    log(
      'info',
      'Start date exceed 24 month, setting start date to current date minus 24 month'
    )
    startDailyDate = moment(endDate).subtract(24, 'month')
    startDailyDateString = startDailyDate.format('YYYY-MM-DD')
  }

  const { response } = await soapRequest({
    url: url,
    headers: sampleHeaders,
    xml: userMesureDetailles(
      pointId,
      userLogin,
      startDailyDateString,
      endDateString
    ),
  }).catch(err => {
    log('error', 'userMesureDetailles')
    log('error', err)
    return err
  })

  xml2js.parseString(
    response.body,
    {
      tagNameProcessors: [parseTags],
      valueProcessors: [parseValue],
      explicitArray: false,
    },
    processData()
  )
}

/**
 * Get half-hour data
 * @param {string} url
 * @param {string} apiAuthKey
 * @param {string} userLogin
 * @param {number} pointId
 */
async function getDataHalfHour(url, apiAuthKey, userLogin, pointId) {
  log('info', 'Fetching data')
  const sampleHeaders = {
    'Content-Type': 'text/xml;charset=UTF-8',
    apikey: apiAuthKey,
  }

  let MAX_HISTO = 4
  // If manual execution, retrieve only 1 week
  if (manualExecution) {
    MAX_HISTO = 1
  }
  for (var i = 0; i < MAX_HISTO; i++) {
    log('info', 'launch process with history')
    const increamentedStartDateString = moment(startLoadDate)
      .subtract(7 * i, 'day')
      .format('YYYY-MM-DD')
    const incrementedEndDateString = moment(endDate)
      .subtract(7 * i, 'day')
      .format('YYYY-MM-DD')
    const { response } = await soapRequest({
      url: url,
      headers: sampleHeaders,
      xml: userMesureDetailles(
        pointId,
        userLogin,
        increamentedStartDateString,
        incrementedEndDateString,
        'COURBE',
        'PA'
      ),
    }).catch(err => {
      log('error', 'userMesureDetailles half-hour')
      log('error', err)
      return err
    })

    xml2js.parseString(
      response.body,
      {
        tagNameProcessors: [parseTags],
        valueProcessors: [parseValue],
        explicitArray: false,
      },
      processData('com.grandlyon.enedis.minute')
    )
  }
}

/**
 * Format tag in order to be manipulated easly
 * @param {string} name
 * @returns {string} name
 */
function parseTags(name) {
  if (name.split(':')[1] !== undefined) {
    return name.split(':')[1]
  }
  return name
}

/**
 *
 * @param {string} value
 * @param {string} name
 * @returns {string|number} value
 */
function parseValue(value, name) {
  // Wh => KWh
  if (name === 'v') {
    return parseFloat((parseInt(value) / 1000).toFixed(2))
  }
  return value
}

/**
 * Parse data
 * @param {string} doctype
 * @returns
 */
function processData(doctype = 'com.grandlyon.enedis.day') {
  return async (err, result) => {
    if (err) {
      log('error', err)
      throw err
    }
    // Return only needed part of info
    const data = parseSgeXmlData(result)
    const processedDailyData = await storeData(
      await formateDataForDoctype(data),
      doctype,
      ['year', 'month', 'day', 'hour', 'minute']
    )

    log('info', 'Agregate enedis daily data for month and year')
    if (doctype === 'com.grandlyon.enedis.day') {
      console.log(processedDailyData.length)

      await agregateMonthAndYearData(processedDailyData)
    }
  }
}

/**
 * Parse data
 */
function processStartDate() {
  return async (err, result) => {
    if (err) {
      log('error', err)
      throw err
    }
    // update start Date with contract openning date
    startDailyDate = moment(parseSgeXmlTechnicalData(result), 'YYYY-MM-DD')
    startDailyDateString = startDailyDate.format('YYYY-MM-DD')
  }
}

/**
 * Save data in the right doctype db and prevent duplicated keys
 * @param {EnedisKonnectorData[]} data
 * @param {string} doctype
 * @param {string[]} filterKeys
 * @returns {Promise<*>}
 */
async function storeData(data, doctype, filterKeys) {
  log('debug', doctype, 'Store into')
  const filteredDocuments = await hydrateAndFilter(data, doctype, {
    keys: filterKeys,
  })
  await addData(filteredDocuments, doctype)
  return filteredDocuments
}

/**
 * Format data for DB storage
 * @param {SGEData[]} data
 * @returns {Promise<EnedisKonnectorData[]>} Parsed timestamp array
 */
async function formateDataForDoctype(data) {
  log('info', 'Formating data')
  return data.map(record => {
    let date = moment(record.d, 'YYYY/MM/DD h:mm:ss')
    return {
      load: record.v,
      year: parseInt(date.format('YYYY')),
      month: parseInt(date.format('M')),
      day: parseInt(date.format('D')),
      hour: parseInt(date.format('H')),
      minute: parseInt(date.format('m')),
    }
  })
}

/**
 * Agregate data from daily data to monthly and yearly data
 */
async function agregateMonthAndYearData(data) {
  // Sum year and month values into object with year or year-month as keys
  if (data && data.length > 0) {
    let monthData = {}
    let yearData = {}
    data.forEach(element => {
      element.year + '-' + element.month in monthData
        ? (monthData[element.year + '-' + element.month] += element.load)
        : (monthData[element.year + '-' + element.month] = element.load)
      element.year in yearData
        ? (yearData[element.year] += element.load)
        : (yearData[element.year] = element.load)
    })
    // Agregation for Month data
    const agregatedMonthData = await buildAgregatedData(
      monthData,
      'com.grandlyon.enedis.month'
    )
    await storeData(agregatedMonthData, 'com.grandlyon.enedis.month', [
      'year',
      'month',
    ])
    // Agregation for Year data
    const agregatedYearData = await buildAgregatedData(
      yearData,
      'com.grandlyon.enedis.year'
    )
    await storeData(agregatedYearData, 'com.grandlyon.enedis.year', ['year'])
  }
}