index.js

// @ts-check
const {
  cozyClient,
  BaseKonnector,
  addData,
  hydrateAndFilter,
  errors,
  log
} = require('cozy-konnector-libs')
const getDataGenericErrors = require('./helpers/getDataGenericErrors')
const { isDev } = require('./helpers/env')
const moment = require('moment')
require('moment-timezone')
moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone
const Sentry = require('@sentry/node')
// eslint-disable-next-line
const Tracing = require('@sentry/tracing') // Needed for tracking performance in Sentry
const { version } = require('../package.json')
const { getAuthToken, getConsents } = require('./requests/grdf')
const { handleConsents, createConsent } = require('./core/core')

Sentry.init({
  dsn:
    'https://fa503fe00434433f805d1c715999b7f5@grandlyon.errors.cozycloud.cc/3',

  // Set tracesSampleRate to 1.0 to capture 100%
  // of transactions for performance monitoring.
  // We recommend adjusting this value in production
  tracesSampleRate: 1.0,
  release: version,
  environment: isDev() ? 'development' : 'production',
  debug: isDev(),
  integrations: [
    // enable HTTP calls tracing
    new Sentry.Integrations.Http({ tracing: true })
  ]
})

Sentry.setTag('method', 'TIERS-DIRECT')

const NO_DATA = process.env.NO_DATA === 'true'
const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === 'true'
const startDate = moment().subtract(manualExecution ? 1 : 3, 'year')
const endDate = moment().startOf('day')

module.exports = new BaseKonnector(start)

/**
 * The start function is run by the BaseKonnector instance only when it got all the account
 * information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
 * the account information come from ./konnector-dev-config.json file
 * cozyParameters are static parameters, independents from the account. Most often, it can be a
 * secret api key.
 * @param {import('./types').fields} fields
 * @param {{secret: import('./types').parameters}} cozyParameters
 */
async function start(fields, cozyParameters) {
  log('info', `isManual execution: ${manualExecution}`)
  log('debug', `FIELDS ${JSON.stringify(fields)}`)
  log('debug', `COZY_PARAMETERS ${JSON.stringify(cozyParameters)}`)

  if (NO_DATA) {
    log(
      'debug',
      'NO_DATA is enabled, konnector will stop before creating GRDF consent'
    )
  }

  const { pce, email, firstname, lastname, postalCode } = fields

  const transaction = Sentry.startTransaction({
    op: 'konnector',
    name: 'start',
    tags: { pce }
  })
  let boToken = ''
  let boBaseUrl = ''
  let grdfId = ''
  let grdfSecret = ''

  if (cozyParameters && Object.keys(cozyParameters).length !== 0) {
    log('debug', 'Found COZY_PARAMETERS')
    boToken = cozyParameters.secret.boToken
    boBaseUrl = cozyParameters.secret.boBaseUrl
    grdfId = cozyParameters.secret.client_id
    grdfSecret = cozyParameters.secret.client_secret
  }
  const boUrlGRDF = new URL('/api/grdf', boBaseUrl).href

  try {
    const { access_token } = await getAuthToken(grdfId, grdfSecret)
    const consents = await getConsents(access_token, pce)
    const noValidConsent = await handleConsents(consents, boUrlGRDF, boToken)

    if (NO_DATA) {
      log('debug', `Stopping konnector before creating consents`)
      process.exit()
    }

    if (noValidConsent) {
      await createConsent({
        bearerToken: access_token,
        pce,
        firstname,
        lastname,
        email,
        postalCode,
        startDate,
        endDate,
        boUrlGRDF,
        boToken
      })
    }

    const grdfData = await getData(
      access_token,
      pce,
      startDate.format('YYYY-MM-DD'),
      endDate.format('YYYY-MM-DD')
    )
    if (grdfData) {
      const processedLoadData = await processData(
        grdfData,
        'com.grandlyon.grdf.day',
        ['year', 'month', 'day']
      )

      await aggregateMonthAndYearData(processedLoadData)
    } else {
      log('debug', 'No consent or data for load curve')
    }
  } catch (error) {
    log('error', 'Start failed', error)
    Sentry.captureException(error, {
      tags: {
        section: 'start',
        pce
      }
    })
    transaction.finish()
    await Sentry.flush()
    throw error
  }
}

/**
 * @param {string} idPCE
 * @param {string} startDate 'YYYY-MM-DD'
 * @param {string} endDate 'YYYY-MM-DD'
 * @returns {string}
 */
function buildGetDataUrl(idPCE, startDate, endDate) {
  const baseUrl = 'https://api.grdf.fr/adict/v2/pce'
  const queryParams = `date_debut=${startDate}&date_fin=${endDate}`
  return `${baseUrl}/${idPCE}/donnees_consos_informatives?${queryParams}`
}

/**
 * @param {string} token
 * @param {string} idPCE
 * @param {string} startDate 'YYYY-MM-DD'
 * @param {string} endDate 'YYYY-MM-DD'
 */
async function getData(token, idPCE, startDate, endDate) {
  const transaction = Sentry.startTransaction({
    op: 'konnector',
    name: 'getData',
    tags: { pce: idPCE }
  })
  log('debug', `getData from ${startDate} to ${endDate}`)
  const url = buildGetDataUrl(idPCE, startDate, endDate)
  log('debug', url)
  const response = await fetch(url, {
    method: 'GET',
    headers: {
      'Content-Type': 'application/x-ndjson',
      Authorization: `Bearer ${token}`
    },
    redirect: 'follow'
  })
    .then(async response => {
      if (response.status !== 200) {
        log('error', `Response failed with status ${response.status}`)
        throw errors.VENDOR_DOWN
      }
      return response.text()
    })
    .then(result => {
      return result.match(/.+/g).map(s => {
        result = JSON.parse(s)
        if (result.statut_restitution !== null) {
          /**
           * Handle no data issue when retrieving grdf data.
           * 1000008 code stands for "Il n'y a pas de données correspondant à ce PCE sur la période demandée".
           * It is NOT an important issue deserving to throw an error
           * If there is no data, return null data in order to be filtered before saving
           */
          if (result.statut_restitution.code !== '1000008') {
            const genError = getDataGenericErrors(
              result.statut_restitution.code
            )
            log(
              'warn',
              'donnees_consos_informatives responded with : ' +
                result.statut_restitution.code +
                ' -> ' +
                result.statut_restitution.message +
                ' Periode ' +
                result.periode.date_debut +
                '/' +
                result.periode.date_fin
            )
            Sentry.captureMessage(
              `Get data threw an error: ${result.statut_restitution.code} - ${result.statut_restitution.message}`,
              {
                tags: {
                  section: 'getData'
                }
              }
            )
            throw genError
          } else {
            return { energie: null }
          }
        }
        return result.consommation
      })
    })
    .catch(error => {
      log('debug', 'Error from getData')
      throw error
    })
  const filteredRep = response.filter(function(el) {
    return el.energie != null || el.volume_brut != null
  })
  transaction.finish()
  return filteredRep
}

/**
 * Parse data
 * Remove existing data from DB using hydrateAndFilter
 * Store filtered data
 * Return the list of filtered data
 */
async function processData(data, doctype, filterKeys) {
  log('debug', 'Process grdf daily data')
  const formattedData = await formatData(data)
  log('debug', 'processData - data formatted')
  // Remove data for existing days into the DB
  const filteredData = await hydrateAndFilter(formattedData, doctype, {
    keys: filterKeys
  })
  log('debug', 'processData - data filtered')
  // Store new day data
  await storeData(filteredData, doctype, filterKeys)
  return filteredData
}

/**
 * Save data in the right doctype db and prevent duplicated keys
 */
async function storeData(data, doctype, filterKeys) {
  log('debug', doctype, 'Store into')
  const filteredDocuments = await hydrateAndFilter(data, doctype, {
    keys: filterKeys
  })
  return addData(filteredDocuments, doctype)
}

/**
 * Format data for DB storage
 * Remove bad data
 */
async function formatData(data) {
  log('debug', 'Formatting data')
  return data.map(record => {
    let date = moment(record.date_debut_consommation, 'YYYY/MM/DD h:mm:ss')
    let load =
      record.energie && record.energie !== 0
        ? record.energie
        : record.volume_brut * record.coeff_calcul.coeff_conversion
    return {
      load: parseFloat(load),
      year: parseInt(date.format('YYYY')),
      month: parseInt(date.format('M')),
      day: parseInt(date.format('D')),
      hour: parseInt(date.format('H')),
      minute: parseInt(date.format('m'))
    }
  })
}

/**
 * Aggregate data from daily data to monthly and yearly data
 */
async function aggregateMonthAndYearData(data) {
  log('debug', 'Aggregate grdf load data for month and year')
  // Sum year and month values into object with year or year-month as keys
  if (data && data.length !== 0) {
    let monthData = {}
    let yearData = {}

    data.forEach(element => {
      element.year + '-' + element.month in monthData
        ? (monthData[element.year + '-' + element.month] += element.load)
        : (monthData[element.year + '-' + element.month] = element.load)
      element.year in yearData
        ? (yearData[element.year] += element.load)
        : (yearData[element.year] = element.load)
    })
    // Aggregation for Month data
    const aggregatedMonthData = await buildAggregatedData(
      monthData,
      'com.grandlyon.grdf.month'
    )
    await storeData(aggregatedMonthData, 'com.grandlyon.grdf.month', [
      'year',
      'month'
    ])
    // Aggregation for Year data
    const aggregatedYearData = await buildAggregatedData(
      yearData,
      'com.grandlyon.grdf.year'
    )
    await storeData(aggregatedYearData, 'com.grandlyon.grdf.year', ['year'])
  }
}

/**
 * Retrieve and remove old data for a specific doctype
 * Return an Array of aggregated data
 */
async function buildAggregatedData(data, doctype) {
  let aggregatedData = []
  for (let [key, value] of Object.entries(data)) {
    const data = await buildDataFromKey(doctype, key, value)
    const oldValue = await resetInProgressAggregatedData(data, doctype)
    data.load += oldValue
    aggregatedData.push(data)
  }
  return aggregatedData
}

/**
 * Format an entry for DB storage
 * using key and value
 * For year doctype: key = "YYYY"
 * For month doctype: key = "YYYY-MM"
 */
async function buildDataFromKey(doctype, key, value) {
  let year, month, day, hour
  if (doctype === 'com.grandlyon.grdf.year') {
    year = key
    month = 1
    day = 0
    hour = 0
  } else if (doctype === 'com.grandlyon.grdf.month') {
    const split = key.split('-')
    year = split[0]
    month = split[1]
    day = 0
    hour = 0
  } else {
    const split = key.split('-')
    year = split[0]
    month = split[1]
    day = split[2]
    hour = split[3]
  }
  return {
    load: Math.round(value * 10000) / 10000,
    year: parseInt(year),
    month: parseInt(month),
    day: parseInt(day),
    hour: parseInt(hour),
    minute: 0
  }
}

/**
 * Function handling special case.
 * The temporary aggregated data need to be remove in order for the most recent one te be saved.
 * ex for com.grandlyon.grdf.year :
 * { load: 76.712, year: 2020, ... } need to be replace by
 * { load: 82.212, year: 2020, ... } after grdf data reprocess
 */
async function resetInProgressAggregatedData(data, doctype) {
  // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
  log('debug', doctype, 'Remove aggregated data for')
  const result = await cozyClient.data.findAll(doctype)
  if (result && result.length > 0) {
    // Filter data to remove
    var filtered = []
    if (doctype === 'com.grandlyon.grdf.year') {
      // Yearly case
      filtered = result.filter(function(el) {
        return el.year == data.year
      })
    } else if (doctype === 'com.grandlyon.grdf.month') {
      // Monthly case
      filtered = result.filter(function(el) {
        return el.year == data.year && el.month == data.month
      })
    } else {
      // Hourly case
      filtered = result.filter(function(el) {
        return (
          el.year == data.year &&
          el.month == data.month &&
          el.day == data.day &&
          el.hour == data.hour
        )
      })
    }
    // Remove data
    let sum = 0.0
    for (const doc of filtered) {
      sum += doc.load
      log('debug', doc, 'Removing this entry for ' + doctype)
      await cozyClient.data.delete(doctype, doc)
    }
    return sum
  }
  return 0.0
}