index.js

// @ts-check
const {
  BaseKonnector,
  hydrateAndFilter,
  errors,
  log,
  updateOrCreate
} = require('cozy-konnector-libs')
const getDataGenericErrors = require('./helpers/getDataGenericErrors')
const { isDev } = require('./helpers/env')
const moment = require('moment')
require('moment-timezone')
moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone
const Sentry = require('@sentry/node')
// eslint-disable-next-line
const Tracing = require('@sentry/tracing') // Needed for tracking performance in Sentry
const { version } = require('../package.json')
const { getAuthToken, getConsents } = require('./requests/grdf')
const { handleConsents, createConsent } = require('./core/core')
const { rangeDate } = require('./constants')
const {
  aggregateMonthlyLoad,
  filterFirstMonthlyLoad,
  aggregateYearlyLoad,
  filterFirstYearlyLoad
} = require('./helpers/utils')
const { formatData } = require('./helpers/format')

Sentry.init({
  dsn:
    'https://fa503fe00434433f805d1c715999b7f5@grandlyon.errors.cozycloud.cc/3',

  // Set tracesSampleRate to 1.0 to capture 100%
  // of transactions for performance monitoring.
  // We recommend adjusting this value in production
  tracesSampleRate: 1.0,
  release: version,
  environment: isDev() ? 'development' : 'production',
  debug: isDev(),
  integrations: [
    // enable HTTP calls tracing
    new Sentry.Integrations.Http({ tracing: true })
  ]
})

Sentry.setTag('method', 'TIERS-DIRECT')

const NO_DATA = process.env.NO_DATA === 'true'
const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === 'true'
// dates related to consents
const startDate = moment().subtract(3, 'year')
const endDate = moment()
  .startOf('day')
  .add(1, 'year')
// dates related to getting data
const dataStartDate = moment().subtract(manualExecution ? 1 : 3, 'year')
const dataEndDate = moment().startOf('day')

module.exports = new BaseKonnector(start)

/**
 * The start function is run by the BaseKonnector instance only when it got all the account
 * information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
 * the account information come from ./konnector-dev-config.json file
 * cozyParameters are static parameters, independents from the account. Most often, it can be a
 * secret api key.
 * @param {import('./types').fields} fields
 * @param {{secret: import('./types').parameters}} cozyParameters
 */
async function start(fields, cozyParameters) {
  log('info', `isManual execution: ${manualExecution}`)
  log('debug', `FIELDS ${JSON.stringify(fields)}`)
  log('debug', `COZY_PARAMETERS ${JSON.stringify(cozyParameters)}`)

  if (NO_DATA) {
    log(
      'debug',
      'NO_DATA is enabled, konnector will stop before creating GRDF consent'
    )
  }

  let { pce, email, firstname, lastname, postalCode } = fields

  if (!pce && fields?.oauth_callback_results?.pce) {
    pce = fields.oauth_callback_results.pce
    log('info', `OAuth callback result found, using pce ${pce}`)
  }

  const transaction = Sentry.startTransaction({
    op: 'konnector',
    name: 'start',
    tags: { pce }
  })
  let boToken = ''
  let boBaseUrl = ''
  let grdfId = ''
  let grdfSecret = ''

  if (cozyParameters && Object.keys(cozyParameters).length !== 0) {
    log('debug', 'Found COZY_PARAMETERS')
    boToken = cozyParameters.secret.boToken
    boBaseUrl = cozyParameters.secret.boBaseUrl
    grdfId = cozyParameters.secret.client_id
    grdfSecret = cozyParameters.secret.client_secret
  }
  const boUrlGRDF = new URL('/api/grdf', boBaseUrl).href

  try {
    if (!pce) {
      log('error', 'No PCE found')
      throw errors.VENDOR_DOWN
    }
    log('info', `using PCE: ${pce}`)
    const { access_token } = await getAuthToken(grdfId, grdfSecret)
    const consents = await getConsents(access_token, pce)
    const noValidConsent = await handleConsents(consents, boUrlGRDF, boToken)

    if (NO_DATA) {
      log('debug', `Stopping konnector before creating consents`)
      process.exit()
    }

    if (noValidConsent) {
      await createConsent({
        bearerToken: access_token,
        pce,
        firstname,
        lastname,
        email,
        postalCode,
        startDate,
        endDate,
        boUrlGRDF,
        boToken
      })
    }

    const grdfData = await getData(
      access_token,
      pce,
      dataStartDate.format('YYYY-MM-DD'),
      dataEndDate.format('YYYY-MM-DD')
    )
    if (!grdfData) {
      log('debug', 'No data found')
      transaction.setStatus(Tracing.SpanStatus.Ok)
      transaction.finish()
      return
    }

    log('debug', 'Process GRDF daily data')
    const filteredDays = await hydrateAndFilter(
      grdfData,
      rangeDate.day.doctype,
      {
        keys: ['year', 'month', 'day', 'load']
      }
    )
    log('debug', 'Store GRDF daily load data')
    await updateOrCreate(
      filteredDays,
      rangeDate.day.doctype,
      rangeDate.day.keys
    )
    const { year: firstYear, month: firstMonth } = grdfData[0]

    log('debug', 'Aggregate GRDF yearly load data')
    const monthlyLoads = aggregateMonthlyLoad(grdfData)

    log('debug', 'Filter first month aggregate if already in database')
    const filteredMonthlyLoads = await filterFirstMonthlyLoad(
      firstMonth,
      firstYear,
      monthlyLoads
    )

    log('debug', 'Store aggregated GRDF monthly load data')
    await updateOrCreate(
      filteredMonthlyLoads,
      rangeDate.month.doctype,
      rangeDate.month.keys
    )

    log('debug', 'Aggregate GRDF yearly load data')
    const yearlyLoads = aggregateYearlyLoad(monthlyLoads)

    log('debug', 'Filter first year aggregate if already in database')
    const filteredYearlyLoads = await filterFirstYearlyLoad(
      firstYear,
      yearlyLoads
    )

    log('debug', 'Store aggregated GRDF yearly load data')
    await updateOrCreate(
      filteredYearlyLoads,
      rangeDate.year.doctype,
      rangeDate.year.keys
    )

    transaction.setStatus(Tracing.SpanStatus.Ok)
    transaction.finish()
  } catch (error) {
    log('error', 'Start failed', error)
    Sentry.captureException(error, {
      tags: {
        section: 'start',
        pce
      }
    })
    transaction.finish()
    await Sentry.flush()
    throw error
  }
}

/**
 * @param {string} idPCE
 * @param {string} startDate 'YYYY-MM-DD'
 * @param {string} endDate 'YYYY-MM-DD'
 * @returns {string}
 */
function buildGetDataUrl(idPCE, startDate, endDate) {
  const baseUrl = 'https://api.grdf.fr/adict/v2/pce'
  const queryParams = `date_debut=${startDate}&date_fin=${endDate}`
  return `${baseUrl}/${idPCE}/donnees_consos_informatives?${queryParams}`
}

/**
 * @param {string} token
 * @param {string} idPCE
 * @param {string} startDate 'YYYY-MM-DD'
 * @param {string} endDate 'YYYY-MM-DD'
 */
async function getData(token, idPCE, startDate, endDate) {
  const transaction = Sentry.startTransaction({
    op: 'konnector',
    name: 'getData',
    tags: { pce: idPCE }
  })
  log('debug', `getData from ${startDate} to ${endDate}`)
  const url = buildGetDataUrl(idPCE, startDate, endDate)

  log('debug', url)
  const response = await fetch(url, {
    method: 'GET',
    headers: {
      'Content-Type': 'application/x-ndjson',
      Authorization: `Bearer ${token}`
    },
    redirect: 'follow'
  })

  if (response.status !== 200) {
    log('error', `Response failed with status ${response.status}`)
    throw errors.VENDOR_DOWN
  }
  const result = await response.text()

  // Rebuild valid JSON from GRDF response
  /** @type {import('./types').GRDFDataRange[]} */
  const data = (result.match(/.+/g) || []).map(s => {
    return JSON.parse(s)
  })

  /** @type {import('./types').FormattedData[]} */
  const formattedData = []

  for (const result of data) {
    if (result.statut_restitution === null) {
      formattedData.push(formatData(result.consommation))
      continue
    }
    /**
     * Handle no data issue when retrieving grdf data.
     * 1000008 code stands for "Il n'y a pas de données correspondant à ce PCE sur la période demandée".
     * It is NOT an important issue deserving to throw an error
     * If there is no data, return null data in order to be filtered before saving
     */
    if (result.statut_restitution.code === '1000008') {
      continue
    }

    const genError = getDataGenericErrors(result.statut_restitution.code)
    log(
      'warn',
      'donnees_consos_informatives responded with : ' +
        result.statut_restitution.code +
        ' -> ' +
        result.statut_restitution.message +
        ' Periode ' +
        result.periode.date_debut +
        '/' +
        result.periode.date_fin
    )
    Sentry.captureMessage(
      `Get data threw an error: ${result.statut_restitution.code} - ${result.statut_restitution.message}`,
      {
        tags: {
          section: 'getData'
        }
      }
    )
    throw genError
  }
  transaction.finish()
  return formattedData
}