Skip to content
Snippets Groups Projects
index.js 11.8 KiB
Newer Older
  • Learn to ignore specific revisions
  • Rémi PAPIN's avatar
    Rémi PAPIN committed
    const {
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      cozyClient,
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      BaseKonnector,
      addData,
      hydrateAndFilter,
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
      errors,
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      log
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
    } = require('cozy-konnector-libs')
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
    const getAccountId = require('./helpers/getAccountId')
    
    const getDataGenericErrors = require('./helpers/getDataGenericErrors')
    const { isDev } = require('./helpers/env')
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
    const moment = require('moment')
    require('moment-timezone')
    moment.locale('fr') // set the language
    moment.tz.setDefault('Europe/Paris') // set the timezone
    
    
    const Sentry = require('@sentry/node')
    // eslint-disable-next-line
    const Tracing = require('@sentry/tracing') // Needed for tracking performance in Sentry
    const { version } = require('../package.json')
    
    Sentry.init({
      dsn:
        'https://fa503fe00434433f805d1c715999b7f5@grandlyon.errors.cozycloud.cc/3',
    
      // Set tracesSampleRate to 1.0 to capture 100%
      // of transactions for performance monitoring.
      // We recommend adjusting this value in production
      tracesSampleRate: 1.0,
      release: version,
    
      environment: isDev() ? 'development' : 'production',
      debug: isDev(),
    
      integrations: [
        // enable HTTP calls tracing
        new Sentry.Integrations.Http({ tracing: true })
      ]
    })
    
    
    async function standaloneStart(token, pce) {
      try {
        const grdfData = await getData(token, pce)
    
        if (!grdfData) {
          log('debug', 'No consent or data for load curve')
          return
        }
        log('debug', 'Process grdf daily data')
        const processedLoadData = await processData(
          grdfData,
          'com.grandlyon.grdf.day',
          ['year', 'month', 'day']
        )
        log('debug', 'Aggregate grdf load data for month and year')
        await aggregateMonthAndYearData(processedLoadData)
      } catch (error) {
    
    const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === 'true'
    
    const startDate = manualExecution
    
    Hugo SUBTIL's avatar
    Hugo SUBTIL committed
      ? moment()
          .subtract(1, 'year')
          .format('YYYY-MM-DD')
      : moment()
          .subtract(3, 'year')
          .format('YYYY-MM-DD')
    
    
    Yoan VALLET's avatar
    Yoan VALLET committed
    const endDate = moment()
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      .startOf('day')
    
    Yoan VALLET's avatar
    Yoan VALLET committed
      .subtract(1, 'day')
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      .format('YYYY-MM-DD')
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
    
    module.exports = new BaseKonnector(start)
    
    
    /**
     * The start function is run by the BaseKonnector instance only when it got all the account
     * information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
     * the account information come from ./konnector-dev-config.json file
     * cozyParameters are static parameters, independents from the account. Most often, it can be a
     * secret api key.
     */
    
    async function start(fields) {
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
      log('debug', 'Starting grdf adict konnector')
    
      if (process.env.NODE_ENV === 'standalone') {
    
        standaloneStart(
          fields.oauth.access_token,
          fields.oauth_callback_results.pce
        )
    
      const transaction = Sentry.startTransaction({
        op: 'konnector',
        name: 'GRDF Konnector'
      })
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
    
      try {
    
        const accountId = getAccountId()
        let body = ''
        let id_pce = ''
    
    
    Yoan VALLET's avatar
    Yoan VALLET committed
        body = await cozyClient.fetchJSON(
          'POST',
          `/accounts/grdfgrandlyon/${accountId}/refresh`
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
        )
    
    
    Yoan VALLET's avatar
    Yoan VALLET committed
        fields.access_token = body.attributes.oauth.access_token
    
        if (this._account?.oauth_callback_results?.pce && fields.access_token) {
    
    Yoan VALLET's avatar
    Yoan VALLET committed
          id_pce = this._account.oauth_callback_results.pce
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
    
    
    Yoan VALLET's avatar
    Yoan VALLET committed
          const grdfData = await getData(fields.access_token, id_pce)
          if (grdfData) {
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
            log('debug', 'Process grdf daily data')
            const processedLoadData = await processData(
              grdfData,
              'com.grandlyon.grdf.day',
              ['year', 'month', 'day']
            )
    
            log('debug', 'Aggregate grdf load data for month and year')
            await aggregateMonthAndYearData(processedLoadData)
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
          } else {
            log('debug', 'No consent or data for load curve')
          }
        } else {
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
          log('debug', 'no id_token found in oauth_callback_results')
          log(
            'debug',
            'callback_result contains: ',
            this._account.oauth_callback_results
          )
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
          throw errors.USER_ACTION_NEEDED_OAUTH_OUTDATED
        }
      } catch (err) {
    
    Yoan VALLET's avatar
    Yoan VALLET committed
        log('error', 'caught an unexpected error')
    
        log('debug', 'CATCH ERROR : ' + err)
    
    Yoan VALLET's avatar
    Yoan VALLET committed
        log('error', err.message)
    
        Sentry.captureException(err)
        await Sentry.flush()
    
      } finally {
        transaction.finish()
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      }
    }
    
    
    function buildGetDataUrl(idPCE, startDate, endDate) {
      const baseUrl = 'https://api.grdf.fr/adict/v2/pce'
      const queryParams = `date_debut=${startDate}&date_fin=${endDate}`
      return `${baseUrl}/${idPCE}/donnees_consos_informatives?${queryParams}`
    }
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
    
    
    async function getData(token, idPCE) {
      const url = buildGetDataUrl(idPCE, startDate, endDate)
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
        method: 'GET',
    
        headers: {
          'Content-Type': 'application/x-ndjson',
          Authorization: `Bearer ${token}`
        },
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
        redirect: 'follow'
    
    Yoan VALLET's avatar
    Yoan VALLET committed
        .then(async response => {
          if (response.status !== 200) {
    
            log('error', `Response failed with status ${response.status}`)
            throw errors.VENDOR_DOWN
    
    Yoan VALLET's avatar
    Yoan VALLET committed
          }
    
    Yoan VALLET's avatar
    Yoan VALLET committed
        })
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
        .then(result => {
    
    Yoan VALLET's avatar
    Yoan VALLET committed
          return result.match(/.+/g).map(s => {
            result = JSON.parse(s)
    
    Hugo NOUTS's avatar
    Hugo NOUTS committed
            if (result.statut_restitution !== null) {
    
    Hugo SUBTIL's avatar
    Hugo SUBTIL committed
              /**
    
               * Handle no data issue when retrieving grdf data.
    
    Hugo SUBTIL's avatar
    Hugo SUBTIL committed
               * 1000008 code stands for "Il n'y a pas de données correspondant à ce PCE sur la période demandée".
    
               * It is NOT an important issue deserving to throw an error
    
    Hugo SUBTIL's avatar
    Hugo SUBTIL committed
               * If there is no data, return null data in order to be filtered before saving
               */
              if (result.statut_restitution.code !== '1000008') {
    
                const genError = getDataGenericErrors(
                  result.statut_restitution.code
                )
                log(
                  'warn',
                  'donnees_consos_informatives responded with : ' +
                    result.statut_restitution.code +
                    ' -> ' +
                    result.statut_restitution.message +
                    ' Periode ' +
                    result.periode.date_debut +
                    '/' +
                    result.periode.date_fin
                )
                Sentry.captureMessage(
                  `Get data threw an error: ${result.statut_restitution.code} - ${result.statut_restitution.message}`
                )
                throw genError
    
    Hugo SUBTIL's avatar
    Hugo SUBTIL committed
              }
    
    Yoan VALLET's avatar
    Yoan VALLET committed
            return result.consommation
          })
        })
        .catch(error => {
          log('debug', 'Error from getData')
          throw error
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
        })
    
      const filteredRep = response.filter(function(el) {
    
    Hugo SUBTIL's avatar
    Hugo SUBTIL committed
        return el.energie != null || el.volume_brut != null
    
      })
      return filteredRep
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
    }
    
    /**
     * Parse data
     * Remove existing data from DB using hydrateAndFilter
     * Store filtered data
     * Return the list of filtered data
     */
    async function processData(data, doctype, filterKeys) {
    
      const formattedData = await formateData(data)
      log('debug', 'processData - data formatted')
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      // Remove data for existing days into the DB
    
      const filteredData = await hydrateAndFilter(formattedData, doctype, {
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
        keys: filterKeys
      })
    
    Yoan VALLET's avatar
    Yoan VALLET committed
      log('debug', 'processData - data filtered')
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      // Store new day data
    
      await storeData(filteredData, doctype, filterKeys)
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      return filteredData
    }
    
    /**
     * Save data in the right doctype db and prevent duplicated keys
     */
    async function storeData(data, doctype, filterKeys) {
      log('debug', doctype, 'Store into')
      const filteredDocuments = await hydrateAndFilter(data, doctype, {
        keys: filterKeys
      })
    
      return addData(filteredDocuments, doctype)
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
    }
    
    /**
     * Format data for DB storage
     * Remove bad data
     */
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
    async function formateData(data) {
    
      log('debug', 'Formatting data')
    
    Yoan VALLET's avatar
    Yoan VALLET committed
      return data.map(record => {
        let date = moment(record.date_debut_consommation, 'YYYY/MM/DD h:mm:ss')
        let load =
    
    Hugo SUBTIL's avatar
    Hugo SUBTIL committed
          record.energie && record.energie !== 0
    
    Yoan VALLET's avatar
    Yoan VALLET committed
            ? record.energie
    
            : record.volume_brut * record.coeff_calcul.coeff_conversion
    
    Yoan VALLET's avatar
    Yoan VALLET committed
        return {
          load: parseFloat(load),
          year: parseInt(date.format('YYYY')),
          month: parseInt(date.format('M')),
          day: parseInt(date.format('D')),
          hour: parseInt(date.format('H')),
          minute: parseInt(date.format('m'))
        }
      })
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
    }
    
    /**
    
     * Aggregate data from daily data to monthly and yearly data
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
     */
    
    async function aggregateMonthAndYearData(data) {
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      // Sum year and month values into object with year or year-month as keys
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      if (data && data.length !== 0) {
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
        let monthData = {}
        let yearData = {}
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
        data.forEach(element => {
          element.year + '-' + element.month in monthData
    
    Yoan VALLET's avatar
    Yoan VALLET committed
            ? (monthData[element.year + '-' + element.month] += element.load)
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
            : (monthData[element.year + '-' + element.month] = element.load)
          element.year in yearData
    
    Yoan VALLET's avatar
    Yoan VALLET committed
            ? (yearData[element.year] += element.load)
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
            : (yearData[element.year] = element.load)
        })
    
        // Aggregation for Month data
        const aggregatedMonthData = await buildAggregatedData(
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
          monthData,
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
          'com.grandlyon.grdf.month'
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
        )
    
        await storeData(aggregatedMonthData, 'com.grandlyon.grdf.month', [
    
          'year',
          'month'
        ])
    
        // Aggregation for Year data
        const aggregatedYearData = await buildAggregatedData(
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
          yearData,
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
          'com.grandlyon.grdf.year'
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
        )
    
        await storeData(aggregatedYearData, 'com.grandlyon.grdf.year', ['year'])
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      }
    }
    
    /**
     * Retrieve and remove old data for a specific doctype
    
     * Return an Array of aggregated data
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
     */
    
    async function buildAggregatedData(data, doctype) {
      let aggregatedData = []
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      for (let [key, value] of Object.entries(data)) {
        const data = await buildDataFromKey(doctype, key, value)
        const oldValue = await resetInProgressAggregatedData(data, doctype)
        data.load += oldValue
    
        aggregatedData.push(data)
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
      }
    
      return aggregatedData
    
    Rémi PAPIN's avatar
    Rémi PAPIN committed
    }
    
    /**
     * Format an entry for DB storage
     * using key and value
     * For year doctype: key = "YYYY"
     * For month doctype: key = "YYYY-MM"
     */
    async function buildDataFromKey(doctype, key, value) {
      let year, month, day, hour
      if (doctype === 'com.grandlyon.grdf.year') {
        year = key
        month = 1
        day = 0
        hour = 0
      } else if (doctype === 'com.grandlyon.grdf.month') {
        const split = key.split('-')
        year = split[0]
        month = split[1]
        day = 0
        hour = 0
      } else {
        const split = key.split('-')
        year = split[0]
        month = split[1]
        day = split[2]
        hour = split[3]
      }
      return {
        load: Math.round(value * 10000) / 10000,
        year: parseInt(year),
        month: parseInt(month),
        day: parseInt(day),
        hour: parseInt(hour),
        minute: 0
      }
    }
    
    /**
     * Function handling special case.
     * The temporary aggregated data need to be remove in order for the most recent one te be saved.
     * ex for com.grandlyon.grdf.year :
     * { load: 76.712, year: 2020, ... } need to be replace by
     * { load: 82.212, year: 2020, ... } after grdf data reprocess
     */
    async function resetInProgressAggregatedData(data, doctype) {
      // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
      log('debug', doctype, 'Remove aggregated data for')
      const result = await cozyClient.data.findAll(doctype)
      if (result && result.length > 0) {
        // Filter data to remove
        var filtered = []
        if (doctype === 'com.grandlyon.grdf.year') {
          // Yearly case
          filtered = result.filter(function(el) {
            return el.year == data.year
          })
        } else if (doctype === 'com.grandlyon.grdf.month') {
          // Monthly case
          filtered = result.filter(function(el) {
            return el.year == data.year && el.month == data.month
          })
        } else {
          // Hourly case
          filtered = result.filter(function(el) {
            return (
              el.year == data.year &&
              el.month == data.month &&
              el.day == data.day &&
              el.hour == data.hour
            )
          })
        }
        // Remove data
        let sum = 0.0
        for (const doc of filtered) {
          sum += doc.load
          log('debug', doc, 'Removing this entry for ' + doctype)
          await cozyClient.data.delete(doctype, doc)
        }
        return sum
      }
      return 0.0
    }