index.js

const {
  cozyClient,
  BaseKonnector,
  addData,
  hydrateAndFilter,
  errors,
  log
} = require('cozy-konnector-libs')

// const https = require('https') //optional for ssl issue
const getAccountId = require('./helpers/getAccountId')
const jwt = require('jsonwebtoken')
const moment = require('moment')
require('moment-timezone')

moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone

const startDate = moment()
  .startOf('day')
  .subtract(4, 'day')
  .format('YYYY-MM-DD')
const endDate = moment().format('YYYY-MM-DD')
// const baseUrl = 'https://api.grdf.fr/adict/v1'
//console.log(startDate)
//console.log(endDate)
/*const timeRange = ['day', 'month', 'year']
const rangeDate = {
  day: {
    doctype: 'com.grandlyon.grdf.day',
    keys: ['year', 'month', 'day']
  },
  month: {
    doctype: 'com.grandlyon.grdf.month',
    keys: ['year', 'month']
  },
  year: {
    doctype: 'com.grandlyon.grdf.year',
    keys: ['year']
  }
}

const client_id = 'metropole_de_lyon_grdf'
const redirectUrl = 'https://grdf.pchugo.wf.alpha.grandlyon.com'*/

module.exports = new BaseKonnector(start)

// The start function is run by the BaseKonnector instance only when it got all the account
// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
// the account information come from ./konnector-dev-config.json file
// cozyParameters are static parameters, independents from the account. Most often, it can be a
// secret api key.
async function start(fields, cozyParameters, doRetry = true) {
  // var grdfPCE = ''
  log('debug', 'Starting grdf adict konnector')
  
  // if (cozyParameters) log('info', 'CozyParameters found')
  // const grdfToken = await authenticate(fields.login, fields.password)

  const accountId = getAccountId()
  try {
    const { access_token } = fields
    log('debug', 'KONNECTOR FIELDS : ' + access_token)
    // log(
    //   'debug',
    //   'KONNECTOR ACCOUNT : ' + this._account.oauth_callback_results.id_token    )
    if (
      this._account &&
      this._account.oauth_callback_results &&
      this._account.oauth_callback_results.id_token
    ) {
      log('debug', 'time to decode jwt')
      const id_pce = await getPCEid(
        this._account.oauth_callback_results.id_token
      ) //19108248691849 PCEid FOR TESTING PURPOSE
      // Store pce and restart
      // const data = [{ id_pce: id_pce }];
      log('debug', 'Fetching grdf adict daily data')
      log('debug', 'THE ID_PCE SENDING TO GETDATA : ' + id_pce)

      const grdfData = await getData(access_token, id_pce)

      if (grdfData.consommation) {
        log('debug', 'Process grdf daily data')
        const processedLoadData = await processData(
          grdfData,
          'com.grandlyon.grdf.day',
          ['year', 'month', 'day']
        )
        log('debug', 'Agregate grdf load data for month and year')
        await agregateMonthAndYearData(processedLoadData)
      } else {
        log('debug', 'No consent or data for load curve')
      }
      // await addData(data, "io.cozy.accounts", accountId);
      // return start(fields, cozyParameters, true);
    } else {
      // log('debug', 'no id_token found in oauth_callback_results')
      // log(
      //   'debug',
      //   'callback_result contains: ',
      //   this._account.oauth_callback_results
      // )
      throw errors.USER_ACTION_NEEDED_OAUTH_OUTDATED
    }
  } catch (err) {
    log('debug', 'ERROR CATCHED IN MAIN')
    log('debug', err)
    log('debug', typeof(err))
    log('debug', typeof(err.message) + err.message)
    if (
      err
      // err.statusCode === 403 ||
      // err.code === 403 ||
      // err.statusCode === 401 ||
      // err.code === 401 ||
      // err.message === "403" || err == 403
    ) {
      log('debug', 'DO RETRY ' + doRetry)
      if (doRetry) {
        log('debug', 'asking refresh from the stack')
        let body
        try {
          body = await cozyClient.fetchJSON(
            'POST',
            `/accounts/grdf-konnector/${accountId}/refresh`
          )
        } catch (err) {
          log('debug', `Error during refresh ${err.message}`)
          throw errors.USER_ACTION_NEEDED_OAUTH_OUTDATED
        }

        log('debug', 'refresh response')
        log('debug', JSON.stringify(body))
        fields.access_token = body.attributes.oauth.access_token
        return start(fields, cozyParameters, false)
      }
      log('error', `Error during authentication: ${err.message}`)
      throw errors.VENDOR_DOWN
    } else {
      log('error', 'caught an unexpected error')
      log('error', err.message)
      throw errors.VENDOR_DOWN
    }
  }
}

async function getPCEid(tokenToDecrypt) {
  // verify a token symmetric - synchronous
  log('debug', 'GETPCEID : ' + tokenToDecrypt)
  var decoded = jwt.decode(tokenToDecrypt)
  const decodedConsentements = JSON.parse(decoded.consentements)
  return decodedConsentements[0].pce
}
//Retrieve data from grdf API
async function getData(token, idPCE) {
  log('debug', 'ENTERING GETDATA')
  var myHeaders = new Headers()
  myHeaders.append('Content-Type', 'application/x-ndjson')
  myHeaders.append('Authorization', 'Bearer ' + token)

  var requestOptions = {
    method: 'GET',
    headers: myHeaders,
    redirect: 'follow'
  }
  var url =
    'https://api.grdf.fr/adict/v1/pce/' +
    idPCE +
    '/donnees_consos_informatives?date_debut=' +
    startDate +
    '&date_fin=' +
    endDate
  const rep = await fetch(url, requestOptions)
    .then(response => {
      log('debug', 'RESPONSE ERROR')
      // log('debug', response.error())
      if(response.status === 403){
        throw new Error("403")
      }
      response.text()
    })
    .then(result => {
      result = JSON.parse(result)
      return result
    })
    .catch(error => {
      log('debug', 'ERROR CATCHED IN GETDATA' + error)
      throw error
    })
  return rep
}

/**
 * Parse data
 * Remove existing data from DB using hydrateAndFilter
 * Store filtered data
 * Return the list of filtered data
 */
async function processData(data, doctype, filterKeys) {
  const intervalData = data.consommation
  const formatedData = await formateData(intervalData)
  const dataArray = [formatedData]
  // Remove data for existing days into the DB
  const filteredData = await hydrateAndFilter(dataArray, doctype, {
    keys: filterKeys
  })
  // Store new day data
  await storeData(filteredData, doctype, filterKeys)
  return filteredData
}

/**
 * Save data in the right doctype db and prevent duplicated keys
 */
async function storeData(data, doctype, filterKeys) {
  log('debug', doctype, 'Store into')
  const filteredDocuments = await hydrateAndFilter(data, doctype, {
    keys: filterKeys
  })
  return await addData(filteredDocuments, doctype)
}

/**
 * Format data for DB storage
 * Remove bad data
 */
async function formateData(data) {
  log('debug', 'Formating data')

  let date = moment(data.date_debut_consommation, 'YYYY/MM/DD h:mm:ss')
  let load =
    data.energie !== 0
      ? data.energie
      : data.volume_brut * data.coeff_calcul.coeff_conversion

  //TRAITEMENT DES MINUTES
  // const load =
  //   doctype === 'com.grandlyon.grdf.minute'
  //     ? record.value / 2
  //     : record.value
  // if (doctype === 'com.grandlyon.grdf.minute') {
  //   date = date.subtract(30, 'minute')
  // }
  return {
    load: parseFloat(load),
    year: parseInt(date.format('YYYY')),
    month: parseInt(date.format('M')),
    day: parseInt(date.format('D')),
    hour: parseInt(date.format('H')),
    minute: parseInt(date.format('m'))
  }
}

/**
 * Agregate data from load data (every 30 min) to Hourly data
 */
/*async function agregateHourlyData(data) {
  // Sum year and month values into object with year or year-month as keys
  if (data && data.length > 0) {
    let hourData = {}
    data.forEach(element => {
      let key =
        element.year +
        '-' +
        element.month +
        '-' +
        element.day +
        '-' +
        element.hour
      key in hourData
        ? (hourData[key] += element.load)
        : (hourData[key] = element.load)
    })
    // Agregation for Month data
    const agregatedMonthData = await buildAgregatedData(
      hourData,
      'com.grandlyon.grdf.hour'
    )
    await storeData(agregatedMonthData, 'com.grandlyon.grdf.hour', [
      'year',
      'month',
      'day',
      'hour'
    ])
  }
}*/
/**
 * Agregate data from daily data to monthly and yearly data
 */
async function agregateMonthAndYearData(data) {
  //console.log('agregateMonthAndYearData')
  //console.log(data)
  // Sum year and month values into object with year or year-month as keys
  if (data && data.length !== 0) {
    let monthData = {}
    let yearData = {}

    data.forEach(element => {
      element.year + '-' + element.month in monthData
        ? (monthData[element.year + '-' + element.month] += element.load)
        : (monthData[element.year + '-' + element.month] = element.load)
      element.year in yearData
        ? (yearData[element.year] += element.load)
        : (yearData[element.year] = element.load)
    })

    //console.log(monthData)
    //console.log(yearData)
    // Agregation for Month data
    const agregatedMonthData = await buildAgregatedData(
      monthData,
      'com.grandlyon.grdf.month'
    )
    //console.log('agregatedMonthData')
    //console.log(agregatedMonthData)
    await storeData(agregatedMonthData, 'com.grandlyon.grdf.month', [
      'year',
      'month'
    ])
    // Agregation for Year data
    const agregatedYearData = await buildAgregatedData(
      yearData,
      'com.grandlyon.grdf.year'
    )
    await storeData(agregatedYearData, 'com.grandlyon.grdf.year', ['year'])
  }
}

/**
 * Retrieve and remove old data for a specific doctype
 * Return an Array of agregated data
 */
async function buildAgregatedData(data, doctype) {
  let agregatedData = []
  for (let [key, value] of Object.entries(data)) {
    const data = await buildDataFromKey(doctype, key, value)
    const oldValue = await resetInProgressAggregatedData(data, doctype)
    data.load += oldValue
    agregatedData.push(data)
  }
  return agregatedData
}

/**
 * Format an entry for DB storage
 * using key and value
 * For year doctype: key = "YYYY"
 * For month doctype: key = "YYYY-MM"
 */
async function buildDataFromKey(doctype, key, value) {
  let year, month, day, hour
  if (doctype === 'com.grandlyon.grdf.year') {
    year = key
    month = 1
    day = 0
    hour = 0
  } else if (doctype === 'com.grandlyon.grdf.month') {
    const split = key.split('-')
    year = split[0]
    month = split[1]
    day = 0
    hour = 0
  } else {
    const split = key.split('-')
    year = split[0]
    month = split[1]
    day = split[2]
    hour = split[3]
  }
  return {
    load: Math.round(value * 10000) / 10000,
    year: parseInt(year),
    month: parseInt(month),
    day: parseInt(day),
    hour: parseInt(hour),
    minute: 0
  }
}

/**
 * Function handling special case.
 * The temporary aggregated data need to be remove in order for the most recent one te be saved.
 * ex for com.grandlyon.grdf.year :
 * { load: 76.712, year: 2020, ... } need to be replace by
 * { load: 82.212, year: 2020, ... } after grdf data reprocess
 */
async function resetInProgressAggregatedData(data, doctype) {
  // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
  log('debug', doctype, 'Remove aggregated data for')
  const result = await cozyClient.data.findAll(doctype)
  if (result && result.length > 0) {
    // Filter data to remove
    var filtered = []
    if (doctype === 'com.grandlyon.grdf.year') {
      // Yearly case
      filtered = result.filter(function(el) {
        return el.year == data.year
      })
    } else if (doctype === 'com.grandlyon.grdf.month') {
      // Monthly case
      filtered = result.filter(function(el) {
        return el.year == data.year && el.month == data.month
      })
    } else {
      // Hourly case
      filtered = result.filter(function(el) {
        return (
          el.year == data.year &&
          el.month == data.month &&
          el.day == data.day &&
          el.hour == data.hour
        )
      })
    }
    // Remove data
    let sum = 0.0
    for (const doc of filtered) {
      sum += doc.load
      log('debug', doc, 'Removing this entry for ' + doctype)
      await cozyClient.data.delete(doctype, doc)
    }
    return sum
  }
  return 0.0
}