index.js

const {
  BaseKonnector,
  log,
  errors,
  addData,
  hydrateAndFilter,
  cozyClient
} = require('cozy-konnector-libs')

const rp = require('request-promise')
const moment = require('moment')
require('moment-timezone')

moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone

const manualExecution =
  process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false

const startDate = manualExecution
  ? moment().subtract(1, 'year').format('MM/DD/YYYY')
  : moment().subtract(3, 'year').format('MM/DD/YYYY')

const endDate = moment().format('MM/DD/YYYY')
// const timeRange = ['day', 'month', 'year']
const rangeDate = {
  day: {
    doctype: 'com.grandlyon.egl.day',
    keys: ['year', 'month', 'day']
  },
  month: {
    doctype: 'com.grandlyon.egl.month',
    keys: ['year', 'month']
  },
  year: {
    doctype: 'com.grandlyon.egl.year',
    keys: ['year']
  }
}

module.exports = new BaseKonnector(start)

// The start function is run by the BaseKonnector instance only when it got all the account
// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
// the account information come from ./konnector-dev-config.json file
async function start(fields, cozyParameters) {
  try {
    // resetting data for demo only
    // await resetData()
    // const baseUrl = fields.eglBaseURL
    // const apiAuthKey = fields.eglAPIAuthKey
    const baseUrl = cozyParameters.secret.eglBaseURL
    const apiAuthKey = cozyParameters.secret.eglAPIAuthKey
    log('info', 'Authenticating ...')
    const response = await authenticate(
      fields.login,
      fields.password,
      baseUrl,
      apiAuthKey
    )
    log('info', 'Successfully logged in')

    const eglData = await getData(response, baseUrl, apiAuthKey)
    if (eglData) {
      log('debug', 'Process egl daily data')
      const processedLoadData = await processData(
        eglData,
        rangeDate.day.doctype,
        rangeDate.day.keys
      )
      log('debug', 'Agregate egl load data for month and year')
      await agregateMonthAndYearData(processedLoadData)
    } else {
      log('debug', 'No data found')
    }
  } catch (error) {
    throw new Error(error.message)
  }
}

/**
 * Parse data
 * Remove existing data from DB using hydrateAndFilter
 * Store filtered data
 * Return the list of filtered data
 */
async function processData(data, doctype, filterKeys) {
  // const formatedData = await formateData(data)
  log('debug', 'processData - data formated')
  // Remove data for existing days into the DB
  const filteredData = await hydrateAndFilter(data, doctype, {
    keys: filterKeys
  })
  log('debug', 'processData - data filtered')
  // Store new day data
  await storeData(filteredData, doctype, filterKeys)
  return filteredData
}

/**
 * Agregate data from daily data to monthly and yearly data
 */
async function agregateMonthAndYearData(data) {
  // Sum year and month values into object with year or year-month as keys
  if (data && data.length !== 0) {
    let monthData = {}
    let yearData = {}

    data.forEach(element => {
      element.year + '-' + element.month in monthData
        ? (monthData[element.year + '-' + element.month] += element.load)
        : (monthData[element.year + '-' + element.month] = element.load)
      element.year in yearData
        ? (yearData[element.year] += element.load)
        : (yearData[element.year] = element.load)
    })
    // Agregation for Month data
    const agregatedMonthData = await buildAgregatedData(
      monthData,
      'com.grandlyon.egl.month'
    )
    await storeData(agregatedMonthData, 'com.grandlyon.egl.month', [
      'year',
      'month'
    ])
    // Agregation for Year data
    const agregatedYearData = await buildAgregatedData(
      yearData,
      'com.grandlyon.egl.year'
    )
    await storeData(agregatedYearData, 'com.grandlyon.egl.year', ['year'])
  }
}

/**
 * Retrieve and remove old data for a specific doctype
 * Return an Array of agregated data
 */
async function buildAgregatedData(data, doctype) {
  log('info', 'entering buildAgregatedData')
  let agregatedData = []
  for (let [key, value] of Object.entries(data)) {
    const data = await buildDataFromKey(doctype, key, value)
    const oldValue = await resetInProgressAggregatedData(data, doctype)
    log('info', 'Dataload + oldvalue is ' + data.load + ' + ' + oldValue)
    data.load += oldValue
    agregatedData.push(data)
  }
  return agregatedData
}

// async function processData(timeStep, response, baseUrl, apiAuthKey) {
//   const doctype = rangeDate[timeStep].doctype;
//   const loadProfile = await getData(response, baseUrl, apiAuthKey);
//   log("info", "Saving data to Cozy");
//   if (doctype === rangeDate.day.doctype) {
//     await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys);
//   } else if (doctype === rangeDate.month.doctype) {
//     await resetInProgressAggregatedData(rangeDate.month.doctype);
//     const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month);
//     log("info", "Saving monthly data");
//     await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys);
//   } else if (doctype === rangeDate.year.doctype) {
//     await resetInProgressAggregatedData(rangeDate.year.doctype);
//     const yearlyData = processYearAggregation(
//       loadProfile,
//       rangeDate.year.doctype
//     );
//     log("info", "Saving yearly data");
//     await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys);
//   } else {
//     throw new Error("Unkonw range type: " + doctype);
//   }
// }

async function authenticate(login, password, baseUrl, apiAuthKey) {
  const authRequest = {
    method: 'POST',
    uri: baseUrl + '/connect.aspx',
    headers: {
      AuthKey: apiAuthKey,
      'Content-Type': 'application/x-www-form-urlencoded'
    },
    form: {
      login: login,
      pass: password
    },
    json: true
  }
  const response = await rp(authRequest)
  if (response.codeRetour === 100) {
    return response
  } else {
    throw new Error(errors.LOGIN_FAILED)
  }
}

async function getData(response, baseUrl, apiAuthKey) {
  log('debug', 'Start date : ' + startDate)
  log('debug', 'End date : ' + endDate)
  const dataRequest = {
    method: 'POST',
    uri: baseUrl + '/getAllAgregatsByAbonnement.aspx',
    headers: {
      AuthKey: apiAuthKey,
      'Content-Type': 'application/x-www-form-urlencoded'
    },
    form: {
      token: response.resultatRetour.token,
      num_abt: response.resultatRetour.num_abt,
      date_debut: startDate,
      date_fin: endDate
    },
    json: true
  }
  try {
    const responseEgl = await rp(dataRequest)
    switch (responseEgl.codeRetour) {
      case 100:
        return format(responseEgl)
      case -2:
        throw new Error(errors.LOGIN_FAILED)
      case -1:
        throw new Error(errors.VENDOR_DOWN)
      default:
        throw new Error(errors.UNKNOWN_ERROR)
    }
  } catch (error) {
    throw new Error(errors.VENDOR_DOWN)
  }
}

function format(response) {
  log('info', 'origin response size is : ' + response.resultatRetour.length)
  const data = response.resultatRetour
    .slice(1)
    .filter(value => value.ValeurIndex)
  const dataLen = data.length
  log('info', 'filtered size is : ' + dataLen)
  const mapData = data.map((value, index) => {
    const time = moment(value.DateReleve, moment.ISO_8601)
    if (index !== 0 && index < dataLen) {
      return {
        load: value.ValeurIndex - data[index - 1].ValeurIndex,
        year: parseInt(time.format('YYYY')),
        month: parseInt(time.format('M')),
        day: parseInt(time.format('D')),
        hour: 0,
        minute: 0,
        type: value.TypeAgregat
      }
    } else {
      return {
        load: null,
        year: parseInt(time.format('YYYY')),
        month: parseInt(time.format('M')),
        day: parseInt(time.format('D')),
        hour: 0,
        minute: 0,
        type: value.TypeAgregat
      }
    }
  })
  const res = [...mapData].filter(v => v.load !== null)
  return res
}

// function processYearAggregation(data, doctype) {
//   log("info", "Start aggregation for : " + doctype);
//   const grouped = data.reduce(reduceYearFunction, {});
//   return Object.values(grouped);
// }

// function processMonthlyAggregation(data, range) {
//   log("info", "Start aggregation for : " + range.doctype);
//   // Filter by year
//   const tmpData = groupBy(data, "year");
//   const keys = Object.keys(tmpData);
//   var dataToStore = [];
//   // Monthly aggregation
//   for (const index in keys) {
//     // Get daily data of a year
//     var monthlyData = tmpData[keys[index]];
//     // Monthly aggregation
//     var aggregatedData = monthlyData.reduce(reduceMonthFunction, {});
//     // Store it
//     dataToStore = dataToStore.concat(Object.values(aggregatedData));
//   }
//   return dataToStore;
// }

// function groupBy(xs, key) {
//   return xs.reduce(function(rv, x) {
//     (rv[x[key]] = rv[x[key]] || []).push(x);
//     return rv;
//   }, {});
// }

// function reduceYearFunction(acc, x) {
//   var id = acc[x.year];
//   if (id) {
//     id.load += x.load;
//   } else {
//     acc[x.year] = x;
//   }
//   return acc;
// }

// function reduceMonthFunction(acc, x) {
//   var id = acc[x.month];
//   if (id) {
//     id.load += x.load;
//   } else {
//     acc[x.month] = x;
//   }
//   return acc;
// }

/**
 * Save data in the right doctype db and prevent duplicated keys
 */
async function storeData(data, doctype, filterKeys) {
  log('debug', 'Store into ' + doctype)
  log('debug', 'Store into keys : ' + filterKeys)
  // data.map(v => {
  //   log("info", "Saving data " + v.load + " for " + v.day + "/" + v.month + "/" + v.year);
  // });
  const filteredDocuments = await hydrateAndFilter(data, doctype, {
    keys: filterKeys
  })
  return await addData(filteredDocuments, doctype)
}

/**
 * Format an entry for DB storage
 * using key and value
 * For year doctype: key = "YYYY"
 * For month doctype: key = "YYYY-MM"
 */
async function buildDataFromKey(doctype, key, value) {
  let year, month, day, hour
  if (doctype === 'com.grandlyon.egl.year') {
    year = key
    month = 1
    day = 0
    hour = 0
  } else if (doctype === 'com.grandlyon.egl.month') {
    const split = key.split('-')
    year = split[0]
    month = split[1]
    day = 0
    hour = 0
  } else {
    const split = key.split('-')
    year = split[0]
    month = split[1]
    day = split[2]
    hour = split[3]
  }
  return {
    load: Math.round(value * 10000) / 10000,
    year: parseInt(year),
    month: parseInt(month),
    day: parseInt(day),
    hour: parseInt(hour),
    minute: 0
  }
}

/**
 * Function handling special case.
 * The temporary aggregated data need to be remove in order for the most recent one te be saved.
 * ex for com.grandlyon.egl.month :
 * { load: 76.712, month: 2020, ... } need to be replace by
 * { load: 82.212, month: 2020, ... } after egl data reprocess
 */
async function resetInProgressAggregatedData(data, doctype) {
  // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
  log('debug', 'Remove aggregated data for ' + doctype)
  const result = await cozyClient.data.findAll(doctype)
  if (result && result.length > 0) {
    // Filter data to remove
    var filtered = []
    if (doctype === 'com.grandlyon.egl.year') {
      // Yearly case
      filtered = result.filter(function(el) {
        return el.year == data.year
      })
    } else if (doctype === 'com.grandlyon.egl.month') {
      // Monthly case
      filtered = result.filter(function(el) {
        return el.year == data.year && el.month == data.month
      })
    } else {
      // Hourly case
      filtered = result.filter(function(el) {
        return (
          el.year == data.year &&
          el.month == data.month &&
          el.day == data.day &&
          el.hour == data.hour
        )
      })
    }
    // Remove data
    let sum = 0.0
    for (const doc of filtered) {
      sum += doc.load
      log('debug', 'Removing this entry for ' + doc.load)
      await cozyClient.data.delete(doctype, doc)
    }
    return sum
  }
  return 0.0
}