index.js

const {
  BaseKonnector,
  log,
  errors,
  addData,
  hydrateAndFilter,
  cozyClient
} = require("cozy-konnector-libs");

const rp = require("request-promise");
const moment = require("moment");
require("moment-timezone");

moment.locale("fr"); // set the language
moment.tz.setDefault("Europe/Paris"); // set the timezone

const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === "true" ? true : false

const startDate = manualExecution
  ? moment().startOf("year").subtract(1, "year").format("MM/DD/YYYY")
  : moment().startOf("year").subtract(3, "year").format("MM/DD/YYYY")
  
const endDate = moment().format("MM/DD/YYYY");
const timeRange = ["day", "month", "year"];
const rangeDate = {
  day: {
    doctype: "com.grandlyon.egl.day",
    keys: ["year", "month", "day"]
  },
  month: {
    doctype: "com.grandlyon.egl.month",
    keys: ["year", "month"]
  },
  year: {
    doctype: "com.grandlyon.egl.year",
    keys: ["year"]
  }
};

module.exports = new BaseKonnector(start);

// The start function is run by the BaseKonnector instance only when it got all the account
// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
// the account information come from ./konnector-dev-config.json file
async function start(fields, cozyParameters) {
  try {
    // resetting data for demo only
    // await resetData()
    const baseUrl = cozyParameters.secret.eglBaseURL;
    const apiAuthKey = cozyParameters.secret.eglAPIAuthKey;
    log("info", "Authenticating ...");
    const response = await authenticate(
      fields.login,
      fields.password,
      baseUrl,
      apiAuthKey
    );
    log("info", "Successfully logged in");
    Promise.all(
      timeRange.map(timeStep =>
        processData(timeStep, response, baseUrl, apiAuthKey)
      )
    );
  } catch (error) {
    throw new Error(error.message);
  }
}
async function processData(timeStep, response, baseUrl, apiAuthKey) {
  const doctype = rangeDate[timeStep].doctype;
  const loadProfile = await getData(response, baseUrl, apiAuthKey);
  log("info", "Saving data to Cozy");
  if (doctype === rangeDate.day.doctype) {
    await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys);
  } else if (doctype === rangeDate.month.doctype) {
    await resetInProgressAggregatedData(rangeDate.month.doctype);
    const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month);
    log("info", "Saving monthly data");
    await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys);
  } else if (doctype === rangeDate.year.doctype) {
    await resetInProgressAggregatedData(rangeDate.year.doctype);
    const yearlyData = processYearAggregation(
      loadProfile,
      rangeDate.year.doctype
    );
    log("info", "Saving yearly data");
    await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys);
  } else {
    throw new Error("Unkonw range type: " + doctype);
  }
}

async function authenticate(login, password, baseUrl, apiAuthKey) {
  const authRequest = {
    method: "POST",
    uri: baseUrl + "/connect.aspx",
    headers: {
      AuthKey: apiAuthKey,
      "Content-Type": "application/x-www-form-urlencoded"
    },
    form: {
      login: login,
      pass: password
    },
    json: true
  };
  const response = await rp(authRequest);
  if (response.codeRetour === 100) {
    return response;
  } else {
    throw new Error(errors.LOGIN_FAILED);
  }
}

async function getData(response, baseUrl, apiAuthKey) {
  log("debug", "Start date : " + startDate);
  log("debug", "End date : " + endDate);
  const dataRequest = {
    method: "POST",
    uri: baseUrl + "/getAllAgregatsByAbonnement.aspx",
    headers: {
      AuthKey: apiAuthKey,
      "Content-Type": "application/x-www-form-urlencoded"
    },
    form: {
      token: response.resultatRetour.token,
      num_abt: response.resultatRetour.num_abt,
      date_debut: startDate,
      date_fin: endDate
    },
    json: true
  };
  try {
    const responseEgl = await rp(dataRequest);
    switch (responseEgl.codeRetour) {
      case 100:
        return format(responseEgl);
      case -2:
        throw new Error(errors.LOGIN_FAILED);
      case -1:
        throw new Error(errors.VENDOR_DOWN);
      default:
        throw new Error(errors.UNKNOWN_ERROR);
    }
  } catch (error) {
    throw new Error(errors.VENDOR_DOWN);
  }
}

function format(response) {
  log("info", "origin response size is : " + response.resultatRetour.length)
  const data = response.resultatRetour
    .slice(1)
    .filter(value => value.ValeurIndex);
  const dataLen = data.length;
  log("info", "filtered size is : " + dataLen);
  const mapData = data.map((value, index) => {
    const time = moment(value.DateReleve, moment.ISO_8601);
    if (index + 1 < dataLen) {
      return {
        load: data[index + 1].ValeurIndex - value.ValeurIndex,
        year: parseInt(time.format("YYYY")),
        month: parseInt(time.format("M")),
        day: parseInt(time.format("D")),
        hour: 0,
        minute: 0,
        type: value.TypeAgregat
      };
    } else {
      log("info", "end of data - date is : " + value.DateReleve);
      return {
        load: null,
        year: parseInt(time.format("YYYY")),
        month: parseInt(time.format("M")),
        day: parseInt(time.format("D")),
        hour: 0,
        minute: 0,
        type: value.TypeAgregat
      };
    }
  });
  return mapData;
}

function processYearAggregation(data, doctype) {
  log("info", "Start aggregation for : " + doctype);
  const grouped = data.reduce(reduceYearFunction, {});
  return Object.values(grouped);
}

function processMonthlyAggregation(data, range) {
  log("info", "Start aggregation for : " + range.doctype);
  // Filter by year
  const tmpData = groupBy(data, "year");
  const keys = Object.keys(tmpData);
  var dataToStore = [];
  // Monthly aggregation
  for (const index in keys) {
    // Get daily data of a year
    var monthlyData = tmpData[keys[index]];
    // Monthly aggregation
    var aggregatedData = monthlyData.reduce(reduceMonthFunction, {});
    // Store it
    dataToStore = dataToStore.concat(Object.values(aggregatedData));
  }
  return dataToStore;
}

function groupBy(xs, key) {
  return xs.reduce(function(rv, x) {
    (rv[x[key]] = rv[x[key]] || []).push(x);
    return rv;
  }, {});
}

function reduceYearFunction(acc, x) {
  var id = acc[x.year];
  if (id) {
    id.load += x.load;
  } else {
    acc[x.year] = x;
  }
  return acc;
}

function reduceMonthFunction(acc, x) {
  var id = acc[x.month];
  if (id) {
    id.load += x.load;
  } else {
    acc[x.month] = x;
  }
  return acc;
}

/**
 * Save data in the right doctype db and prevent duplicated keys
 */
async function storeData(data, doctype, filterKeys) {
  log("debug", "Store into " + doctype);
  log("debug", "Store into keys : " + filterKeys);
  // data.map(v => {
  //   log("info", "Saving data " + v.load + " for " + v.day + "/" + v.month + "/" + v.year);
  // });
  const filteredDocuments = await hydrateAndFilter(data, doctype, {
    keys: filterKeys
  })
  return await addData(filteredDocuments, doctype)
}

/**
 * Function handling special case.
 * The temporary aggregated data need to be remove in order for the most recent one te be saved.
 * ex for com.grandlyon.egl.month :
 * { load: 76.712, month: 2020, ... } need to be replace by
 * { load: 82.212, month: 2020, ... } after grdf data reprocess
 */
async function resetInProgressAggregatedData(doctype) {
  // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
  log("debug", doctype, "Remove aggregated data for");
  const result = await cozyClient.data.findAll(doctype);
  if (result.error || result.length <= 0) {
    // eslint-disable-next-line no-console
    console.warn("Error while fetching loads, doctype not found ");
  } else {
    const currentDate = moment();
    // Filter data to remove
    var filtered = [];
    if (doctype === rangeDate.year.doctype) {
      // Yearly case
      filtered = result.filter(function(el) {
        return el.year == currentDate.year();
      });
    } else {
      // Monthly case
      filtered = result.filter(function(el) {
        return (
          el.year == currentDate.year() &&
          el.month == parseInt(moment().format("M"))
        );
      });
    }
    // Remove data
    for (const doc of filtered) {
      log("debug", doc, "Removing this entry for " + doctype);
      await cozyClient.data.delete(doctype, doc);
    }
  }
}