index.js

const {
  BaseKonnector,
  log,
  errors,
  addData,
  hydrateAndFilter,
  cozyClient
} = require("cozy-konnector-libs");

// const fetch = require('node-fetch')
const rp = require("request-promise");
const moment = require("moment");
require("moment-timezone");

moment.locale("fr"); // set the language
moment.tz.setDefault("Europe/Paris"); // set the timezone

const manualExecution =
  process.env.COZY_JOB_MANUAL_EXECUTION === "true" ? true : false;

const startDate = manualExecution
  ? moment()
      .subtract(1, "year")
      .format("MM/DD/YYYY")
  : moment()
      .subtract(3, "year")
      .format("MM/DD/YYYY");

const endDate = moment().format("MM/DD/YYYY");
// const timeRange = ['day', 'month', 'year']
const rangeDate = {
  day: {
    doctype: "com.grandlyon.egl.day",
    keys: ["year", "month", "day"]
  },
  month: {
    doctype: "com.grandlyon.egl.month",
    keys: ["year", "month"]
  },
  year: {
    doctype: "com.grandlyon.egl.year",
    keys: ["year"]
  }
};

module.exports = new BaseKonnector(start);

// The start function is run by the BaseKonnector instance only when it got all the account
// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
// the account information come from ./konnector-dev-config.json file
async function start(fields, cozyParameters) {
  try {
    // Local debug data
    // const baseUrl = fields.eglBaseURL
    // const apiAuthKey = fields.eglAPIAuthKey
    const baseUrl = cozyParameters.secret.eglBaseURL;
    const apiAuthKey = cozyParameters.secret.eglAPIAuthKey;
    log("info", "Authenticating ...");
    const response = await authenticate(
      fields.login,
      fields.password,
      baseUrl,
      apiAuthKey
    );
    log("info", "Successfully logged in");

    const eglData = await getData(response, baseUrl, apiAuthKey);
    if (eglData) {
      log("debug", "Process egl daily data");
      const processedLoadData = await processData(
        eglData,
        rangeDate.day.doctype,
        rangeDate.day.keys
      );
      log("debug", "Agregate egl load data for month and year");
      await agregateMonthAndYearData(processedLoadData);
    } else {
      log("debug", "No data found");
    }
  } catch (error) {
    throw new Error(error.message);
  }
}

/**
 * Parse data
 * Remove existing data from DB using hydrateAndFilter
 * Store filtered data
 * Return the list of filtered data
 */
async function processData(data, doctype, filterKeys) {
  // const formatedData = await formateData(data)
  log("debug", "processData - data formated");
  // Remove data for existing days into the DB
  const filteredData = await hydrateAndFilter(data, doctype, {
    keys: filterKeys
  });
  log("debug", "processData - data filtered");
  // Store new day data
  await storeData(filteredData, doctype, filterKeys);
  return filteredData;
}

/**
 * Agregate data from daily data to monthly and yearly data
 */
async function agregateMonthAndYearData(data) {
  // Sum year and month values into object with year or year-month as keys
  if (data && data.length !== 0) {
    let monthData = {};
    let yearData = {};

    data.forEach(element => {
      element.year + "-" + element.month in monthData
        ? (monthData[element.year + "-" + element.month] += element.load)
        : (monthData[element.year + "-" + element.month] = element.load);
      element.year in yearData
        ? (yearData[element.year] += element.load)
        : (yearData[element.year] = element.load);
    });
    // Agregation for Month data
    const agregatedMonthData = await buildAgregatedData(
      monthData,
      "com.grandlyon.egl.month"
    );
    await storeData(agregatedMonthData, "com.grandlyon.egl.month", [
      "year",
      "month"
    ]);
    // Agregation for Year data
    const agregatedYearData = await buildAgregatedData(
      yearData,
      "com.grandlyon.egl.year"
    );
    await storeData(agregatedYearData, "com.grandlyon.egl.year", ["year"]);
  }
}

/**
 * Retrieve and remove old data for a specific doctype
 * Return an Array of agregated data
 */
async function buildAgregatedData(data, doctype) {
  log("info", "entering buildAgregatedData");
  let agregatedData = [];
  for (let [key, value] of Object.entries(data)) {
    const data = await buildDataFromKey(doctype, key, value);
    const oldValue = await resetInProgressAggregatedData(data, doctype);
    log("info", "Dataload + oldvalue is " + data.load + " + " + oldValue);
    data.load += oldValue;
    agregatedData.push(data);
  }
  return agregatedData;
}

async function authenticate(login, password, baseUrl, apiAuthKey) {
  const authRequest = {
    method: "POST",
    uri: baseUrl + "/connect.aspx",
    headers: {
      AuthKey: apiAuthKey,
      "Content-Type": "application/x-www-form-urlencoded"
    },
    formData: {
      login: login,
      pass: password
    },
    json: true
  };
  const response = await rp(authRequest);
  if (response.codeRetour === 100) {
    return response;
  } else {
    throw new Error(errors.LOGIN_FAILED);
  }
}

async function getData(response, baseUrl, apiAuthKey) {
  log("debug", "Start date : " + startDate);
  log("debug", "End date : " + endDate);
  const dataRequest = {
    method: "POST",
    uri: baseUrl + "/getAllAgregatsByAbonnement.aspx",
    headers: {
      AuthKey: apiAuthKey,
      "Content-Type": "application/x-www-form-urlencoded"
    },
    form: {
      token: response.resultatRetour.token,
      num_abt: response.resultatRetour.num_abt,
      date_debut: startDate,
      date_fin: endDate
    },
    json: true
  };
  try {
    // Sort data by date

    const responseEgl = await rp(dataRequest).then(eglRawData => {
      eglRawData.resultatRetour.sort(function(a, b) {
        return new Date(a.DateReleve) - new Date(b.DateReleve);
      });
      return eglRawData;
    });
    switch (responseEgl.codeRetour) {
      case 100:
        return format(responseEgl);
      case -2:
        throw errors.LOGIN_FAILED;
      case -1:
        throw errors.VENDOR_DOWN;
      default:
        throw errors.UNKNOWN_ERROR;
    }
  } catch (error) {
    log("debug", "Error from getAllAgregatsByAbonnement");
    throw new Error(errors.VENDOR_DOWN);
  }
}

function format(response) {
  log("info", "origin response size is : " + response.resultatRetour.length);
  // Store first value as reference for index processing
  let refValue = response.resultatRetour[0];
  // Create copy of data without first value
  const data = response.resultatRetour
    .slice(1)
    .filter(value => value.ValeurIndex);
  log("info", "filtered size is : " + data.length);
  return data.map(value => {
    const time = moment(value.DateReleve, moment.ISO_8601);
    const procesedLoad = value.ValeurIndex - refValue.ValeurIndex;
    if (procesedLoad < 0) {
      log(
        "error",
        `processing load for day ${parseInt(time.format("D"))}/${parseInt(
          time.format("M")
        )}/${parseInt(time.format("YYYY"))}, value is : ${procesedLoad}`
      );
      throw errors.VENDOR_DOWN;
    }
    // Change index ref value
    refValue = value;
    return {
      load: procesedLoad,
      year: parseInt(time.format("YYYY")),
      month: parseInt(time.format("M")),
      day: parseInt(time.format("D")),
      hour: 0,
      minute: 0,
      type: value.TypeAgregat
    };
  });
}

/**
 * Save data in the right doctype db and prevent duplicated keys
 */
async function storeData(data, doctype, filterKeys) {
  log("debug", "Store into " + doctype);
  log("debug", "Store into keys : " + filterKeys);
  // data.map(v => {
  //   log("info", "Saving data " + v.load + " for " + v.day + "/" + v.month + "/" + v.year);
  // });
  const filteredDocuments = await hydrateAndFilter(data, doctype, {
    keys: filterKeys
  });
  return await addData(filteredDocuments, doctype);
}

/**
 * Format an entry for DB storage
 * using key and value
 * For year doctype: key = "YYYY"
 * For month doctype: key = "YYYY-MM"
 */
async function buildDataFromKey(doctype, key, value) {
  let year, month, day, hour;
  if (doctype === "com.grandlyon.egl.year") {
    year = key;
    month = 1;
    day = 0;
    hour = 0;
  } else if (doctype === "com.grandlyon.egl.month") {
    const split = key.split("-");
    year = split[0];
    month = split[1];
    day = 0;
    hour = 0;
  } else {
    const split = key.split("-");
    year = split[0];
    month = split[1];
    day = split[2];
    hour = split[3];
  }
  return {
    load: Math.round(value * 10000) / 10000,
    year: parseInt(year),
    month: parseInt(month),
    day: parseInt(day),
    hour: parseInt(hour),
    minute: 0
  };
}

/**
 * Function handling special case.
 * The temporary aggregated data need to be remove in order for the most recent one te be saved.
 * ex for com.grandlyon.egl.month :
 * { load: 76.712, month: 2020, ... } need to be replace by
 * { load: 82.212, month: 2020, ... } after egl data reprocess
 */
async function resetInProgressAggregatedData(data, doctype) {
  // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
  log("debug", "Remove aggregated data for " + doctype);
  const result = await cozyClient.data.findAll(doctype);
  if (result && result.length > 0) {
    // Filter data to remove
    var filtered = [];
    if (doctype === "com.grandlyon.egl.year") {
      // Yearly case
      filtered = result.filter(function(el) {
        return el.year == data.year;
      });
    } else if (doctype === "com.grandlyon.egl.month") {
      // Monthly case
      filtered = result.filter(function(el) {
        return el.year == data.year && el.month == data.month;
      });
    } else {
      // Hourly case
      filtered = result.filter(function(el) {
        return (
          el.year == data.year &&
          el.month == data.month &&
          el.day == data.day &&
          el.hour == data.hour
        );
      });
    }
    // Remove data
    let sum = 0.0;
    for (const doc of filtered) {
      sum += doc.load;
      log("debug", "Removing this entry for " + doc.load);
      await cozyClient.data.delete(doctype, doc);
    }
    return sum;
  }
  return 0.0;
}