Skip to content

GitLab

Commit cc0df612 authored by Hugo NOUTS's avatar Hugo NOUTS
Browse files

Merge branch 'dev' into 'master' 

revert to request-promise

See merge request !14
parents 2b78ba6d 316e67a6
Hide whitespace changes
Inline Side-by-side
manifest.konnector
View file @ cc0df612
{
"version": "1.0.4",
"version": "1.0.3",
"name": "EGL",
"type": "konnector",
"language": "node",
......
package.json
View file @ cc0df612
{
"name": "egl",
"version": "1.0.4",
"version": "1.0.3",
"description": "",
"repository": {
"type": "git",
......@@ -41,8 +41,7 @@
"dependencies": {
"cozy-konnector-libs": "4.42.3",
"moment": "^2.24.0",
"moment-timezone": "^0.5.26",
"node-fetch": "2"
"moment-timezone": "^0.5.26"
},
"devDependencies": {
"@types/moment-timezone": "^0.5.30",
......
src/index.js
View file @ cc0df612
......@@ -5,44 +5,45 @@ const {
addData,
hydrateAndFilter,
cozyClient
} = require('cozy-konnector-libs')
} = require("cozy-konnector-libs");
const fetch = require('node-fetch')
const moment = require('moment')
require('moment-timezone')
// const fetch = require('node-fetch')
const rp = require("request-promise");
const moment = require("moment");
require("moment-timezone");
moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone
moment.locale("fr"); // set the language
moment.tz.setDefault("Europe/Paris"); // set the timezone
const manualExecution =
process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false
process.env.COZY_JOB_MANUAL_EXECUTION === "true" ? true : false;
const startDate = manualExecution
? moment()
.subtract(1, 'year')
.format('MM/DD/YYYY')
.subtract(1, "year")
.format("MM/DD/YYYY")
: moment()
.subtract(3, 'year')
.format('MM/DD/YYYY')
.subtract(3, "year")
.format("MM/DD/YYYY");
const endDate = moment().format('MM/DD/YYYY')
const endDate = moment().format("MM/DD/YYYY");
// const timeRange = ['day', 'month', 'year']
const rangeDate = {
day: {
doctype: 'com.grandlyon.egl.day',
keys: ['year', 'month', 'day']
doctype: "com.grandlyon.egl.day",
keys: ["year", "month", "day"]
},
month: {
doctype: 'com.grandlyon.egl.month',
keys: ['year', 'month']
doctype: "com.grandlyon.egl.month",
keys: ["year", "month"]
},
year: {
doctype: 'com.grandlyon.egl.year',
keys: ['year']
doctype: "com.grandlyon.egl.year",
keys: ["year"]
}
}
};
module.exports = new BaseKonnector(start)
module.exports = new BaseKonnector(start);
// The start function is run by the BaseKonnector instance only when it got all the account
// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
......@@ -52,32 +53,32 @@ async function start(fields, cozyParameters) {
// Local debug data
// const baseUrl = fields.eglBaseURL
// const apiAuthKey = fields.eglAPIAuthKey
const baseUrl = cozyParameters.secret.eglBaseURL
const apiAuthKey = cozyParameters.secret.eglAPIAuthKey
log('info', 'Authenticating ...')
const baseUrl = cozyParameters.secret.eglBaseURL;
const apiAuthKey = cozyParameters.secret.eglAPIAuthKey;
log("info", "Authenticating ...");
const response = await authenticate(
fields.login,
fields.password,
baseUrl,
apiAuthKey
)
log('info', 'Successfully logged in')
);
log("info", "Successfully logged in");
const eglData = await getData(response, baseUrl, apiAuthKey)
const eglData = await getData(response, baseUrl, apiAuthKey);
if (eglData) {
log('debug', 'Process egl daily data')
log("debug", "Process egl daily data");
const processedLoadData = await processData(
eglData,
rangeDate.day.doctype,
rangeDate.day.keys
)
log('debug', 'Agregate egl load data for month and year')
await agregateMonthAndYearData(processedLoadData)
);
log("debug", "Agregate egl load data for month and year");
await agregateMonthAndYearData(processedLoadData);
} else {
log('debug', 'No data found')
log("debug", "No data found");
}
} catch (error) {
throw new Error(error.message)
throw new Error(error.message);
}
}
......@@ -89,15 +90,15 @@ async function start(fields, cozyParameters) {
*/
async function processData(data, doctype, filterKeys) {
// const formatedData = await formateData(data)
log('debug', 'processData - data formated')
log("debug", "processData - data formated");
// Remove data for existing days into the DB
const filteredData = await hydrateAndFilter(data, doctype, {
keys: filterKeys
})
log('debug', 'processData - data filtered')
});
log("debug", "processData - data filtered");
// Store new day data
await storeData(filteredData, doctype, filterKeys)
return filteredData
await storeData(filteredData, doctype, filterKeys);
return filteredData;
}
/**
......@@ -106,32 +107,32 @@ async function processData(data, doctype, filterKeys) {
async function agregateMonthAndYearData(data) {
// Sum year and month values into object with year or year-month as keys
if (data && data.length !== 0) {
let monthData = {}
let yearData = {}
let monthData = {};
let yearData = {};
data.forEach(element => {
element.year + '-' + element.month in monthData
? (monthData[element.year + '-' + element.month] += element.load)
: (monthData[element.year + '-' + element.month] = element.load)
element.year + "-" + element.month in monthData
? (monthData[element.year + "-" + element.month] += element.load)
: (monthData[element.year + "-" + element.month] = element.load);
element.year in yearData
? (yearData[element.year] += element.load)
: (yearData[element.year] = element.load)
})
: (yearData[element.year] = element.load);
});
// Agregation for Month data
const agregatedMonthData = await buildAgregatedData(
monthData,
'com.grandlyon.egl.month'
)
await storeData(agregatedMonthData, 'com.grandlyon.egl.month', [
'year',
'month'
])
"com.grandlyon.egl.month"
);
await storeData(agregatedMonthData, "com.grandlyon.egl.month", [
"year",
"month"
]);
// Agregation for Year data
const agregatedYearData = await buildAgregatedData(
yearData,
'com.grandlyon.egl.year'
)
await storeData(agregatedYearData, 'com.grandlyon.egl.year', ['year'])
"com.grandlyon.egl.year"
);
await storeData(agregatedYearData, "com.grandlyon.egl.year", ["year"]);
}
}
......@@ -140,130 +141,131 @@ async function agregateMonthAndYearData(data) {
* Return an Array of agregated data
*/
async function buildAgregatedData(data, doctype) {
log('info', 'entering buildAgregatedData')
let agregatedData = []
log("info", "entering buildAgregatedData");
let agregatedData = [];
for (let [key, value] of Object.entries(data)) {
const data = await buildDataFromKey(doctype, key, value)
const oldValue = await resetInProgressAggregatedData(data, doctype)
log('info', 'Dataload + oldvalue is ' + data.load + ' + ' + oldValue)
data.load += oldValue
agregatedData.push(data)
const data = await buildDataFromKey(doctype, key, value);
const oldValue = await resetInProgressAggregatedData(data, doctype);
log("info", "Dataload + oldvalue is " + data.load + " + " + oldValue);
data.load += oldValue;
agregatedData.push(data);
}
return agregatedData
return agregatedData;
}
async function authenticate(login, password, baseUrl, apiAuthKey) {
const authRequest = {
method: "POST",
uri: baseUrl + "/connect.aspx",
headers: {
AuthKey: apiAuthKey,
"Content-Type": "application/x-www-form-urlencoded"
},
body: new URLSearchParams({
formData: {
login: login,
pass: password
})
}
const data = await fetch(baseUrl + "/connect.aspx", authRequest)
const response = await data.json()
},
json: true
};
const response = await rp(authRequest);
if (response.codeRetour === 100) {
return response
return response;
} else {
throw new Error(errors.LOGIN_FAILED)
throw new Error(errors.LOGIN_FAILED);
}
}
async function getData(response, baseUrl, apiAuthKey) {
log('debug', 'Start date : ' + startDate)
log('debug', 'End date : ' + endDate)
log("debug", "Start date : " + startDate);
log("debug", "End date : " + endDate);
const dataRequest = {
method: "POST",
uri: baseUrl + "/getAllAgregatsByAbonnement.aspx",
headers: {
AuthKey: apiAuthKey,
'Content-Type': 'application/x-www-form-urlencoded'
"Content-Type": "application/x-www-form-urlencoded"
},
body: new URLSearchParams({
form: {
token: response.resultatRetour.token,
num_abt: response.resultatRetour.num_abt,
date_debut: startDate,
date_fin: endDate
})
}
},
json: true
};
try {
// Sort data by date
const data = await fetch(
baseUrl + "/getAllAgregatsByAbonnement.aspx",
dataRequest
)
const responseEgl = await data.json()
responseEgl.resultatRetour.sort(function(a, b) {
return new Date(a.DateReleve) - new Date(b.DateReleve);
})
const responseEgl = await rp(dataRequest).then(eglRawData => {
eglRawData.resultatRetour.sort(function(a, b) {
return new Date(a.DateReleve) - new Date(b.DateReleve);
});
return eglRawData;
});
switch (responseEgl.codeRetour) {
case 100:
return format(responseEgl)
return format(responseEgl);
case -2:
throw errors.LOGIN_FAILED
throw errors.LOGIN_FAILED;
case -1:
throw errors.VENDOR_DOWN
throw errors.VENDOR_DOWN;
default:
throw errors.UNKNOWN_ERROR
throw errors.UNKNOWN_ERROR;
}
} catch (error) {
log('debug', 'Error from getAllAgregatsByAbonnement')
throw new Error(errors.VENDOR_DOWN)
log("debug", "Error from getAllAgregatsByAbonnement");
throw new Error(errors.VENDOR_DOWN);
}
}
function format(response) {
log('info', 'origin response size is : ' + response.resultatRetour.length)
log("info", "origin response size is : " + response.resultatRetour.length);
// Store first value as reference for index processing
let refValue = response.resultatRetour[0]
let refValue = response.resultatRetour[0];
// Create copy of data without first value
const data = response.resultatRetour
.slice(1)
.filter(value => value.ValeurIndex)
log('info', 'filtered size is : ' + data.length)
.filter(value => value.ValeurIndex);
log("info", "filtered size is : " + data.length);
return data.map(value => {
const time = moment(value.DateReleve, moment.ISO_8601)
const procesedLoad = value.ValeurIndex - refValue.ValeurIndex
const time = moment(value.DateReleve, moment.ISO_8601);
const procesedLoad = value.ValeurIndex - refValue.ValeurIndex;
if (procesedLoad < 0) {
log(
'error',
`processing load for day ${parseInt(time.format('D'))}/${parseInt(
time.format('M')
)}/${parseInt(time.format('YYYY'))}, value is : ${procesedLoad}`
)
throw errors.VENDOR_DOWN
"error",
`processing load for day ${parseInt(time.format("D"))}/${parseInt(
time.format("M")
)}/${parseInt(time.format("YYYY"))}, value is : ${procesedLoad}`
);
throw errors.VENDOR_DOWN;
}
// Change index ref value
refValue = value
refValue = value;
return {
load: procesedLoad,
year: parseInt(time.format('YYYY')),
month: parseInt(time.format('M')),
day: parseInt(time.format('D')),
year: parseInt(time.format("YYYY")),
month: parseInt(time.format("M")),
day: parseInt(time.format("D")),
hour: 0,
minute: 0,
type: value.TypeAgregat
}
})
};
});
}
/**
* Save data in the right doctype db and prevent duplicated keys
*/
async function storeData(data, doctype, filterKeys) {
log('debug', 'Store into ' + doctype)
log('debug', 'Store into keys : ' + filterKeys)
log("debug", "Store into " + doctype);
log("debug", "Store into keys : " + filterKeys);
// data.map(v => {
// log("info", "Saving data " + v.load + " for " + v.day + "/" + v.month + "/" + v.year);
// });
const filteredDocuments = await hydrateAndFilter(data, doctype, {
keys: filterKeys
})
return await addData(filteredDocuments, doctype)
});
return await addData(filteredDocuments, doctype);
}
/**
......@@ -273,24 +275,24 @@ async function storeData(data, doctype, filterKeys) {
* For month doctype: key = "YYYY-MM"
*/
async function buildDataFromKey(doctype, key, value) {
let year, month, day, hour
if (doctype === 'com.grandlyon.egl.year') {
year = key
month = 1
day = 0
hour = 0
} else if (doctype === 'com.grandlyon.egl.month') {
const split = key.split('-')
year = split[0]
month = split[1]
day = 0
hour = 0
let year, month, day, hour;
if (doctype === "com.grandlyon.egl.year") {
year = key;
month = 1;
day = 0;
hour = 0;
} else if (doctype === "com.grandlyon.egl.month") {
const split = key.split("-");
year = split[0];
month = split[1];
day = 0;
hour = 0;
} else {
const split = key.split('-')
year = split[0]
month = split[1]
day = split[2]
hour = split[3]
const split = key.split("-");
year = split[0];
month = split[1];
day = split[2];
hour = split[3];
}
return {
load: Math.round(value * 10000) / 10000,
......@@ -299,7 +301,7 @@ async function buildDataFromKey(doctype, key, value) {
day: parseInt(day),
hour: parseInt(hour),
minute: 0
}
};
}
/**
......@@ -311,21 +313,21 @@ async function buildDataFromKey(doctype, key, value) {
*/
async function resetInProgressAggregatedData(data, doctype) {
// /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
log('debug', 'Remove aggregated data for ' + doctype)
const result = await cozyClient.data.findAll(doctype)
log("debug", "Remove aggregated data for " + doctype);
const result = await cozyClient.data.findAll(doctype);
if (result && result.length > 0) {
// Filter data to remove
var filtered = []
if (doctype === 'com.grandlyon.egl.year') {
var filtered = [];
if (doctype === "com.grandlyon.egl.year") {
// Yearly case
filtered = result.filter(function(el) {
return el.year == data.year
})
} else if (doctype === 'com.grandlyon.egl.month') {
return el.year == data.year;
});
} else if (doctype === "com.grandlyon.egl.month") {
// Monthly case
filtered = result.filter(function(el) {
return el.year == data.year && el.month == data.month
})
return el.year == data.year && el.month == data.month;
});
} else {
// Hourly case
filtered = result.filter(function(el) {
......@@ -334,17 +336,17 @@ async function resetInProgressAggregatedData(data, doctype) {
el.month == data.month &&
el.day == data.day &&
el.hour == data.hour
)
})
);
});
}
// Remove data
let sum = 0.0
let sum = 0.0;
for (const doc of filtered) {
sum += doc.load
log('debug', 'Removing this entry for ' + doc.load)
await cozyClient.data.delete(doctype, doc)
sum += doc.load;
log("debug", "Removing this entry for " + doc.load);
await cozyClient.data.delete(doctype, doc);
}
return sum
return sum;
}
return 0.0
return 0.0;
}
yarn.lock
View file @ cc0df612
......@@ -4406,13 +4406,6 @@ nock@^12.0.3:
lodash "^4.17.13"
propagate "^2.0.0"
node-fetch@2:
version "2.6.7"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
dependencies:
whatwg-url "^5.0.0"
node-fetch@2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
......@@ -6323,11 +6316,6 @@ tough-cookie@~2.4.3:
psl "^1.1.24"
punycode "^1.4.1"
tr46@~0.0.3:
version "0.0.3"
resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
tslib@^1.9.0:
version "1.14.1"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
......@@ -6585,11 +6573,6 @@ watchpack@^2.2.0:
glob-to-regexp "^0.4.1"
graceful-fs "^4.1.2"
webidl-conversions@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=
webpack-cli@3.3.12:
version "3.3.12"
resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-3.3.12.tgz#94e9ada081453cd0aa609c99e500012fd3ad2d4a"
......@@ -6701,14 +6684,6 @@ webpack@5.38.1:
watchpack "^2.2.0"
webpack-sources "^2.3.0"
whatwg-url@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0=
dependencies:
tr46 "~0.0.3"
webidl-conversions "^3.0.0"
which-module@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment