Skip to content
Snippets Groups Projects
Commit cba9031d authored by git-directory-deploy's avatar git-directory-deploy
Browse files

publish: update egl 

generated from commit 55779530
parent 252d283e
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 291 additions and 162 deletions
index.js
+ 291
− 162
View file @ cba9031d
......@@ -95,39 +95,46 @@ const {
addData,
hydrateAndFilter,
cozyClient
} = __webpack_require__(1);
} = __webpack_require__(1)
 
const rp = __webpack_require__(24);
const moment = __webpack_require__(1492);
__webpack_require__(1630);
const rp = __webpack_require__(24)
const moment = __webpack_require__(1492)
__webpack_require__(1630)
 
moment.locale("fr"); // set the language
moment.tz.setDefault("Europe/Paris"); // set the timezone
moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone
 
const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === "true" ? true : false
const manualExecution =
process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false
 
const startDate = manualExecution
? moment().startOf("year").subtract(1, "year").format("MM/DD/YYYY")
: moment().startOf("year").subtract(3, "year").format("MM/DD/YYYY")
const endDate = moment().format("MM/DD/YYYY");
const timeRange = ["day", "month", "year"];
? moment()
.startOf('year')
.subtract(1, 'year')
.format('MM/DD/YYYY')
: moment()
.startOf('year')
.subtract(3, 'year')
.format('MM/DD/YYYY')
const endDate = moment().format('MM/DD/YYYY')
// const timeRange = ['day', 'month', 'year']
const rangeDate = {
day: {
doctype: "com.grandlyon.egl.day",
keys: ["year", "month", "day"]
doctype: 'com.grandlyon.egl.day',
keys: ['year', 'month', 'day']
},
month: {
doctype: "com.grandlyon.egl.month",
keys: ["year", "month"]
doctype: 'com.grandlyon.egl.month',
keys: ['year', 'month']
},
year: {
doctype: "com.grandlyon.egl.year",
keys: ["year"]
doctype: 'com.grandlyon.egl.year',
keys: ['year']
}
};
}
 
module.exports = new BaseKonnector(start);
module.exports = new BaseKonnector(start)
 
// The start function is run by the BaseKonnector instance only when it got all the account
// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
......@@ -136,80 +143,159 @@ async function start(fields, cozyParameters) {
try {
// resetting data for demo only
// await resetData()
const baseUrl = cozyParameters.secret.eglBaseURL;
const apiAuthKey = cozyParameters.secret.eglAPIAuthKey;
log("info", "Authenticating ...");
const baseUrl = cozyParameters.secret.eglBaseURL
const apiAuthKey = cozyParameters.secret.eglAPIAuthKey
log('info', 'Authenticating ...')
const response = await authenticate(
fields.login,
fields.password,
baseUrl,
apiAuthKey
);
log("info", "Successfully logged in");
Promise.all(
timeRange.map(timeStep =>
processData(timeStep, response, baseUrl, apiAuthKey)
)
log('info', 'Successfully logged in')
const eglData = await getData(response, baseUrl, apiAuthKey)
if (eglData) {
log('debug', 'Process egl daily data')
const processedLoadData = await processData(
eglData,
rangeDate.day.doctype,
rangeDate.day.keys
)
);
log('debug', 'Agregate egl load data for month and year')
await agregateMonthAndYearData(processedLoadData)
} else {
log('debug', 'No data found')
}
} catch (error) {
throw new Error(error.message);
}
}
async function processData(timeStep, response, baseUrl, apiAuthKey) {
const doctype = rangeDate[timeStep].doctype;
const loadProfile = await getData(response, baseUrl, apiAuthKey);
log("info", "Saving data to Cozy");
if (doctype === rangeDate.day.doctype) {
await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys);
} else if (doctype === rangeDate.month.doctype) {
await resetInProgressAggregatedData(rangeDate.month.doctype);
const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month);
log("info", "Saving monthly data");
await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys);
} else if (doctype === rangeDate.year.doctype) {
await resetInProgressAggregatedData(rangeDate.year.doctype);
const yearlyData = processYearAggregation(
loadProfile,
rangeDate.year.doctype
);
log("info", "Saving yearly data");
await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys);
} else {
throw new Error("Unkonw range type: " + doctype);
throw new Error(error.message)
}
}
 
/**
* Parse data
* Remove existing data from DB using hydrateAndFilter
* Store filtered data
* Return the list of filtered data
*/
async function processData(data, doctype, filterKeys) {
// const formatedData = await formateData(data)
log('debug', 'processData - data formated')
// Remove data for existing days into the DB
const filteredData = await hydrateAndFilter(data, doctype, {
keys: filterKeys
})
log('debug', 'processData - data filtered')
// Store new day data
await storeData(filteredData, doctype, filterKeys)
return filteredData
}
/**
* Agregate data from daily data to monthly and yearly data
*/
async function agregateMonthAndYearData(data) {
// Sum year and month values into object with year or year-month as keys
if (data && data.length !== 0) {
let monthData = {}
let yearData = {}
data.forEach(element => {
element.year + '-' + element.month in monthData
? (monthData[element.year + '-' + element.month] += element.load)
: (monthData[element.year + '-' + element.month] = element.load)
element.year in yearData
? (yearData[element.year] += element.load)
: (yearData[element.year] = element.load)
})
// Agregation for Month data
const agregatedMonthData = await buildAgregatedData(
monthData,
'com.grandlyon.egl.month'
)
await storeData(agregatedMonthData, 'com.grandlyon.egl.month', [
'year',
'month'
])
// Agregation for Year data
const agregatedYearData = await buildAgregatedData(
yearData,
'com.grandlyon.egl.year'
)
await storeData(agregatedYearData, 'com.grandlyon.egl.year', ['year'])
}
}
/**
* Retrieve and remove old data for a specific doctype
* Return an Array of agregated data
*/
async function buildAgregatedData(data, doctype) {
let agregatedData = []
for (let [key, value] of Object.entries(data)) {
const data = await buildDataFromKey(doctype, key, value)
const oldValue = await resetInProgressAggregatedData(data, doctype)
data.load += oldValue
agregatedData.push(data)
}
return agregatedData
}
// async function processData(timeStep, response, baseUrl, apiAuthKey) {
// const doctype = rangeDate[timeStep].doctype;
// const loadProfile = await getData(response, baseUrl, apiAuthKey);
// log("info", "Saving data to Cozy");
// if (doctype === rangeDate.day.doctype) {
// await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys);
// } else if (doctype === rangeDate.month.doctype) {
// await resetInProgressAggregatedData(rangeDate.month.doctype);
// const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month);
// log("info", "Saving monthly data");
// await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys);
// } else if (doctype === rangeDate.year.doctype) {
// await resetInProgressAggregatedData(rangeDate.year.doctype);
// const yearlyData = processYearAggregation(
// loadProfile,
// rangeDate.year.doctype
// );
// log("info", "Saving yearly data");
// await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys);
// } else {
// throw new Error("Unkonw range type: " + doctype);
// }
// }
async function authenticate(login, password, baseUrl, apiAuthKey) {
const authRequest = {
method: "POST",
uri: baseUrl + "/connect.aspx",
method: 'POST',
uri: baseUrl + '/connect.aspx',
headers: {
AuthKey: apiAuthKey,
"Content-Type": "application/x-www-form-urlencoded"
'Content-Type': 'application/x-www-form-urlencoded'
},
form: {
login: login,
pass: password
},
json: true
};
const response = await rp(authRequest);
}
const response = await rp(authRequest)
if (response.codeRetour === 100) {
return response;
return response
} else {
throw new Error(errors.LOGIN_FAILED);
throw new Error(errors.LOGIN_FAILED)
}
}
 
async function getData(response, baseUrl, apiAuthKey) {
log("debug", "Start date : " + startDate);
log("debug", "End date : " + endDate);
log('debug', 'Start date : ' + startDate)
log('debug', 'End date : ' + endDate)
const dataRequest = {
method: "POST",
uri: baseUrl + "/getAllAgregatsByAbonnement.aspx",
method: 'POST',
uri: baseUrl + '/getAllAgregatsByAbonnement.aspx',
headers: {
AuthKey: apiAuthKey,
"Content-Type": "application/x-www-form-urlencoded"
'Content-Type': 'application/x-www-form-urlencoded'
},
form: {
token: response.resultatRetour.token,
......@@ -218,116 +304,116 @@ async function getData(response, baseUrl, apiAuthKey) {
date_fin: endDate
},
json: true
};
}
try {
const responseEgl = await rp(dataRequest);
const responseEgl = await rp(dataRequest)
switch (responseEgl.codeRetour) {
case 100:
return format(responseEgl);
return format(responseEgl)
case -2:
throw new Error(errors.LOGIN_FAILED);
throw new Error(errors.LOGIN_FAILED)
case -1:
throw new Error(errors.VENDOR_DOWN);
throw new Error(errors.VENDOR_DOWN)
default:
throw new Error(errors.UNKNOWN_ERROR);
throw new Error(errors.UNKNOWN_ERROR)
}
} catch (error) {
throw new Error(errors.VENDOR_DOWN);
throw new Error(errors.VENDOR_DOWN)
}
}
 
function format(response) {
log("info", "origin response size is : " + response.resultatRetour.length)
log('info', 'origin response size is : ' + response.resultatRetour.length)
const data = response.resultatRetour
.slice(1)
.filter(value => value.ValeurIndex);
const dataLen = data.length;
log("info", "filtered size is : " + dataLen);
.filter(value => value.ValeurIndex)
const dataLen = data.length
log('info', 'filtered size is : ' + dataLen)
const mapData = data.map((value, index) => {
const time = moment(value.DateReleve, moment.ISO_8601);
const time = moment(value.DateReleve, moment.ISO_8601)
if (index + 1 < dataLen) {
return {
load: data[index + 1].ValeurIndex - value.ValeurIndex,
year: parseInt(time.format("YYYY")),
month: parseInt(time.format("M")),
day: parseInt(time.format("D")),
year: parseInt(time.format('YYYY')),
month: parseInt(time.format('M')),
day: parseInt(time.format('D')),
hour: 0,
minute: 0,
type: value.TypeAgregat
};
}
} else {
log("info", "end of data - date is : " + value.DateReleve);
log('info', 'end of data - date is : ' + value.DateReleve)
return {
load: null,
year: parseInt(time.format("YYYY")),
month: parseInt(time.format("M")),
day: parseInt(time.format("D")),
year: parseInt(time.format('YYYY')),
month: parseInt(time.format('M')),
day: parseInt(time.format('D')),
hour: 0,
minute: 0,
type: value.TypeAgregat
};
}
}
});
return mapData;
}
function processYearAggregation(data, doctype) {
log("info", "Start aggregation for : " + doctype);
const grouped = data.reduce(reduceYearFunction, {});
return Object.values(grouped);
}
function processMonthlyAggregation(data, range) {
log("info", "Start aggregation for : " + range.doctype);
// Filter by year
const tmpData = groupBy(data, "year");
const keys = Object.keys(tmpData);
var dataToStore = [];
// Monthly aggregation
for (const index in keys) {
// Get daily data of a year
var monthlyData = tmpData[keys[index]];
// Monthly aggregation
var aggregatedData = monthlyData.reduce(reduceMonthFunction, {});
// Store it
dataToStore = dataToStore.concat(Object.values(aggregatedData));
}
return dataToStore;
}
function groupBy(xs, key) {
return xs.reduce(function(rv, x) {
(rv[x[key]] = rv[x[key]] || []).push(x);
return rv;
}, {});
}
function reduceYearFunction(acc, x) {
var id = acc[x.year];
if (id) {
id.load += x.load;
} else {
acc[x.year] = x;
}
return acc;
}
function reduceMonthFunction(acc, x) {
var id = acc[x.month];
if (id) {
id.load += x.load;
} else {
acc[x.month] = x;
}
return acc;
}
})
return mapData
}
// function processYearAggregation(data, doctype) {
// log("info", "Start aggregation for : " + doctype);
// const grouped = data.reduce(reduceYearFunction, {});
// return Object.values(grouped);
// }
// function processMonthlyAggregation(data, range) {
// log("info", "Start aggregation for : " + range.doctype);
// // Filter by year
// const tmpData = groupBy(data, "year");
// const keys = Object.keys(tmpData);
// var dataToStore = [];
// // Monthly aggregation
// for (const index in keys) {
// // Get daily data of a year
// var monthlyData = tmpData[keys[index]];
// // Monthly aggregation
// var aggregatedData = monthlyData.reduce(reduceMonthFunction, {});
// // Store it
// dataToStore = dataToStore.concat(Object.values(aggregatedData));
// }
// return dataToStore;
// }
// function groupBy(xs, key) {
// return xs.reduce(function(rv, x) {
// (rv[x[key]] = rv[x[key]] || []).push(x);
// return rv;
// }, {});
// }
// function reduceYearFunction(acc, x) {
// var id = acc[x.year];
// if (id) {
// id.load += x.load;
// } else {
// acc[x.year] = x;
// }
// return acc;
// }
// function reduceMonthFunction(acc, x) {
// var id = acc[x.month];
// if (id) {
// id.load += x.load;
// } else {
// acc[x.month] = x;
// }
// return acc;
// }
 
/**
* Save data in the right doctype db and prevent duplicated keys
*/
async function storeData(data, doctype, filterKeys) {
log("debug", "Store into " + doctype);
log("debug", "Store into keys : " + filterKeys);
log('debug', 'Store into ' + doctype)
log('debug', 'Store into keys : ' + filterKeys)
// data.map(v => {
// log("info", "Saving data " + v.load + " for " + v.day + "/" + v.month + "/" + v.year);
// });
......@@ -337,44 +423,87 @@ async function storeData(data, doctype, filterKeys) {
return await addData(filteredDocuments, doctype)
}
 
/**
* Format an entry for DB storage
* using key and value
* For year doctype: key = "YYYY"
* For month doctype: key = "YYYY-MM"
*/
async function buildDataFromKey(doctype, key, value) {
let year, month, day, hour
if (doctype === 'com.grandlyon.egl.year') {
year = key
month = 1
day = 0
hour = 0
} else if (doctype === 'com.grandlyon.egl.month') {
const split = key.split('-')
year = split[0]
month = split[1]
day = 0
hour = 0
} else {
const split = key.split('-')
year = split[0]
month = split[1]
day = split[2]
hour = split[3]
}
return {
load: Math.round(value * 10000) / 10000,
year: parseInt(year),
month: parseInt(month),
day: parseInt(day),
hour: parseInt(hour),
minute: 0
}
}
/**
* Function handling special case.
* The temporary aggregated data need to be remove in order for the most recent one te be saved.
* ex for com.grandlyon.egl.month :
* { load: 76.712, month: 2020, ... } need to be replace by
* { load: 82.212, month: 2020, ... } after grdf data reprocess
* { load: 82.212, month: 2020, ... } after egl data reprocess
*/
async function resetInProgressAggregatedData(doctype) {
async function resetInProgressAggregatedData(data, doctype) {
// /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
log("debug", "Remove aggregated data for " + doctype);
const result = await cozyClient.data.findAll(doctype);
if (result.error || result.length <= 0) {
// eslint-disable-next-line no-console
console.warn("Error while fetching loads, doctype not found ");
} else {
const currentDate = moment();
log('debug', 'Remove aggregated data for ' + doctype)
const result = await cozyClient.data.findAll(doctype)
if (result && result.length > 0) {
// Filter data to remove
var filtered = [];
if (doctype === rangeDate.year.doctype) {
var filtered = []
if (doctype === 'com.grandlyon.egl.year') {
// Yearly case
filtered = result.filter(function(el) {
return el.year == currentDate.year();
});
} else {
return el.year == data.year
})
} else if (doctype === 'com.grandlyon.egl.month') {
// Monthly case
filtered = result.filter(function(el) {
return el.year == data.year && el.month == data.month
})
} else {
// Hourly case
filtered = result.filter(function(el) {
return (
el.year == currentDate.year() &&
el.month == parseInt(moment().format("M"))
);
});
el.year == data.year &&
el.month == data.month &&
el.day == data.day &&
el.hour == data.hour
)
})
}
// Remove data
let sum = 0.0
for (const doc of filtered) {
log("debug", "Removing this entry for " + doc);
await cozyClient.data.delete(doctype, doc);
sum += doc.load
log('debug', 'Removing this entry for ' + doc.load)
await cozyClient.data.delete(doctype, doc)
}
return sum
}
return 0.0
}
 
 
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment