Skip to content
Snippets Groups Projects
Commit d400f76a authored by Hugo NOUTS's avatar Hugo NOUTS
Browse files

Merge branch 'dev' into 'master' 

Dev

See merge request web-et-numerique/llle_project/egl-konnector!4
parents 1426dcd9 cb8024d5
No related branches found
No related tags found
1 merge request!4Dev
Expand all Hide whitespace changes
Inline Side-by-side
manifest.konnector
+ 1
− 1
View file @ d400f76a
{
"version": "0.1.3",
"version": "0.2.0",
"name": "EGL",
"type": "konnector",
"language": "node",
......
package.json
+ 12
− 10
View file @ d400f76a
{
"name": "egl",
"version": "0.1.3",
"version": "0.2.0",
"description": "",
"repository": {
"type": "git",
......@@ -34,26 +34,28 @@
"build": "webpack",
"lint": "eslint --fix .",
"deploy": "git-directory-deploy --directory build/ --branch ${DEPLOY_BRANCH:-build} --repo=${DEPLOY_REPOSITORY:-https://forge.grandlyon.com/web-et-numerique/llle_project/egl-konnector.git}",
"deploy-dev": "git-directory-deploy --directory build/ --branch ${DEPLOY_BRANCH:-build-dev} --repo=${DEPLOY_REPOSITORY:-https://forge.grandlyon.com/web-et-numerique/llle_project/egl-konnector.git}",
"cozyPublish": "cozy-app-publish --token $REGISTRY_TOKEN --build-commit $(git rev-parse ${DEPLOY_BRANCH:-build})",
"travisDeployKey": "./bin/generate_travis_deploy_key"
},
"dependencies": {
"cozy-konnector-libs": "4.15.7",
"cozy-konnector-libs": "4.34.5",
"moment": "^2.24.0",
"moment-timezone": "^0.5.26"
},
"devDependencies": {
"copy-webpack-plugin": "5.0.3",
"cozy-app-publish": "0.15.0",
"cozy-jobs-cli": "1.8.8",
"@types/moment-timezone": "^0.5.30",
"copy-webpack-plugin": "6.1.1",
"cozy-app-publish": "0.25.0",
"cozy-jobs-cli": "1.13.6",
"eslint": "5.16.0",
"eslint-config-cozy-app": "1.1.12",
"eslint-config-cozy-app": "1.6.0",
"eslint-plugin-prettier": "3.0.1",
"git-directory-deploy": "1.5.1",
"husky": "2.2.0",
"husky": "4.3.0",
"konitor": "0.10.2",
"svgo": "1.2.2",
"webpack": "4.30.0",
"webpack-cli": "3.3.1"
"svgo": "1.3.2",
"webpack": "4.44.2",
"webpack-cli": "3.3.12"
}
}
src/index.js
+ 267
− 112
View file @ d400f76a
......@@ -6,6 +6,7 @@ const {
hydrateAndFilter,
cozyClient
} = require('cozy-konnector-libs')
const rp = require('request-promise')
const moment = require('moment')
require('moment-timezone')
......@@ -13,13 +14,21 @@ require('moment-timezone')
moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone
const startDate = moment()
.startOf('year')
.subtract(3, 'year')
.subtract(1, 'day')
.format('MM/DD/YYYY')
const manualExecution =
process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false
const startDate = manualExecution
? moment()
.startOf('year')
.subtract(1, 'year')
.format('MM/DD/YYYY')
: moment()
.startOf('year')
.subtract(3, 'year')
.format('MM/DD/YYYY')
const endDate = moment().format('MM/DD/YYYY')
const timeRange = ['day', 'month', 'year']
// const timeRange = ['day', 'month', 'year']
const rangeDate = {
day: {
doctype: 'com.grandlyon.egl.day',
......@@ -44,10 +53,10 @@ async function start(fields, cozyParameters) {
try {
// resetting data for demo only
// await resetData()
// const baseUrl = fields.eglBaseURL
// const apiAuthKey = fields.eglAPIAuthKey
const baseUrl = cozyParameters.secret.eglBaseURL
const apiAuthKey = cozyParameters.secret.eglAPIAuthKey
log('debug', fields, 'Fields')
log('info', 'Authenticating ...')
const response = await authenticate(
fields.login,
......@@ -56,41 +65,120 @@ async function start(fields, cozyParameters) {
apiAuthKey
)
log('info', 'Successfully logged in')
Promise.all(
timeRange.map(timeStep =>
processData(timeStep, response, baseUrl, apiAuthKey)
const eglData = await getData(response, baseUrl, apiAuthKey)
if (eglData) {
log('debug', 'Process egl daily data')
const processedLoadData = await processData(
eglData,
rangeDate.day.doctype,
rangeDate.day.keys
)
)
log('debug', 'Agregate egl load data for month and year')
await agregateMonthAndYearData(processedLoadData)
} else {
log('debug', 'No data found')
}
} catch (error) {
throw new Error(error.message)
}
}
async function processData(timeStep, response, baseUrl, apiAuthKey) {
const doctype = rangeDate[timeStep].doctype
log('info', 'Getting data')
const loadProfile = await getData(response, baseUrl, apiAuthKey)
log('info', 'Saving data to Cozy')
if (doctype === rangeDate.day.doctype) {
log('info', 'Saving daily data')
await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys)
} else if (doctype === rangeDate.month.doctype) {
log('info', 'Saving monthly data')
await resetInProgressAggregatedData(rangeDate.month.doctype)
const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month)
await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys)
} else if (doctype === rangeDate.year.doctype) {
log('info', 'Saving yearly data')
await resetInProgressAggregatedData(rangeDate.year.doctype)
const yearlyData = processYearAggregation(
loadProfile,
rangeDate.year.doctype
/**
* Parse data
* Remove existing data from DB using hydrateAndFilter
* Store filtered data
* Return the list of filtered data
*/
async function processData(data, doctype, filterKeys) {
// const formatedData = await formateData(data)
log('debug', 'processData - data formated')
// Remove data for existing days into the DB
const filteredData = await hydrateAndFilter(data, doctype, {
keys: filterKeys
})
log('debug', 'processData - data filtered')
// Store new day data
await storeData(filteredData, doctype, filterKeys)
return filteredData
}
/**
* Agregate data from daily data to monthly and yearly data
*/
async function agregateMonthAndYearData(data) {
// Sum year and month values into object with year or year-month as keys
if (data && data.length !== 0) {
let monthData = {}
let yearData = {}
data.forEach(element => {
element.year + '-' + element.month in monthData
? (monthData[element.year + '-' + element.month] += element.load)
: (monthData[element.year + '-' + element.month] = element.load)
element.year in yearData
? (yearData[element.year] += element.load)
: (yearData[element.year] = element.load)
})
// Agregation for Month data
const agregatedMonthData = await buildAgregatedData(
monthData,
'com.grandlyon.egl.month'
)
await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys)
} else {
throw new Error('Unkonw range type: ' + doctype)
await storeData(agregatedMonthData, 'com.grandlyon.egl.month', [
'year',
'month'
])
// Agregation for Year data
const agregatedYearData = await buildAgregatedData(
yearData,
'com.grandlyon.egl.year'
)
await storeData(agregatedYearData, 'com.grandlyon.egl.year', ['year'])
}
}
/**
* Retrieve and remove old data for a specific doctype
* Return an Array of agregated data
*/
async function buildAgregatedData(data, doctype) {
log('info', 'entering buildAgregatedData')
let agregatedData = []
for (let [key, value] of Object.entries(data)) {
const data = await buildDataFromKey(doctype, key, value)
const oldValue = await resetInProgressAggregatedData(data, doctype)
log('info', 'Dataload + oldvalue is ' + data.load + ' + ' + oldValue)
data.load += oldValue
agregatedData.push(data)
}
return agregatedData
}
// async function processData(timeStep, response, baseUrl, apiAuthKey) {
// const doctype = rangeDate[timeStep].doctype;
// const loadProfile = await getData(response, baseUrl, apiAuthKey);
// log("info", "Saving data to Cozy");
// if (doctype === rangeDate.day.doctype) {
// await storeData(loadProfile, rangeDate.day.doctype, rangeDate.day.keys);
// } else if (doctype === rangeDate.month.doctype) {
// await resetInProgressAggregatedData(rangeDate.month.doctype);
// const monthlyData = processMonthlyAggregation(loadProfile, rangeDate.month);
// log("info", "Saving monthly data");
// await storeData(monthlyData, rangeDate.month.doctype, rangeDate.month.keys);
// } else if (doctype === rangeDate.year.doctype) {
// await resetInProgressAggregatedData(rangeDate.year.doctype);
// const yearlyData = processYearAggregation(
// loadProfile,
// rangeDate.year.doctype
// );
// log("info", "Saving yearly data");
// await storeData(yearlyData, rangeDate.year.doctype, rangeDate.year.keys);
// } else {
// throw new Error("Unkonw range type: " + doctype);
// }
// }
async function authenticate(login, password, baseUrl, apiAuthKey) {
const authRequest = {
method: 'POST',
......@@ -114,8 +202,8 @@ async function authenticate(login, password, baseUrl, apiAuthKey) {
}
async function getData(response, baseUrl, apiAuthKey) {
log('debug', startDate, 'Start date')
log('debug', endDate, 'End date')
log('debug', 'Start date : ' + startDate)
log('debug', 'End date : ' + endDate)
const dataRequest = {
method: 'POST',
uri: baseUrl + '/getAllAgregatsByAbonnement.aspx',
......@@ -149,80 +237,140 @@ async function getData(response, baseUrl, apiAuthKey) {
}
function format(response) {
const data = response.resultatRetour.slice(1).map((value, index) => {
log('info', 'origin response size is : ' + response.resultatRetour.length)
const data = response.resultatRetour
.slice(1)
.filter(value => value.ValeurIndex)
const dataLen = data.length
log('info', 'filtered size is : ' + dataLen)
const mapData = data.map((value, index) => {
const time = moment(value.DateReleve, moment.ISO_8601)
return {
load: value.ValeurIndex - response.resultatRetour[index].ValeurIndex,
year: parseInt(time.format('YYYY')),
month: parseInt(time.format('M')),
day: parseInt(time.format('D')),
hour: 0,
minute: 0,
type: value.TypeAgregat
if (index + 1 < dataLen) {
return {
load: data[index + 1].ValeurIndex - value.ValeurIndex,
year: parseInt(time.format('YYYY')),
month: parseInt(time.format('M')),
day: parseInt(time.format('D')),
hour: 0,
minute: 0,
type: value.TypeAgregat
}
} else {
return {
load: null,
year: parseInt(time.format('YYYY')),
month: parseInt(time.format('M')),
day: parseInt(time.format('D')),
hour: 0,
minute: 0,
type: value.TypeAgregat
}
}
})
return data
const res = [...mapData].filter(v => v.load !== null)
return res
}
function processYearAggregation(data, doctype) {
log('info', 'Start aggregation for : ' + doctype)
const grouped = data.reduce(reduceYearFunction, {})
return Object.values(grouped)
}
// function processYearAggregation(data, doctype) {
// log("info", "Start aggregation for : " + doctype);
// const grouped = data.reduce(reduceYearFunction, {});
// return Object.values(grouped);
// }
function processMonthlyAggregation(data, range) {
log('info', 'Start aggregation for : ' + range.doctype)
// Filter by year
const tmpData = groupBy(data, 'year')
const keys = Object.keys(tmpData)
var dataToStore = []
// Monthly aggregation
for (const index in keys) {
// Get daily data of a year
var monthlyData = tmpData[keys[index]]
// Monthly aggregation
var aggregatedData = monthlyData.reduce(reduceMonthFunction, {})
// Store it
dataToStore = dataToStore.concat(Object.values(aggregatedData))
}
return dataToStore
}
// function processMonthlyAggregation(data, range) {
// log("info", "Start aggregation for : " + range.doctype);
// // Filter by year
// const tmpData = groupBy(data, "year");
// const keys = Object.keys(tmpData);
// var dataToStore = [];
// // Monthly aggregation
// for (const index in keys) {
// // Get daily data of a year
// var monthlyData = tmpData[keys[index]];
// // Monthly aggregation
// var aggregatedData = monthlyData.reduce(reduceMonthFunction, {});
// // Store it
// dataToStore = dataToStore.concat(Object.values(aggregatedData));
// }
// return dataToStore;
// }
function groupBy(xs, key) {
return xs.reduce(function(rv, x) {
;(rv[x[key]] = rv[x[key]] || []).push(x)
return rv
}, {})
}
// function groupBy(xs, key) {
// return xs.reduce(function(rv, x) {
// (rv[x[key]] = rv[x[key]] || []).push(x);
// return rv;
// }, {});
// }
function reduceYearFunction(acc, x) {
var id = acc[x.year]
if (id) {
id.load += x.load
} else {
acc[x.year] = x
}
return acc
}
// function reduceYearFunction(acc, x) {
// var id = acc[x.year];
// if (id) {
// id.load += x.load;
// } else {
// acc[x.year] = x;
// }
// return acc;
// }
function reduceMonthFunction(acc, x) {
var id = acc[x.month]
if (id) {
id.load += x.load
} else {
acc[x.month] = x
}
return acc
}
// function reduceMonthFunction(acc, x) {
// var id = acc[x.month];
// if (id) {
// id.load += x.load;
// } else {
// acc[x.month] = x;
// }
// return acc;
// }
async function storeData(data, doctype, keys) {
/**
* Save data in the right doctype db and prevent duplicated keys
*/
async function storeData(data, doctype, filterKeys) {
log('debug', 'Store into ' + doctype)
log('debug', keys, 'Store into keys ')
return hydrateAndFilter(data, doctype, {
keys: keys
}).then(filteredDocuments => {
addData(filteredDocuments, doctype)
log('debug', 'Store into keys : ' + filterKeys)
// data.map(v => {
// log("info", "Saving data " + v.load + " for " + v.day + "/" + v.month + "/" + v.year);
// });
const filteredDocuments = await hydrateAndFilter(data, doctype, {
keys: filterKeys
})
return await addData(filteredDocuments, doctype)
}
/**
* Format an entry for DB storage
* using key and value
* For year doctype: key = "YYYY"
* For month doctype: key = "YYYY-MM"
*/
async function buildDataFromKey(doctype, key, value) {
let year, month, day, hour
if (doctype === 'com.grandlyon.egl.year') {
year = key
month = 1
day = 0
hour = 0
} else if (doctype === 'com.grandlyon.egl.month') {
const split = key.split('-')
year = split[0]
month = split[1]
day = 0
hour = 0
} else {
const split = key.split('-')
year = split[0]
month = split[1]
day = split[2]
hour = split[3]
}
return {
load: Math.round(value * 10000) / 10000,
year: parseInt(year),
month: parseInt(month),
day: parseInt(day),
hour: parseInt(hour),
minute: 0
}
}
/**
......@@ -230,37 +378,44 @@ async function storeData(data, doctype, keys) {
* The temporary aggregated data need to be remove in order for the most recent one te be saved.
* ex for com.grandlyon.egl.month :
* { load: 76.712, month: 2020, ... } need to be replace by
* { load: 82.212, month: 2020, ... } after grdf data reprocess
* { load: 82.212, month: 2020, ... } after egl data reprocess
*/
async function resetInProgressAggregatedData(doctype) {
async function resetInProgressAggregatedData(data, doctype) {
// /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
log('debug', doctype, 'Remove aggregated data for')
log('debug', 'Remove aggregated data for ' + doctype)
const result = await cozyClient.data.findAll(doctype)
if (result.error || result.length <= 0) {
// eslint-disable-next-line no-console
console.warn('Error while fetching loads, doctype not found ')
} else {
const currentDate = moment()
if (result && result.length > 0) {
// Filter data to remove
var filtered = []
if (doctype === rangeDate.year.doctype) {
if (doctype === 'com.grandlyon.egl.year') {
// Yearly case
filtered = result.filter(function(el) {
return el.year == currentDate.year()
return el.year == data.year
})
} else {
} else if (doctype === 'com.grandlyon.egl.month') {
// Monthly case
filtered = result.filter(function(el) {
return el.year == data.year && el.month == data.month
})
} else {
// Hourly case
filtered = result.filter(function(el) {
return (
el.year == currentDate.year() &&
el.month == parseInt(moment().format('M'))
el.year == data.year &&
el.month == data.month &&
el.day == data.day &&
el.hour == data.hour
)
})
}
// Remove data
let sum = 0.0
for (const doc of filtered) {
log('debug', doc, 'Removing this entry for ' + doctype)
sum += doc.load
log('debug', 'Removing this entry for ' + doc.load)
await cozyClient.data.delete(doctype, doc)
}
return sum
}
return 0.0
}
webpack.config.js
+ 4
− 2
View file @ d400f76a
......@@ -36,14 +36,16 @@ module.exports = {
filename: 'index.js'
},
plugins: [
new CopyPlugin([
new CopyPlugin({
patterns: [
{ from: 'manifest.konnector' },
{ from: 'package.json' },
{ from: 'README.md' },
{ from: 'assets', transform: optimizeSVGIcon },
{ from: '.travis.yml' },
{ from: 'LICENSE' }
]),
]
}),
new webpack.DefinePlugin({
__WEBPACK_PROVIDED_MANIFEST__: JSON.stringify(readManifest())
})
......
yarn.lock
+ 3105
− 1840
View file @ d400f76a
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment