diff --git a/.gitignore b/.gitignore
index 35d328e6d61fa10cf76e954c16d272bb6dd0b0f5..b6c5e5d6cae8c12ec44d39ef7b9bc8cd18b98eb1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,7 +28,7 @@ desktop.ini
.floo
.flooignore
.idea/
-
+.vscode/
# Default
# /!\ KEEP THIS SECTION THE LAST ONE
!.gitkeep
diff --git a/src/aggregate.js b/src/aggregate.js
new file mode 100644
index 0000000000000000000000000000000000000000..808d34f0c36877aa7ab15856851e60a8a9168965
--- /dev/null
+++ b/src/aggregate.js
@@ -0,0 +1,106 @@
+// @ts-check
+const { log, cozyClient } = require('cozy-konnector-libs')
+
+/**
+ * Retrieve and remove old data for a specific doctype
+ * Return an Array of agregated data
+ */
+async function buildAgregatedData(data, doctype) {
+ let agregatedData = []
+ // eslint-disable-next-line no-unused-vars
+ for (let [key, value] of Object.entries(data)) {
+ const data = await buildDataFromKey(doctype, key, value)
+ const oldValue = await resetInProgressAggregatedData(data, doctype)
+ data.load += oldValue
+ agregatedData.push(data)
+ }
+ return agregatedData
+}
+
+/**
+ * Format an entry for DB storage
+ * using key and value
+ * For year doctype: key = "YYYY"
+ * For month doctype: key = "YYYY-MM"
+ */
+async function buildDataFromKey(doctype, key, value) {
+ let year, month, day, hour
+ if (doctype === 'com.grandlyon.enedis.year') {
+ year = key
+ month = 1
+ day = 0
+ hour = 0
+ } else if (doctype === 'com.grandlyon.enedis.month') {
+ const split = key.split('-')
+ year = split[0]
+ month = split[1]
+ day = 0
+ hour = 0
+ } else {
+ const split = key.split('-')
+ year = split[0]
+ month = split[1]
+ day = split[2]
+ hour = split[3]
+ }
+ return {
+ load: Math.round(value * 10000) / 10000,
+ year: parseInt(year),
+ month: parseInt(month),
+ day: parseInt(day),
+ hour: parseInt(hour),
+ minute: 0,
+ }
+}
+
+/**
+ * Function handling special case.
+ * The temporary aggregated data need to be remove in order for the most recent one te be saved.
+ * ex for com.grandlyon.enedis.year :
+ * { load: 76.712, year: 2020, ... } need to be replace by
+ * { load: 82.212, year: 2020, ... } after enedis data reprocess
+ */
+async function resetInProgressAggregatedData(data, doctype) {
+ // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
+ log('debug', doctype, 'Remove aggregated data for')
+ const result = await cozyClient.data.findAll(doctype)
+ if (result && result.length > 0) {
+ // Filter data to remove
+ var filtered = []
+ if (doctype === 'com.grandlyon.enedis.year') {
+ // Yearly case
+ filtered = result.filter(function(el) {
+ return el.year == data.year
+ })
+ } else if (doctype === 'com.grandlyon.enedis.month') {
+ // Monthly case
+ filtered = result.filter(function(el) {
+ return el.year == data.year && el.month == data.month
+ })
+ } else {
+ // Hourly case
+ filtered = result.filter(function(el) {
+ return (
+ el.year == data.year &&
+ el.month == data.month &&
+ el.day == data.day &&
+ el.hour == data.hour
+ )
+ })
+ }
+ // Remove data
+ let sum = 0.0
+ // eslint-disable-next-line no-unused-vars
+ for (const doc of filtered) {
+ sum += doc.load
+ log('debug', doc, 'Removing this entry for ' + doctype)
+ await cozyClient.data.delete(doctype, doc)
+ }
+ return sum
+ }
+ return 0.0
+}
+
+module.exports = {
+ buildAgregatedData,
+}
diff --git a/src/index.js b/src/index.js
index a44fa7573596c58e671f8c9c7c3b66908734c2ed..ab1d32997f2ddfb06e0bf1023d49b5021eca1fe5 100644
--- a/src/index.js
+++ b/src/index.js
@@ -9,6 +9,8 @@ const soapRequest = require('easy-soap-request')
const moment = require('moment')
require('moment-timezone')
const xml2js = require('xml2js')
+const { buildAgregatedData } = require('./aggregate')
+const { userTechnicalData, userMesureDetailles } = require('./request')
moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone
@@ -19,8 +21,7 @@ let startDailyDate = manualExecution
? moment().subtract(12, 'month')
: moment().subtract(6, 'month')
let startDailyDateString = startDailyDate.format('YYYY-MM-DD')
-// const startLoadDate = moment().subtract(7, 'day')
-// const startLoadDateString = startLoadDate.format('YYYY-MM-DD')
+const startLoadDate = moment().subtract(7, 'day')
const endDate = moment()
const endDateString = endDate.format('YYYY-MM-DD')
@@ -46,8 +47,7 @@ async function start(fields, cozyParameters) {
//TODO: authentification ?
log('info', 'Successfully logged in')
- //TODO: get compteur start data
-
+ log('info', 'Querying data...')
await getDataStartDate(
`${baseUrl}/enedis_SGE_ConsultationDonneesTechniquesContractuelles/1.0`,
apiAuthKey,
@@ -60,7 +60,13 @@ async function start(fields, cozyParameters) {
loginUtilisateur,
fields.pointId
)
- log('info', 'Konnector process end')
+ await getDataHalfHour(
+ `${baseUrl}/enedis_SGE_ConsultationMesuresDetaillees/1.0`,
+ apiAuthKey,
+ loginUtilisateur,
+ fields.pointId
+ )
+ log('info', 'Querying data: done')
}
/**
*
@@ -98,7 +104,7 @@ async function getDataStartDate(url, apiAuthKey, userLogin, pointId) {
}
/**
- *
+ * Get hour data
* @param {string} url
* @param {string} apiAuthKey
* @param {string} userLogin
@@ -111,6 +117,17 @@ async function getData(url, apiAuthKey, userLogin, pointId) {
apikey: apiAuthKey,
}
+ // If start date exceed the maximum amount of data we can get with one query
+ // get only 24 month
+ if (moment(endDate).diff(startDailyDate, 'months', true) > 24) {
+ log(
+ 'info',
+ 'Start date exceed 24 month, setting start date to current date minus 24 month'
+ )
+ startDailyDate = moment(endDate).subtract(24, 'month')
+ startDailyDateString = startDailyDate.format('YYYY-MM-DD')
+ }
+
const { response } = await soapRequest({
url: url,
headers: sampleHeaders,
@@ -137,6 +154,62 @@ async function getData(url, apiAuthKey, userLogin, pointId) {
)
}
+/**
+ * Get half-hour data
+ * @param {string} url
+ * @param {string} apiAuthKey
+ * @param {string} userLogin
+ * @param {number} pointId
+ */
+async function getDataHalfHour(url, apiAuthKey, userLogin, pointId) {
+ log('info', 'Fetching data')
+ const sampleHeaders = {
+ 'Content-Type': 'text/xml;charset=UTF-8',
+ apikey: apiAuthKey,
+ }
+
+ let MAX_HISTO = 4
+ // If manual execution, retrieve only 1 week
+ if (manualExecution) {
+ MAX_HISTO = 1
+ }
+ for (var i = 0; i < MAX_HISTO; i++) {
+ log('info', 'launch process with history')
+ const increamentedStartDateString = moment(startLoadDate)
+ .subtract(7 * i, 'day')
+ .format('YYYY-MM-DD')
+ const incrementedEndDateString = moment(endDate)
+ .subtract(7 * i, 'day')
+ .format('YYYY-MM-DD')
+ const { response } = await soapRequest({
+ url: url,
+ headers: sampleHeaders,
+ xml: userMesureDetailles(
+ pointId,
+ userLogin,
+ increamentedStartDateString,
+ incrementedEndDateString,
+ 'COURBE',
+ 'PA'
+ ),
+ }).catch(err => {
+ log('error', 'userMesureDetailles half-hour')
+ log('error', err)
+ return err
+ })
+
+ xml2js.parseString(
+ response.body,
+ {
+ tagNameProcessors: [parseTags],
+ valueProcessors: [parseValue],
+ explicitArray: false,
+ },
+ processData('com.grandlyon.enedis.minute')
+ )
+ }
+}
+
/**
* Format tag in order to be manipulated easly
* @param {string} name
@@ -165,8 +238,10 @@ function parseValue(value, name) {
/**
* Parse data
+ * @param {string} doctype
+ * @returns
*/
-function processData() {
+function processData(doctype = 'com.grandlyon.enedis.day') {
return async (err, result) => {
if (err) {
log('error', err)
@@ -174,11 +249,18 @@ function processData() {
}
// Return only needed part of info
const data = parseSgeXmlData(result)
- return storeData(
+ const processedDailyData = await storeData(
await formateDataForDoctype(data),
- 'com.grandlyon.enedis.day',
- ['year', 'month', 'day']
+ doctype,
+ ['year', 'month', 'day', 'hour', 'minute']
)
+
+ log('info', 'Agregate enedis daily data for month and year')
+ if (doctype === 'com.grandlyon.enedis.day') {
+ console.log(processedDailyData.length)
+
+ await agregateMonthAndYearData(processedDailyData)
+ }
}
}
@@ -237,66 +319,8 @@ async function storeData(data, doctype, filterKeys) {
const filteredDocuments = await hydrateAndFilter(data, doctype, {
keys: filterKeys,
})
- return await addData(filteredDocuments, doctype)
-}
-
-/**
- * Query SGE in order to get info
- * @param {number} pointId
- * @param {string} userLogin
- * @param {string} startDt
- * @param {string} endDt
- * @returns {string}
- */
-function userMesureDetailles(pointId, userLogin, startDt, endDt) {
- log('info', `Query data between ${startDt} and ${endDt}`)
- return `<?xml version='1.0' encoding='utf-8'?>
- <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
- xmlns:v2="http://www.enedis.fr/sge/b2b/services/consultationmesuresdetaillees/v2.0"
- xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0">
- <soapenv:Header/>
- <soapenv:Body>
- <v2:consulterMesuresDetaillees>
- <demande>
- <initiateurLogin>${userLogin}</initiateurLogin>
- <pointId>${pointId}</pointId>
- <mesuresTypeCode>ENERGIE</mesuresTypeCode>
- <grandeurPhysique>EA</grandeurPhysique>
- <soutirage>true</soutirage>
- <injection>false</injection>
- <dateDebut>${startDt}</dateDebut>
- <dateFin>${endDt}</dateFin>
- <mesuresCorrigees>false</mesuresCorrigees>
- <accordClient>true</accordClient>
- </demande>
- </v2:consulterMesuresDetaillees>
- </soapenv:Body>
- </soapenv:Envelope>
- `
-}
-
-/**
- * Get user technical data
- * @param {string} pointId
- * @param {string} userLogin
- * @returns {string}
- */
-function userTechnicalData(pointId, userLogin) {
- log('info', `Query userMesureDetailles`)
- return `<?xml version='1.0' encoding='utf-8'?>
- <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
- xmlns:v2="http://www.enedis.fr/sge/b2b/services/consulterdonneestechniquescontractuelles/v1.0"
- xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0">
- <soapenv:Header/>
- <soapenv:Body>
- <v2:consulterDonneesTechniquesContractuelles>
- <pointId>${pointId}</pointId>
- <loginUtilisateur>${userLogin}</loginUtilisateur>
- <autorisationClient>true</autorisationClient>
- </v2:consulterDonneesTechniquesContractuelles>
- </soapenv:Body>
- </soapenv:Envelope>
- `
+ await addData(filteredDocuments, doctype)
+ return filteredDocuments
}
/**
@@ -306,10 +330,8 @@ function userTechnicalData(pointId, userLogin) {
*/
async function formateDataForDoctype(data) {
log('info', 'Formating data')
- // record
return data.map(record => {
let date = moment(record.d, 'YYYY/MM/DD h:mm:ss')
-
return {
load: record.v,
year: parseInt(date.format('YYYY')),
@@ -320,3 +342,37 @@ async function formateDataForDoctype(data) {
}
})
}
+
+/**
+ * Agregate data from daily data to monthly and yearly data
+ */
+async function agregateMonthAndYearData(data) {
+ // Sum year and month values into object with year or year-month as keys
+ if (data && data.length > 0) {
+ let monthData = {}
+ let yearData = {}
+ data.forEach(element => {
+ element.year + '-' + element.month in monthData
+ ? (monthData[element.year + '-' + element.month] += element.load)
+ : (monthData[element.year + '-' + element.month] = element.load)
+ element.year in yearData
+ ? (yearData[element.year] += element.load)
+ : (yearData[element.year] = element.load)
+ })
+ // Agregation for Month data
+ const agregatedMonthData = await buildAgregatedData(
+ monthData,
+ 'com.grandlyon.enedis.month'
+ )
+ await storeData(agregatedMonthData, 'com.grandlyon.enedis.month', [
+ 'year',
+ 'month',
+ ])
+ // Agregation for Year data
+ const agregatedYearData = await buildAgregatedData(
+ yearData,
+ 'com.grandlyon.enedis.year'
+ )
+ await storeData(agregatedYearData, 'com.grandlyon.enedis.year', ['year'])
+ }
+}
diff --git a/src/request.js b/src/request.js
new file mode 100644
index 0000000000000000000000000000000000000000..cfdef54bd46ce4e0f30edefc664007f6b8460bb0
--- /dev/null
+++ b/src/request.js
@@ -0,0 +1,104 @@
+// @ts-check
+const { log } = require('cozy-konnector-libs')
+
+/**
+ * Query SGE in order to get info
+ * @param {number} pointId
+ * @param {string} userLogin
+ * @param {string} startDt
+ * @param {string} endDt
+ * @returns {string}
+ */
+function userMesureDetailles(
+ pointId,
+ userLogin,
+ startDt,
+ endDt,
+ mesureType = 'ENERGIE',
+ unit = 'EA'
+) {
+ log(
+ 'info',
+ `Query data ${mesureType}/${unit} between ${startDt} and ${endDt}`
+ )
+ return `<?xml version='1.0' encoding='utf-8'?>
+ <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
+ xmlns:v2="http://www.enedis.fr/sge/b2b/services/consultationmesuresdetaillees/v2.0"
+ xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0">
+ <soapenv:Header/>
+ <soapenv:Body>
+ <v2:consulterMesuresDetaillees>
+ <demande>
+ <initiateurLogin>${userLogin}</initiateurLogin>
+ <pointId>${pointId}</pointId>
+ <mesuresTypeCode>${mesureType}</mesuresTypeCode>
+ <grandeurPhysique>${unit}</grandeurPhysique>
+ <soutirage>true</soutirage>
+ <injection>false</injection>
+ <dateDebut>${startDt}</dateDebut>
+ <dateFin>${endDt}</dateFin>
+ <mesuresCorrigees>false</mesuresCorrigees>
+ <accordClient>true</accordClient>
+ </demande>
+ </v2:consulterMesuresDetaillees>
+ </soapenv:Body>
+ </soapenv:Envelope>
+ `
+}
+
+function userMesureDetaillesHalfHour(pointId, userLogin, startDt, endDt) {
+ log('info', `Query data between ${startDt} and ${endDt}`)
+ return `<?xml version='1.0' encoding='utf-8'?>
+ <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
+ xmlns:v2="http://www.enedis.fr/sge/b2b/services/consultationmesuresdetaillees/v2.0"
+ xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0">
+ <soapenv:Header/>
+ <soapenv:Body>
+ <v2:consulterMesuresDetaillees>
+ <demande>
+ <initiateurLogin>${userLogin}</initiateurLogin>
+ <pointId>${pointId}</pointId>
+ <mesuresTypeCode>COURBE</mesuresTypeCode>
+ <grandeurPhysique>PA</grandeurPhysique>
+ <soutirage>true</soutirage>
+ <injection>false</injection>
+ <dateDebut>${startDt}</dateDebut>
+ <dateFin>${endDt}</dateFin>
+ <mesuresCorrigees>false</mesuresCorrigees>
+ <accordClient>true</accordClient>
+ </demande>
+ </v2:consulterMesuresDetaillees>
+ </soapenv:Body>
+ </soapenv:Envelope>
+ `
+}
+
+/**
+ * Get user technical data
+ * @param {number} pointId
+ * @param {string} userLogin
+ * @returns {string}
+ */
+function userTechnicalData(pointId, userLogin) {
+ log('info', `Query userMesureDetailles`)
+ return `<?xml version='1.0' encoding='utf-8'?>
+ <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
+ xmlns:v2="http://www.enedis.fr/sge/b2b/services/consulterdonneestechniquescontractuelles/v1.0"
+ xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0">
+ <soapenv:Header/>
+ <soapenv:Body>
+ <v2:consulterDonneesTechniquesContractuelles>
+ <pointId>${pointId}</pointId>
+ <loginUtilisateur>${userLogin}</loginUtilisateur>
+ <autorisationClient>true</autorisationClient>
+ </v2:consulterDonneesTechniquesContractuelles>
+ </soapenv:Body>
+ </soapenv:Envelope>
+ `
+}
+
+module.exports = {
+ userTechnicalData,
+ userMesureDetaillesHalfHour,
+ userMesureDetailles,
+}