diff --git a/manifest.konnector b/manifest.konnector
index ab21de6a5f7cead232b78b05db04fb33df8d44be..f27aab230b206a1632db7e146b65f0b7279c96b8 100644
--- a/manifest.konnector
+++ b/manifest.konnector
@@ -26,9 +26,21 @@
"type": "io.cozy.accounts",
"verbs": ["GET"]
},
- "enedis data": {
+ "half hour enedis load profile data": {
+ "type": "io.enedis.minute"
+ },
+ "hourly enedis load profile": {
+ "type": "io.enedis.hour"
+ },
+ "daily enedis load profile": {
"type": "io.enedis.day"
- }
+ },
+ "monthly enedis load profile": {
+ "type": "io.enedis.month"
+ },
+ "year enedis load profile": {
+ "type": "io.enedis.year"
+ },
},
"developer": {
"name": "Cozy Cloud",
diff --git a/package.json b/package.json
index 2b6b70f34bdb88cb7c26bdab7b717f316e87870f..9f64bbf45907625e871e07c2aa934d8dbf11c756 100644
--- a/package.json
+++ b/package.json
@@ -39,7 +39,9 @@
},
"dependencies": {
"core-util-is": "^1.0.2",
- "cozy-konnector-libs": "4.32.0"
+ "cozy-konnector-libs": "4.32.0",
+ "moment": "^2.25.3",
+ "moment-timezone": "^0.5.28"
},
"devDependencies": {
"copy-webpack-plugin": "5.0.5",
diff --git a/src/index.js b/src/index.js
index 6682cbc7dbbfc49803c901d9d996e29153960e30..6edfa0e035d0d2c9a1233388102c8e893fae4061 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,36 +1,22 @@
const {
BaseKonnector,
- requestFactory,
- signin,
- errors,
- scrape,
- saveBills,
log,
- utils,
+ addData,
hydrateAndFilter,
- addData
+ cozyClient
} = require('cozy-konnector-libs')
const moment = require('moment')
const rp = require('request-promise')
+require('moment-timezone')
+moment.locale('fr') // set the language
+moment.tz.setDefault('Europe/Paris') // set the timezone
+
+/*** Connector Constants ***/
const startDate = moment()
- .subtract(31, 'day')
+ .subtract(32, 'month')
.format('YYYY-MM-DD')
const endDate = moment().format('YYYY-MM-DD')
-
-// const request = requestFactory({
-// // The debug mode shows all the details about HTTP requests and responses. Very useful for
-// // debugging but very verbose. This is why it is commented out by default
-// // debug: true,
-// // Activates [cheerio](https://cheerio.js.org/) parsing on each page
-// cheerio: true,
-// // If cheerio is activated do not forget to deactivate json parsing (which is activated by
-// // default in cozy-konnector-libs
-// json: false,
-// // This allows request-promise to keep cookies between requests
-// jar: true
-// })
-
const baseUrl = 'https://gw.hml.api.enedis.fr'
// The start function is run by the BaseKonnector instance only when it got all the account
@@ -41,37 +27,25 @@ const baseUrl = 'https://gw.hml.api.enedis.fr'
async function start(fields) {
try {
const { access_token } = fields
- console.log(access_token)
+ const usage_point_id = 22516914714270
log('info', 'Fetching enedis data')
- const $ = await getData(access_token)
- log('info', 'Parsing data')
- const documents = await parseDocuments($)
+ const fetchedData = await getDailyData(access_token, usage_point_id)
+ log('info', 'Process data')
+ await processData(fetchedData)
log('info', 'Saving data to Cozy')
- storeData(documents)
} catch (err) {
log('error', err.message)
}
-
- // await saveBills(documents, fields, {
- // // This is a bank identifier which will be used to link bills to bank operations. These
- // // identifiers should be at least a word found in the title of a bank operation related to this
- // // bill. It is not case sensitive.
- // identifiers: ['books'],
- // sourceAccount: this.accountId,
- // sourceAccountIdentifier: fields.login
- // })
}
-// This shows authentication using the [signin function](https://github.com/konnectors/libs/blob/master/packages/cozy-konnector-libs/docs/api.md#module_signin)
-// even if this in another domain here, but it works as an example
-
-async function getData(token) {
- var usagePointID = 32320647321714
+// Retrieve data from the API
+// Format: { value: "Wh", "date": "YYYY-MM-DD" }
+async function getDailyData(token, usagePointID) {
const dataRequest = {
method: 'GET',
uri:
baseUrl +
- '/v3/metering_data/daily_consumption?start=' +
+ '/v4/metering_data/daily_consumption?start=' +
startDate +
'&end=' +
endDate +
@@ -90,25 +64,146 @@ async function getData(token) {
}
}
-function storeData(data) {
- data = JSON.parse(data)
- var dataToStore = data.usage_point[0].meter_reading.interval_reading
- console.log(dataToStore)
- return hydrateAndFilter(dataToStore, 'io.enedis.day', {
- keys: ['rank']
- }).then(filteredDocuments => {
- addData(filteredDocuments, 'io.enedis.day')
+async function processData(data) {
+ const parsedData = JSON.parse(data)
+ const intervalData = parsedData.meter_reading.interval_reading
+ const formatedData = await formateData(intervalData)
+ // Remove data for existing days into the DB
+ const filteredData = await hydrateAndFilter(formatedData, 'io.enedis.day', {
+ keys: ['year', 'month', 'day']
})
+ // Store new day data
+ await storeData(filteredData, 'io.enedis.day', ['year', 'month', 'day'])
+ // Sum year and month values into object with year or year-month as keys
+ if (filteredData && filteredData.length > 0) {
+ let yearData = {}
+ let monthData = {}
+ filteredData.forEach(element => {
+ element.year in yearData
+ ? (yearData[element.year] += element.load)
+ : (yearData[element.year] = element.load)
+ element.year + '-' + element.month in monthData
+ ? (monthData[element.year + '-' + element.month] += element.load)
+ : (monthData[element.year + '-' + element.month] = element.load)
+ })
+
+ const agregatedYearData = await agregateData('io.enedis.year', yearData)
+ await storeData(agregatedYearData, 'io.enedis.year', ['year'])
+
+ const agregatedMonthData = await agregateData('io.enedis.month', monthData)
+ await storeData(agregatedMonthData, 'io.enedis.month', ['year', 'month'])
+ }
+}
+
+/**
+ * Save data in the right doctype db and prevent duplicated keys
+ */
+async function storeData(data, doctype, filterKeys) {
+ log('debug', doctype, 'Store into')
+ const filteredDocuments = await hydrateAndFilter(data, doctype, {
+ keys: filterKeys
+ })
+ return await addData(filteredDocuments, doctype)
}
-// The goal of this function is to parse a HTML page wrapped by a cheerio instance
-// and return an array of JS objects which will be saved to the cozy by saveBills
-// (https://github.com/konnectors/libs/blob/master/packages/cozy-konnector-libs/docs/api.md#savebills)
-function parseDocuments($) {
- // You can find documentation about the scrape function here:
- // https://github.com/konnectors/libs/blob/master/packages/cozy-konnector-libs/docs/api.md#scrape
- log($)
- return $
+/**
+ * Format data for DB storage
+ * Remove bad data
+ */
+async function formateData(data) {
+ log('info', 'Formating data')
+ log('debug', start, 'Start date')
+ return data.map(record => {
+ const date = moment(record.date, 'YYYY/MM/DD')
+ if (record.value != -2) {
+ return {
+ load: parseFloat(record.value / 1000),
+ year: parseInt(date.format('YYYY')),
+ month: parseInt(date.format('M')),
+ day: parseInt(date.format('D')),
+ hour: parseInt(date.format('H')),
+ minute: parseInt(date.format('m'))
+ }
+ }
+ })
+}
+
+/**
+ * Retrieve and remove old data for a specific doctype
+ * Return an Array of agregated data
+ */
+async function agregateData(doctype, data) {
+ let agregatedData = []
+ for (let [key, value] of Object.entries(data)) {
+ const data = await buildDataFromKey(doctype, key, value)
+ const oldValue = await resetInProgressAggregatedData(doctype, data)
+ data.load += oldValue
+ agregatedData.push(data)
+ }
+ return agregatedData
+}
+
+/**
+ * Format an entry for DB storage
+ * using key and value
+ * For year doctype: key = "YYYY"
+ * For month doctype: key = "YYYY-MM"
+ */
+async function buildDataFromKey(doctype, key, value) {
+ let year, month
+ if (doctype === 'io.enedis.year') {
+ year = key
+ month = 1
+ } else {
+ const split = key.split('-')
+ year = split[0]
+ month = split[1]
+ }
+ return {
+ load: Math.round(value * 1000) / 1000,
+ year: parseInt(year),
+ month: parseInt(month),
+ day: 0,
+ hour: 0,
+ minute: 0
+ }
+}
+
+/**
+ * Function handling special case.
+ * The temporary aggregated data need to be remove in order for the most recent one te be saved.
+ * ex for io.enedis.year :
+ * { load: 76.712, year: 2020, ... } need to be replace by
+ * { load: 82.212, year: 2020, ... } after enedis data reprocess
+ */
+async function resetInProgressAggregatedData(doctype, data) {
+ // /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
+ log('debug', doctype, 'Remove aggregated data for')
+ const result = await cozyClient.data.findAll(doctype)
+ if (result && result.length > 0) {
+ // Filter data to remove
+ var filtered = []
+ if (doctype === 'io.enedis.year') {
+ // Yearly case
+ filtered = result.filter(function(el) {
+ return el.year == data.year
+ })
+ } else {
+ // Monthly case
+ filtered = result.filter(function(el) {
+ return el.year == data.year && el.month == data.month
+ })
+ }
+ // Remove data
+ let sum = 0.0
+ for (const doc of filtered) {
+ sum += doc.load
+ log('debug', doc, 'Removing this entry for ' + doctype)
+ await cozyClient.data.delete(doctype, doc)
+ }
+ return sum
+ }
+ return 0.0
}
module.exports = new BaseKonnector(start)
diff --git a/yarn.lock b/yarn.lock
index 93d5cbd6514d4acf44785f2db1575a64958860a9..21ae288d2fdbd5c3a7d22fe6d6fb5a678faa7515 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3173,6 +3173,18 @@ mkdirp@0.5.1, mkdirp@0.x.x, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
dependencies:
minimist "0.0.8"
+moment-timezone@^0.5.28:
+ version "0.5.28"
+ resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.28.tgz#f093d789d091ed7b055d82aa81a82467f72e4338"
+ integrity sha512-TDJkZvAyKIVWg5EtVqRzU97w0Rb0YVbfpqyjgu6GwXCAohVRqwZjf4fOzDE6p1Ch98Sro/8hQQi65WDXW5STPw==
+ dependencies:
+ moment ">= 2.9.0"
+
+"moment@>= 2.9.0", moment@^2.25.3:
+ version "2.25.3"
+ resolved "https://registry.yarnpkg.com/moment/-/moment-2.25.3.tgz#252ff41319cf41e47761a1a88cab30edfe9808c0"
+ integrity sha512-PuYv0PHxZvzc15Sp8ybUCoQ+xpyPWvjOuK72a5ovzp2LI32rJXOiIfyoFoYvG3s6EwwrdkMyWuRiEHSZRLJNdg==
+
move-concurrently@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92"