Newer
Older

Hugo NOUTS
committed
const getDataGenericErrors = require('./helpers/getDataGenericErrors')
const { isDev } = require('./helpers/env')
const moment = require('moment')
require('moment-timezone')
moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone
const Sentry = require('@sentry/node')
// eslint-disable-next-line
const Tracing = require('@sentry/tracing') // Needed for tracking performance in Sentry
const { version } = require('../package.json')
Sentry.init({
dsn:
'https://fa503fe00434433f805d1c715999b7f5@grandlyon.errors.cozycloud.cc/3',
// Set tracesSampleRate to 1.0 to capture 100%
// of transactions for performance monitoring.
// We recommend adjusting this value in production
tracesSampleRate: 1.0,
release: version,

Hugo NOUTS
committed
environment: isDev() ? 'development' : 'production',
debug: isDev(),
integrations: [
// enable HTTP calls tracing
new Sentry.Integrations.Http({ tracing: true })
]
})

Hugo NOUTS
committed
async function standaloneStart(token, pce) {
try {
const grdfData = await getData(token, pce)
log('debug', grdfData)

Hugo NOUTS
committed
if (!grdfData) {
log('debug', 'No consent or data for load curve')
return
}
log('debug', 'Process grdf daily data')
const processedLoadData = await processData(
grdfData,
'com.grandlyon.grdf.day',
['year', 'month', 'day']
)
log('debug', 'Aggregate grdf load data for month and year')
await aggregateMonthAndYearData(processedLoadData)
} catch (error) {
log('error', error)

Hugo NOUTS
committed
log('error', 'Standalone failed')
}
}
const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === 'true'
const startDate = manualExecution
? moment()
.subtract(1, 'year')
.format('YYYY-MM-DD')
: moment()
.subtract(3, 'year')
.format('YYYY-MM-DD')
/**
* The start function is run by the BaseKonnector instance only when it got all the account
* information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
* the account information come from ./konnector-dev-config.json file
* cozyParameters are static parameters, independents from the account. Most often, it can be a
* secret api key.
*/
async function start(fields) {

Hugo NOUTS
committed
if (process.env.NODE_ENV === 'standalone') {
standaloneStart(
fields.oauth.access_token,
fields.oauth_callback_results.pce
)

Hugo NOUTS
committed
return
}
const transaction = Sentry.startTransaction({
op: 'konnector',
name: 'GRDF Konnector'
})
const accountId = getAccountId()
let body = ''
let id_pce = ''
body = await cozyClient.fetchJSON(
'POST',
`/accounts/grdfgrandlyon/${accountId}/refresh`
fields.access_token = body.attributes.oauth.access_token
if (this._account?.oauth_callback_results?.pce && fields.access_token) {
const grdfData = await getData(fields.access_token, id_pce)
if (grdfData) {
log('debug', 'Process grdf daily data')
const processedLoadData = await processData(
grdfData,
'com.grandlyon.grdf.day',
['year', 'month', 'day']
)
log('debug', 'Aggregate grdf load data for month and year')
await aggregateMonthAndYearData(processedLoadData)
} else {
log('debug', 'No consent or data for load curve')
}
} else {
log('debug', 'no id_token found in oauth_callback_results')
log(
'debug',
'callback_result contains: ',
this._account.oauth_callback_results
)
throw errors.USER_ACTION_NEEDED_OAUTH_OUTDATED
}
} catch (err) {
log('debug', 'CATCH ERROR : ' + err)
Sentry.captureException(err)
await Sentry.flush()
throw err

Hugo NOUTS
committed
function buildGetDataUrl(idPCE, startDate, endDate) {
const baseUrl = 'https://api.grdf.fr/adict/v2/pce'
const queryParams = `date_debut=${startDate}&date_fin=${endDate}`
return `${baseUrl}/${idPCE}/donnees_consos_informatives?${queryParams}`
}

Hugo NOUTS
committed
async function getData(token, idPCE) {
const url = buildGetDataUrl(idPCE, startDate, endDate)
log('debug', url)

Hugo NOUTS
committed
const response = await fetch(url, {

Hugo NOUTS
committed
headers: {
'Content-Type': 'application/x-ndjson',
Authorization: `Bearer ${token}`
},

Hugo NOUTS
committed
})
.then(async response => {
if (response.status !== 200) {
log('error', `Response failed with status ${response.status}`)
throw errors.VENDOR_DOWN
return response.text()
return result.match(/.+/g).map(s => {
result = JSON.parse(s)
* Handle no data issue when retrieving grdf data.
* 1000008 code stands for "Il n'y a pas de données correspondant à ce PCE sur la période demandée".

Hugo NOUTS
committed
* It is NOT an important issue deserving to throw an error
* If there is no data, return null data in order to be filtered before saving
*/
if (result.statut_restitution.code !== '1000008') {

Hugo NOUTS
committed
const genError = getDataGenericErrors(
result.statut_restitution.code
)
log(
'warn',
'donnees_consos_informatives responded with : ' +
result.statut_restitution.code +
' -> ' +
result.statut_restitution.message +
' Periode ' +
result.periode.date_debut +
'/' +
result.periode.date_fin
)
Sentry.captureMessage(
`Get data threw an error: ${result.statut_restitution.code} - ${result.statut_restitution.message}`
)
throw genError
} else {
return { energie: null }
return result.consommation
})
})
.catch(error => {
log('debug', 'Error from getData')
throw error

Hugo NOUTS
committed
const filteredRep = response.filter(function(el) {
}
/**
* Parse data
* Remove existing data from DB using hydrateAndFilter
* Store filtered data
* Return the list of filtered data
*/
async function processData(data, doctype, filterKeys) {
const formattedData = await formateData(data)
log('debug', 'processData - data formatted')
const filteredData = await hydrateAndFilter(formattedData, doctype, {
await storeData(filteredData, doctype, filterKeys)
return filteredData
}
/**
* Save data in the right doctype db and prevent duplicated keys
*/
async function storeData(data, doctype, filterKeys) {
log('debug', doctype, 'Store into')
const filteredDocuments = await hydrateAndFilter(data, doctype, {
keys: filterKeys
})
return addData(filteredDocuments, doctype)
}
/**
* Format data for DB storage
* Remove bad data
*/
return data.map(record => {
let date = moment(record.date_debut_consommation, 'YYYY/MM/DD h:mm:ss')
let load =
: record.volume_brut * record.coeff_calcul.coeff_conversion
return {
load: parseFloat(load),
year: parseInt(date.format('YYYY')),
month: parseInt(date.format('M')),
day: parseInt(date.format('D')),
hour: parseInt(date.format('H')),
minute: parseInt(date.format('m'))
}
})
* Aggregate data from daily data to monthly and yearly data
async function aggregateMonthAndYearData(data) {
// Sum year and month values into object with year or year-month as keys
data.forEach(element => {
element.year + '-' + element.month in monthData
? (monthData[element.year + '-' + element.month] += element.load)
: (monthData[element.year + '-' + element.month] = element.load)
element.year in yearData
// Aggregation for Month data
const aggregatedMonthData = await buildAggregatedData(
await storeData(aggregatedMonthData, 'com.grandlyon.grdf.month', [
// Aggregation for Year data
const aggregatedYearData = await buildAggregatedData(
await storeData(aggregatedYearData, 'com.grandlyon.grdf.year', ['year'])
}
}
/**
* Retrieve and remove old data for a specific doctype
async function buildAggregatedData(data, doctype) {
let aggregatedData = []
for (let [key, value] of Object.entries(data)) {
const data = await buildDataFromKey(doctype, key, value)
const oldValue = await resetInProgressAggregatedData(data, doctype)
data.load += oldValue
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
}
/**
* Format an entry for DB storage
* using key and value
* For year doctype: key = "YYYY"
* For month doctype: key = "YYYY-MM"
*/
async function buildDataFromKey(doctype, key, value) {
let year, month, day, hour
if (doctype === 'com.grandlyon.grdf.year') {
year = key
month = 1
day = 0
hour = 0
} else if (doctype === 'com.grandlyon.grdf.month') {
const split = key.split('-')
year = split[0]
month = split[1]
day = 0
hour = 0
} else {
const split = key.split('-')
year = split[0]
month = split[1]
day = split[2]
hour = split[3]
}
return {
load: Math.round(value * 10000) / 10000,
year: parseInt(year),
month: parseInt(month),
day: parseInt(day),
hour: parseInt(hour),
minute: 0
}
}
/**
* Function handling special case.
* The temporary aggregated data need to be remove in order for the most recent one te be saved.
* ex for com.grandlyon.grdf.year :
* { load: 76.712, year: 2020, ... } need to be replace by
* { load: 82.212, year: 2020, ... } after grdf data reprocess
*/
async function resetInProgressAggregatedData(data, doctype) {
// /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
log('debug', doctype, 'Remove aggregated data for')
const result = await cozyClient.data.findAll(doctype)
if (result && result.length > 0) {
// Filter data to remove
var filtered = []
if (doctype === 'com.grandlyon.grdf.year') {
// Yearly case
filtered = result.filter(function(el) {
return el.year == data.year
})
} else if (doctype === 'com.grandlyon.grdf.month') {
// Monthly case
filtered = result.filter(function(el) {
return el.year == data.year && el.month == data.month
})
} else {
// Hourly case
filtered = result.filter(function(el) {
return (
el.year == data.year &&
el.month == data.month &&
el.day == data.day &&
el.hour == data.hour
)
})
}
// Remove data
let sum = 0.0
for (const doc of filtered) {
sum += doc.load
log('debug', doc, 'Removing this entry for ' + doctype)
await cozyClient.data.delete(doctype, doc)
}
return sum
}
return 0.0
}