-
Rémi PAILHAREY authoredRémi PAILHAREY authored
index.js 18.05 KiB
// @ts-check
const {
BaseKonnector,
log,
hydrateAndFilter,
addData,
errors,
} = require('cozy-konnector-libs')
const soapRequest = require('easy-soap-request')
const moment = require('moment')
require('moment-timezone')
const xml2js = require('xml2js')
const { buildAggregatedData } = require('./helpers/aggregate')
const {
parseSgeXmlData,
formateDataForDoctype,
parseTags,
parseValue,
parseValueHalfHour,
parsePointId,
} = require('./helpers/parsing')
const {
consultationMesuresDetailleesMaxPower,
consultationMesuresDetaillees,
} = require('./requests/sge')
const {
updateBoConsent,
createBoConsent,
getBoConsent,
deleteBoConsent,
} = require('./requests/bo')
const {
verifyUserIdentity,
activateContract,
verifyContract,
terminateContract,
getContractStartDate,
} = require('./core')
const { getAccount, saveAccountData } = require('./requests/cozy')
const { isLocal, isDev } = require('./helpers/env')
const Sentry = require('@sentry/node')
// eslint-disable-next-line
const Tracing = require('@sentry/tracing') // Needed for tracking performance in Sentry
const { version } = require('../package.json')
moment.locale('fr') // set the language
moment.tz.setDefault('Europe/Paris') // set the timezone
/** Connector Constants **/
const manualExecution =
process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false
let startDailyDate = manualExecution
? moment().subtract(12, 'month')
: moment().subtract(6, 'month')
let startDailyDateString = startDailyDate.format('YYYY-MM-DD')
const startLoadDate = moment().subtract(7, 'day')
const endDate = moment()
const endDateString = endDate.format('YYYY-MM-DD')
const ACCOUNT_ID = isLocal() ? 'default_account_id' : 'enedissgegrandlyon'
const NO_DATA = process.env.NO_DATA === 'true'
module.exports = new BaseKonnector(start)
/**
* Sentry
*/
Sentry.init({
dsn: 'https://18747a93401447f2a81b83cd8c4bbbdf@grandlyon.errors.cozycloud.cc/5',
// Set tracesSampleRate to 1.0 to capture 100% // of transactions for performance monitoring.
// We recommend adjusting this value in production
tracesSampleRate: isLocal() ? 0 : 1.0,
release: version,
environment: isDev() ? 'development' : 'production',
debug: isDev(),
integrations: [
// enable HTTP calls tracing
new Sentry.Integrations.Http({ tracing: true }),
],
})
/**
* The start function is run by the BaseKonnector instance only when it got all the account
* information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
* the account information come from ./konnector-dev-config.json file
* cozyParameters are static parameters, independents from the account. Most often, it can be a
* secret api key.
* @param {fields} fields
* @param {{secret: fields}} cozyParameters
*/
async function start(fields, cozyParameters) {
const transaction = Sentry.startTransaction({
op: 'konnector',
name: 'SGE Konnector',
})
transaction.startChild({ op: 'Konnector starting' })
try {
log('info', 'Konnector configuration ...')
log('info', `isManual execution: ${manualExecution}`)
if (NO_DATA) {
log(
'debug',
'NO_DATA is enabled, konnector will stop after verifyUserIdentity()'
)
}
const pointId = parsePointId(parseInt(fields.pointId))
let baseUrl = fields.wso2BaseUrl
let apiAuthKey = fields.apiToken
let contractId = fields.contractId
let sgeLogin = fields.sgeLogin
let boToken = fields.boToken
let boBaseUrl = fields.boBaseUrl
if (cozyParameters && Object.keys(cozyParameters).length !== 0) {
log('debug', 'Found COZY_PARAMETERS')
baseUrl = cozyParameters.secret.wso2BaseUrl
apiAuthKey = cozyParameters.secret.apiToken
contractId = cozyParameters.secret.contractId
sgeLogin = cozyParameters.secret.sgeLogin
boBaseUrl = cozyParameters.secret.boBaseUrl
boToken = cozyParameters.secret.boToken
}
// Prevent missing configuration
if (
!baseUrl ||
!apiAuthKey ||
!contractId ||
!sgeLogin ||
!boToken ||
!boBaseUrl
) {
const errorMessage = 'Missing configuration secrets'
log('error', errorMessage)
Sentry.captureException(errorMessage, {
tags: { section: 'start' },
})
throw new Error(errors.VENDOR_DOWN) }
/**
* If it's first start we have to do the following operations:
* - verify pdl are matching
* - BO: create backoffice consent
* - get contract start date and store it
* - activate half-hour
* - BO: update consent with service ID
*/
log('info', 'User Logging...')
if (isFirstStart(await getAccount(ACCOUNT_ID))) {
log('info', 'First start...')
transaction.startChild({ op: 'First start' })
const user = await verifyUserIdentity(
fields,
baseUrl,
apiAuthKey,
sgeLogin
)
exitIfDebug(user)
let consent = await createBoConsent(
boBaseUrl,
boToken,
pointId,
user.lastname,
user.firstname,
user.address,
user.postalCode,
user.inseeCode,
user.city,
user.hasBeenThroughtSafetyOnBoarding
)
// handle user contract start date in order to properly request data
const userContractStartDate = await getContractStartDate(
baseUrl,
apiAuthKey,
sgeLogin,
pointId
)
startDailyDate = moment(userContractStartDate, 'YYYY-MM-DD')
startDailyDateString = startDailyDate.format('YYYY-MM-DD')
const contractStartDate = moment().format('YYYY-MM-DD')
const contractEndDate = moment()
.add(1, 'year') // SGE force 1 year duration
.format('YYYY-MM-DD')
let serviceId = await verifyContract(
baseUrl,
apiAuthKey,
sgeLogin,
contractId,
user.pointId
)
if (!serviceId) {
serviceId = await activateContract(
baseUrl,
apiAuthKey,
sgeLogin,
contractId,
user.lastname,
user.pointId,
contractStartDate,
contractEndDate )
}
consent = await updateBoConsent(
boBaseUrl,
boToken,
consent,
serviceId.toString()
)
// Save bo id into account
const accountData = await getAccount(ACCOUNT_ID)
await saveAccountData(ACCOUNT_ID, {
...accountData.data,
consentId: consent.ID,
expirationDate: contractEndDate,
inseeCode: user.inseeCode,
})
} else {
log('info', 'Alternate start...')
transaction.startChild({ op: 'Alternate start' })
const accountData = await getAccount(ACCOUNT_ID)
const userConsent = await getBoConsent(
boBaseUrl,
boToken,
accountData.data.consentId
)
const user = await verifyUserIdentity(
fields,
baseUrl,
apiAuthKey,
sgeLogin,
true,
accountData.data.inseeCode
)
exitIfDebug(user)
if (!userConsent) {
const errorMessage = 'No user consent found'
log('error', errorMessage)
Sentry.captureException(errorMessage, {
tags: { section: 'start' },
})
throw new Error(errors.VENDOR_DOWN)
}
const consentEndDate = Date.parse(userConsent.endDate)
const today = Date.now()
if (
user.lastname.toLocaleUpperCase() !==
userConsent.lastname.toLocaleUpperCase() ||
!user ||
consentEndDate < today
) {
await deleteConsent(
userConsent,
baseUrl,
apiAuthKey,
sgeLogin,
contractId,
pointId,
boBaseUrl,
boToken,
consentEndDate < today
)
}
}
log('info', 'Successfully logged in')
await gatherData(baseUrl, apiAuthKey, sgeLogin, pointId)
transaction.finish()
log('info', 'Konnector success')
} catch (error) {
const errorMessage = `SGE konnector encountered an error. Response data: ${JSON.stringify(
error.message
)}`
Sentry.captureMessage(errorMessage, {
tags: {
section: 'start',
},
})
transaction.setStatus(Tracing.spanStatusfromHttpCode(409))
transaction.finish()
await Sentry.flush()
throw error
}
}
/**
* Delete User Consent
* @param {Consent} userConsent
* @param {string} baseUrl
* @param {string} apiAuthKey
* @param {string} sgeLogin
* @param {string} contractId
* @param {string} pointId
* @param {string} boBaseUrl
* @param {string} boToken
* @param {boolean} isConsentExpired
*/
async function deleteConsent(
userConsent,
baseUrl,
apiAuthKey,
sgeLogin,
contractId,
pointId,
boBaseUrl,
boToken,
isConsentExpired
) {
log('error', `Invalid or not found consent for user`)
Sentry.captureMessage(`Invalid or not found consent for user`)
if (userConsent.serviceID) {
await terminateContract(
baseUrl,
apiAuthKey,
sgeLogin,
contractId,
pointId,
userConsent.serviceID
)
await deleteBoConsent(boBaseUrl, boToken, userConsent.ID || 0)
} else {
const errorMessage = `No service id retrieved from BO`
log('error', errorMessage)
Sentry.captureException(errorMessage, {
tags: { section: 'start' },
})
throw new Error(errors.VENDOR_DOWN)
}
if (isConsentExpired) {
Sentry.captureException('Consent expired', {
tags: { section: 'start' },
})
throw new Error(errors.USER_ACTION_NEEDED_OAUTH_OUTDATED)
}
throw new Error(errors.TERMS_VERSION_MISMATCH)
}
/**
* Main method for gathering data
* @param {string} baseUrl
* @param {string} apiAuthKey
* @param {string} sgeLogin
* @param {string} pointId
*/
async function gatherData(baseUrl, apiAuthKey, sgeLogin, pointId) {
log('info', 'Querying data...')
const userContractStartDate = await getContractStartDate(
baseUrl,
apiAuthKey,
sgeLogin,
pointId
)
startDailyDate = moment(userContractStartDate, 'YYYY-MM-DD')
startDailyDateString = startDailyDate.format('YYYY-MM-DD')
await getData(
`${baseUrl}/enedis_SGE_ConsultationMesuresDetaillees_v3/1.0`,
apiAuthKey,
sgeLogin,
pointId
)
await getMaxPowerData(
`${baseUrl}/enedis_SGE_ConsultationMesuresDetaillees_v3/1.0`,
apiAuthKey,
sgeLogin,
pointId
)
await getDataHalfHour(
`${baseUrl}/enedis_SGE_ConsultationMesuresDetaillees_v3/1.0`,
apiAuthKey,
sgeLogin,
pointId
)
log('info', 'Querying data: done')
}
/**
* Get hour data
* @param {string} url
* @param {string} apiAuthKey
* @param {string} userLogin
* @param {string} pointId
*/
async function getData(url, apiAuthKey, userLogin, pointId) {
log('info', 'Fetching data')
const sgeHeaders = {
'Content-Type': 'text/xml;charset=UTF-8',
apikey: apiAuthKey,
}
limitStartDate()
const { response } = await soapRequest({
url: url,
headers: sgeHeaders,
xml: consultationMesuresDetaillees(
pointId,
userLogin,
startDailyDateString,
endDateString,
'ENERGIE',
'EA'
),
}).catch(err => {
log('error', 'consultationMesuresDetaillees')
log('error', err) Sentry.captureException(`consultationMesuresDetaillees: ${err}`, {
tags: { section: 'getData' },
})
return err
})
xml2js.parseString(
response.body,
{
tagNameProcessors: [parseTags],
valueProcessors: [parseValue],
explicitArray: false,
},
processData()
)
}
/**
* Get Max power data
* @param {string} url
* @param {string} apiAuthKey
* @param {string} userLogin
* @param {string} pointId
*/
async function getMaxPowerData(url, apiAuthKey, userLogin, pointId) {
log('info', 'Fetching Max Power data')
const sgeHeaders = {
'Content-Type': 'text/xml;charset=UTF-8',
apikey: apiAuthKey,
}
limitStartDate()
const { response } = await soapRequest({
url: url,
headers: sgeHeaders,
xml: consultationMesuresDetailleesMaxPower(
pointId,
userLogin,
startDailyDateString,
endDateString
),
}).catch(err => {
log('error', 'getMaxPowerData')
log('error', err)
Sentry.captureException(`getMaxPowerData: ${err}`, {
tags: { section: 'getMaxPowerData' },
})
return err
})
xml2js.parseString(
response.body,
{
tagNameProcessors: [parseTags],
valueProcessors: [parseValue],
explicitArray: false,
},
processData('com.grandlyon.enedis.maxpower')
)
}
/**
* If start date exceed the maximum amount of data we can get with one query
* get only 36 month. Or 12 month if manual execution
* On manual execution, set the start date to one year ago.
*/
function limitStartDate() {
const livingDuration = moment(endDate).diff(startDailyDate, 'months', true)
// We need to prevent case that there is less than 12 month data if (manualExecution && livingDuration > 12) {
startDailyDate = moment(endDate).subtract(12, 'month')
startDailyDateString = startDailyDate.format('YYYY-MM-DD')
} else if (livingDuration > 36) {
log(
'info',
'Start date exceed 36 month, setting start date to current date minus 36 month'
)
startDailyDate = moment(endDate).subtract(36, 'month')
startDailyDateString = startDailyDate.format('YYYY-MM-DD')
}
}
/**
* Get half-hour data
* @param {string} url
* @param {string} apiAuthKey
* @param {string} userLogin
* @param {string} pointId
*/
async function getDataHalfHour(url, apiAuthKey, userLogin, pointId) {
log('info', 'Fetching data')
const sgeHeaders = {
'Content-Type': 'text/xml;charset=UTF-8',
apikey: apiAuthKey,
}
// If manual execution, retrieve only 1 week otherwise retrieve 4 weeks
const MAX_HISTO = manualExecution ? 1 : 4
for (let i = 0; i < MAX_HISTO; i++) {
log('info', 'launch process with history')
const incrementedStartDateString = moment(startLoadDate)
.subtract(7 * i, 'day')
.format('YYYY-MM-DD')
const incrementedEndDateString = moment(endDate)
.subtract(7 * i, 'day')
.format('YYYY-MM-DD')
const { response } = await soapRequest({
url: url,
headers: sgeHeaders,
xml: consultationMesuresDetaillees(
pointId,
userLogin,
incrementedStartDateString,
incrementedEndDateString,
'COURBE',
'PA'
),
}).catch(err => {
log('error', 'consultationMesuresDetaillees half-hour')
log('error', err)
Sentry.captureException(
`consultationMesuresDetaillees half-hour: ${err}`,
{
tags: { section: 'getDataHalfHour' },
}
)
return err
})
xml2js.parseString(
response.body,
{
tagNameProcessors: [parseTags],
valueProcessors: [parseValueHalfHour],
explicitArray: false,
},
processData('com.grandlyon.enedis.minute') )
}
}
/**
* Parse data
* @param {string} doctype
* @returns
*/
function processData(doctype = 'com.grandlyon.enedis.day') {
return async (err, result) => {
if (err) {
log('error', err)
Sentry.captureException('error while processing daily data')
throw err
}
// Return only needed part of info
log('info', doctype)
try {
const data = parseSgeXmlData(result)
const processedDailyData = await storeData(
await formateDataForDoctype(data),
doctype,
['year', 'month', 'day', 'hour', 'minute']
)
log('info', 'Aggregate enedis daily data for month and year')
if (doctype === 'com.grandlyon.enedis.day') {
log('info', 'Aggregating...')
await aggregateMonthAndYearData(processedDailyData)
}
} catch (e) {
if (doctype === 'com.grandlyon.enedis.minute') {
const errorMessage = `No half-hour activated. Issue: ${result.Envelope.Body.Fault.faultstring}`
Sentry.captureMessage(errorMessage, {
tags: { section: 'processData' },
})
log('warn', errorMessage)
} else {
log('warn', `Unknown error ${e}`)
}
}
}
}
/**
* Save data in the right doctype db and prevent duplicated keys
* @param {EnedisKonnectorData[]} data
* @param {string} doctype
* @param {string[]} filterKeys
* @returns {Promise<*>}
*/
async function storeData(data, doctype, filterKeys) {
log('debug', doctype, 'Store into')
const filteredDocuments = await hydrateAndFilter(data, doctype, {
keys: filterKeys,
})
await addData(filteredDocuments, doctype)
return filteredDocuments
}
/**
* Aggregate data from daily data to monthly and yearly data
*/
async function aggregateMonthAndYearData(data) {
// Sum year and month values into object with year or year-month as keys
if (data && data.length > 0) {
let monthData = {}
let yearData = {}
data.forEach(element => { element.year + '-' + element.month in monthData
? (monthData[element.year + '-' + element.month] += element.load)
: (monthData[element.year + '-' + element.month] = element.load)
element.year in yearData
? (yearData[element.year] += element.load)
: (yearData[element.year] = element.load)
})
// Aggregation for Month data
const aggregatedMonthData = await buildAggregatedData(
monthData,
'com.grandlyon.enedis.month'
)
await storeData(aggregatedMonthData, 'com.grandlyon.enedis.month', [
'year',
'month',
])
// Aggregation for Year data
const aggregatedYearData = await buildAggregatedData(
yearData,
'com.grandlyon.enedis.year'
)
await storeData(aggregatedYearData, 'com.grandlyon.enedis.year', ['year'])
}
}
/**
* @returns {boolean}
*/
function isFirstStart(account) {
if (account && account.data && account.data.consentId) {
log('info', 'Konnector not first start')
return false
}
log('info', 'Konnector first start')
return true
}
/**
* Check if konnector is launched in local with NO_DATA option
* If so, logs result from verifyUserIdentity() and stops the konnector before getting any data
* @param {User} user - The user object to log
*/
function exitIfDebug(user) {
if (NO_DATA) {
log(
'debug',
`Stopping konnector before getting data, user found from verifyUserIdentity():`
)
log('debug', user)
process.exit()
}
}