diff --git a/package.json b/package.json index 66f38956166c45037afe3a2470eec6c2098506ae..c4d42884080422367410ea35e05aa9749ebc1c94 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,11 @@ "travisDeployKey": "./bin/generate_travis_deploy_key" }, "dependencies": { - "cozy-konnector-libs": "4.52.1" + "cozy-konnector-libs": "4.52.1", + "easy-soap-request": "^4.7.0", + "moment": "^2.29.3", + "moment-timezone": "^0.5.34", + "xml2js": "^0.4.23" }, "devDependencies": { "cozy-jobs-cli": "1.17.6", diff --git a/src/index.js b/src/index.js index be21704979cb340475979e699619d819bb0afd2a..4555f68b837490d8f64f9653389a6972375cda37 100644 --- a/src/index.js +++ b/src/index.js @@ -1,26 +1,28 @@ // @ts-check const { BaseKonnector, - requestFactory, - scrape, log, - utils, + hydrateAndFilter, + addData, } = require('cozy-konnector-libs') -const request = requestFactory({ - // The debug mode shows all the details about HTTP requests and responses. Very useful for - // debugging but very verbose. This is why it is commented out by default - // debug: true, - // Activates [cheerio](https://cheerio.js.org/) parsing on each page - cheerio: true, - // If cheerio is activated do not forget to deactivate json parsing (which is activated by - // default in cozy-konnector-libs - json: false, - // This allows request-promise to keep cookies between requests - jar: true, -}) +const soapRequest = require('easy-soap-request') +const moment = require('moment') +require('moment-timezone') +const xml2js = require('xml2js') +moment.locale('fr') // set the language +moment.tz.setDefault('Europe/Paris') // set the timezone -const VENDOR = 'template' -const baseUrl = 'http://books.toscrape.com' +/*** Connector Constants ***/ +const manualExecution = + process.env.COZY_JOB_MANUAL_EXECUTION === 'true' ? true : false +const startDailyDate = manualExecution + ? moment().subtract(12, 'month') + : moment().subtract(6, 'month') +const startDailyDateString = startDailyDate.format('YYYY-MM-DD') +// const startLoadDate = moment().subtract(7, 'day') +// const startLoadDateString = startLoadDate.format('YYYY-MM-DD') +const endDate = moment() +const endDateString = endDate.format('YYYY-MM-DD') module.exports = new BaseKonnector(start) @@ -30,96 +32,197 @@ module.exports = new BaseKonnector(start) // cozyParameters are static parameters, independents from the account. Most often, it can be a // secret api key. async function start(fields, cozyParameters) { + log('info', 'Gathering data ...') + console.log(fields.wso2BaseUrl) + console.log(cozyParameters) + let baseUrl = fields.wso2BaseUrl + let apiAuthKey = fields.apiToken + let loginUtilisateur = fields.loginUtilisateur log('info', 'Authenticating ...') - if (cozyParameters) log('debug', 'Found COZY_PARAMETERS') - await authenticate.bind(this)(fields.login, fields.password) + if (cozyParameters && Object.keys(cozyParameters).length !== 0) { + log('debug', 'Found COZY_PARAMETERS') + baseUrl = cozyParameters.secret.wso2BaseUrl + apiAuthKey = cozyParameters.secret.apiToken + loginUtilisateur = cozyParameters.secret.loginUtilisateur + } + //TODO: authentification ? log('info', 'Successfully logged in') - // The BaseKonnector instance expects a Promise as return of the function - log('info', 'Fetching the list of documents') - const $ = await request(`${baseUrl}/index.html`) - // cheerio (https://cheerio.js.org/) uses the same api as jQuery (http://jquery.com/) - log('info', 'Parsing list of documents') - const documents = await parseDocuments($) - // Here we use the saveBills function even if what we fetch are not bills, - // but this is the most common case in connectors - log('info', 'Saving data to Cozy') - await this.saveBills(documents, fields, { - // This is a bank identifier which will be used to link bills to bank operations. These - // identifiers should be at least a word found in the title of a bank operation related to this - // bill. It is not case sensitive. - identifiers: ['books'], - }) + //TODO: get compteur start data + await getData( + `${baseUrl}/enedis_SGE_ConsultationMesuresDetaillees/1.0`, + apiAuthKey, + loginUtilisateur, + fields.pointId + ) + log('info', 'Konnector process end') } -// This shows authentication using the [signin function](https://github.com/konnectors/libs/blob/master/packages/cozy-konnector-libs/docs/api.md#module_signin) -// even if this in another domain here, but it works as an example -function authenticate(username, password) { - return this.signin({ - url: `http://quotes.toscrape.com/login`, - formSelector: 'form', - formData: { username, password }, - // The validate function will check if the login request was a success. Every website has a - // different way to respond: HTTP status code, error message in HTML ($), HTTP redirection - // (fullResponse.request.uri.href)... - validate: (statusCode, $, fullResponse) => { - log( - 'debug', - fullResponse.request.uri.href, - 'not used here but should be useful for other connectors' - ) - // The login in toscrape.com always works except when no password is set - if ($(`a[href='/logout']`).length === 1) { - return true - } else { - // cozy-konnector-libs has its own logging function which format these logs with colors in - // standalone and dev mode and as JSON in production mode - log('error', $('.error').text()) - return false - } - }, +/** + * + * @param {string} url + * @param {string} apiAuthKey + * @param {string} userLogin + * @param {number} pointId + */ +async function getData(url, apiAuthKey, userLogin, pointId) { + log('info', 'Fetching data') + const sampleHeaders = { + 'Content-Type': 'text/xml;charset=UTF-8', + apikey: apiAuthKey, + } + const { response } = await soapRequest({ + url: url, + headers: sampleHeaders, + xml: userMesureDetailles( + pointId, + userLogin, + startDailyDateString, + endDateString + ), + }).catch(err => { + log('error', err) + return err }) -} -// The goal of this function is to parse a HTML page wrapped by a cheerio instance -// and return an array of JS objects which will be saved to the cozy by saveBills -// (https://github.com/konnectors/libs/blob/master/packages/cozy-konnector-libs/docs/api.md#savebills) -function parseDocuments($) { - // You can find documentation about the scrape function here: - // https://github.com/konnectors/libs/blob/master/packages/cozy-konnector-libs/docs/api.md#scrape - const docs = scrape( - $, + xml2js.parseString( + response.body, { - title: { - sel: 'h3 a', - attr: 'title', - }, - amount: { - sel: '.price_color', - parse: normalizePrice, - }, - fileurl: { - sel: 'img', - attr: 'src', - parse: src => `${baseUrl}/${src}`, - }, + tagNameProcessors: [parseTags], + valueProcessors: [parseValue], + explicitArray: false, }, - 'article' + processData() ) - return docs.map(doc => ({ - ...doc, - // The saveBills function needs a date field - // even if it is a little artificial here (these are not real bills) - date: new Date(), - currency: 'EUR', - filename: `${utils.formatDate(new Date())}_${VENDOR}_${doc.amount.toFixed( - 2 - )}EUR${doc.vendorRef ? '_' + doc.vendorRef : ''}.jpg`, - vendor: VENDOR, - })) } -// Convert a price string to a float -function normalizePrice(price) { - return parseFloat(price.replace('£', '').trim()) +/** + * Format tag in order to be manipulated easly + * @param {string} name + * @returns {string} name + */ +function parseTags(name) { + if (name.split(':')[1] !== undefined) { + return name.split(':')[1] + } + return name +} + +/** + * + * @param {string} value + * @param {string} name + * @returns {string|number} value + */ +function parseValue(value, name) { + // Wh => KWh + if (name === 'v') { + return parseFloat((parseInt(value) / 1000).toFixed(2)) + } + return value +} + +/** + * Parse data + */ +function processData() { + return async (err, result) => { + if (err) { + log('error', err) + throw err + } + // Return only needed part of info + const data = parseSgeXmlData(result) + return storeData( + await formateDataForDoctype(data), + 'com.grandlyon.enedis.day', + ['year', 'month', 'day'] + ) + } +} + +/** + * + * @param {*} result + * @returns {SGEData[]} + */ +function parseSgeXmlData(result) { + log('info', 'Parsing list of documents') + let json = JSON.stringify(result) + return JSON.parse(json)['Envelope']['Body'][ + 'consulterMesuresDetailleesResponse' + ]['grandeur']['mesure'] +} + +/** + * Save data in the right doctype db and prevent duplicated keys + * @param {EnedisKonnectorData[]} data + * @param {string} doctype + * @param {string[]} filterKeys + * @returns + */ +async function storeData(data, doctype, filterKeys) { + log('debug', doctype, 'Store into') + const filteredDocuments = await hydrateAndFilter(data, doctype, { + keys: filterKeys, + }) + return await addData(filteredDocuments, doctype) +} + +/** + * Query SGE in order to get info + * @param {number} pointId + * @param {string} userLogin + * @param {string} startDt + * @param {string} endDt + * @returns {string} + */ +function userMesureDetailles(pointId, userLogin, startDt, endDt) { + log('info', `Query data between ${startDt} and ${endDt}`) + return `<?xml version='1.0' encoding='utf-8'?> + <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" + xmlns:v2="http://www.enedis.fr/sge/b2b/services/consultationmesuresdetaillees/v2.0" + xmlns:v1="http://www.enedis.fr/sge/b2b/technique/v1.0"> + <soapenv:Header/> + <soapenv:Body> + <v2:consulterMesuresDetaillees> + <demande> + <initiateurLogin>${userLogin}</initiateurLogin> + <pointId>${pointId}</pointId> + <mesuresTypeCode>ENERGIE</mesuresTypeCode> + <grandeurPhysique>EA</grandeurPhysique> + <soutirage>true</soutirage> + <injection>false</injection> + <dateDebut>${startDt}</dateDebut> + <dateFin>${endDt}</dateFin> + <mesuresCorrigees>false</mesuresCorrigees> + <accordClient>true</accordClient> + </demande> + </v2:consulterMesuresDetaillees> + </soapenv:Body> + </soapenv:Envelope> + ` +} + +/** + * Format data for DB storage + * @param {SGEData[]} data + * @returns {Promise<EnedisKonnectorData[]>} Parsed timestamp array + */ +async function formateDataForDoctype(data) { + log('info', 'Formating data') + console.log(data) + // record + return data.map(record => { + let date = moment(record.d, 'YYYY/MM/DD h:mm:ss') + + return { + load: record.v, + year: parseInt(date.format('YYYY')), + month: parseInt(date.format('M')), + day: parseInt(date.format('D')), + hour: parseInt(date.format('H')), + minute: parseInt(date.format('m')), + } + }) } diff --git a/src/types.js b/src/types.js new file mode 100644 index 0000000000000000000000000000000000000000..fd86ac644973a019f9a47e35352ed0307ea39aed --- /dev/null +++ b/src/types.js @@ -0,0 +1,16 @@ +/** + * EnedisKonnectorData definition + * @typedef {object} EnedisKonnectorData + * @property {number} year + * @property {number} month + * @property {number} day + * @property {number} hour + * @property {number} minute + */ + +/** + * SGEData definition + * @typedef {object} SGEData + * @property {number} v + * @property {string} d + */ diff --git a/yarn.lock b/yarn.lock index 2985990f679a5ca343b58634ee8056f5132b5f2a..b132129a871c880fbf6d6d2b97ea53ce06183817 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1797,6 +1797,13 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== +axios@^0.26.1: + version "0.26.1" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9" + integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA== + dependencies: + follow-redirects "^1.14.8" + babel-eslint@10.0.1: version "10.0.1" resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.0.1.tgz#919681dc099614cd7d31d45c8908695092a1faed" @@ -2809,6 +2816,13 @@ domutils@^2.5.2, domutils@^2.6.0, domutils@^2.7.0: domelementtype "^2.2.0" domhandler "^4.2.0" +easy-soap-request@^4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/easy-soap-request/-/easy-soap-request-4.7.0.tgz#d57af87fc91a3c8ff606e3ca1102a2c37a095ff8" + integrity sha512-rFvXxk65ROqUkSSgVbxS4X9skRkdBPU4eoETCzgWaKh5seVibLD8feY5F0YV6UTnY7ErcM9DWMG3RNaYm02xHA== + dependencies: + axios "^0.26.1" + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -3568,6 +3582,11 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.5.tgz#76c8584f4fc843db64702a6bd04ab7a8bd666da3" integrity sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg== +follow-redirects@^1.14.8: + version "1.14.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7" + integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w== + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -4612,6 +4631,18 @@ mkdirp@^0.5.1: dependencies: minimist "^1.2.6" +moment-timezone@^0.5.34: + version "0.5.34" + resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.34.tgz#a75938f7476b88f155d3504a9343f7519d9a405c" + integrity sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg== + dependencies: + moment ">= 2.9.0" + +"moment@>= 2.9.0", moment@^2.29.3: + version "2.29.3" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.3.tgz#edd47411c322413999f7a5940d526de183c031f3" + integrity sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw== + morgan@^1.9.1: version "1.10.0" resolved "https://registry.yarnpkg.com/morgan/-/morgan-1.10.0.tgz#091778abc1fc47cd3509824653dae1faab6b17d7" @@ -5686,7 +5717,7 @@ safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.2, safe-buffer@^5.2.1, resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -sax@~1.2.4: +sax@>=0.6.0, sax@~1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== @@ -6622,6 +6653,19 @@ write@1.0.3: dependencies: mkdirp "^0.5.1" +xml2js@^0.4.23: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xmlbuilder@~11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" + integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + y18n@^5.0.5: version "5.0.8" resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"