From 1fd9d9f94387e55e43525c25ec7671b59ce313a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Pailharey?= <rpailharey@grandlyon.com>
Date: Wed, 25 Oct 2023 18:10:26 +0200
Subject: [PATCH] wip: luxon + filter aggregate
---
package.json | 6 +--
src/index.js | 127 +++++++++++++++++++++++++++++----------------------
yarn.lock | 24 ++--------
3 files changed, 79 insertions(+), 78 deletions(-)
diff --git a/package.json b/package.json
index fcad023..19a4b31 100644
--- a/package.json
+++ b/package.json
@@ -41,13 +41,11 @@
"dependencies": {
"@sentry/node": "7.30.0",
"@sentry/tracing": "7.30.0",
+ "axios": "1.2.2",
"cozy-konnector-libs": "4.56.4",
- "moment": "^2.24.0",
- "moment-timezone": "^0.5.26",
- "axios": "1.2.2"
+ "luxon": "^3.4.3"
},
"devDependencies": {
- "@types/moment-timezone": "^0.5.30",
"copy-webpack-plugin": "6.1.1",
"cozy-app-publish": "0.25.0",
"cozy-jobs-cli": "1.20.2",
diff --git a/src/index.js b/src/index.js
index 576bc8a..7232875 100644
--- a/src/index.js
+++ b/src/index.js
@@ -2,17 +2,13 @@ const {
BaseKonnector,
log,
errors,
- addData,
updateOrCreate,
hydrateAndFilter,
+ cozyClient,
} = require('cozy-konnector-libs')
const axios = require('axios').default
-const moment = require('moment')
-require('moment-timezone')
-
-moment.locale('fr') // set the language
-moment.tz.setDefault('Europe/Paris') // set the timezone
+const { DateTime } = require('luxon')
const Sentry = require('@sentry/node')
// eslint-disable-next-line
@@ -22,15 +18,15 @@ const { isDev } = require('./helpers/env')
const manualExecution = process.env.COZY_JOB_MANUAL_EXECUTION === 'true'
-const startDate = manualExecution
- ? moment()
- .subtract(1, 'year')
- .format('MM/DD/YYYY')
- : moment()
- .subtract(3, 'year')
- .format('MM/DD/YYYY')
+// const startDate = moment().subtract(manualExecution ? 1 : 3, 'month')
+const startDate = DateTime.now()
+ .setLocale('en')
+ .minus({ year: manualExecution ? 1 : 3 })
+const startDateString = startDate.toLocaleString(DateTime.DATE_SHORT)
+const endDateString = DateTime.now()
+ .setLocale('en')
+ .toLocaleString(DateTime.DATE_SHORT)
-const endDate = moment().format('MM/DD/YYYY')
const rangeDate = {
day: {
doctype: 'com.grandlyon.egl.day',
@@ -99,38 +95,61 @@ async function start(fields, cozyParameters) {
const eglData = await getData(response, baseUrl, apiAuthKey)
- if (eglData) {
- log('debug', 'Process EGL daily data')
- const filteredDocuments = await hydrateAndFilter(
- eglData,
- rangeDate.day.doctype,
- {
- keys: rangeDate.day.keys,
- }
- )
- await addData(filteredDocuments, rangeDate.day.doctype)
-
- log('debug', 'Aggregate EGL monthly load data')
- const monthlyLoads = aggregateMonthlyLoad(eglData)
- log('debug', 'Store aggregated EGL monthly load data')
- await updateOrCreate(
- monthlyLoads,
- rangeDate.month.doctype,
- rangeDate.month.keys
- )
-
- log('debug', 'Aggregate EGL yearly load data')
- const yearlyLoads = aggregateYearlyLoad(monthlyLoads)
- log('debug', 'Store aggregated EGL yearly load data')
- await updateOrCreate(
- yearlyLoads,
- rangeDate.year.doctype,
- rangeDate.year.keys
- )
- } else {
+ if (!eglData) {
log('debug', 'No data found')
+ transaction.setStatus(Tracing.SpanStatus.Ok)
+ transaction.finish()
+ return
}
+ log('debug', 'Process EGL daily data')
+ const filteredDays = await hydrateAndFilter(
+ eglData,
+ rangeDate.day.doctype,
+ {
+ keys: ['year', 'month', 'day', 'load'],
+ }
+ )
+ await updateOrCreate(
+ filteredDays,
+ rangeDate.day.doctype,
+ rangeDate.day.keys
+ )
+
+ const monthlyLoads = aggregateMonthlyLoad(eglData)
+ const monthlyLoadRef = await cozyClient.data.defineIndex(
+ rangeDate.month.doctype,
+ rangeDate.month.keys
+ )
+ const firstMonthlyLoad = await cozyClient.data.query(monthlyLoadRef, {
+ selector: { year: startDate.year, month: startDate.month },
+ limit: 1,
+ })
+ log('warn', firstMonthlyLoad)
+ const filteredMonthlyLoads = monthlyLoads.filter(
+ monthlyLoad =>
+ monthlyLoad.year !== startDate.year ||
+ monthlyLoad.month !== startDate.month
+ )
+ log('debug', 'Store aggregated EGL monthly load data')
+ await updateOrCreate(
+ filteredMonthlyLoads,
+ rangeDate.month.doctype,
+ rangeDate.month.keys
+ )
+
+ log('debug', 'Aggregate EGL yearly load data')
+ const yearlyLoads = aggregateYearlyLoad(monthlyLoads)
+ const filteredYearlyLoads = yearlyLoads.filter(
+ monthlyLoad => monthlyLoad.year !== startDate.year
+ )
+ log('debug', 'Store aggregated EGL yearly load data')
+ await updateOrCreate(
+ filteredYearlyLoads,
+ rangeDate.year.doctype,
+ rangeDate.year.keys
+ )
+
transaction.setStatus(Tracing.SpanStatus.Ok)
transaction.finish()
} catch (error) {
@@ -313,8 +332,8 @@ async function getData(response, baseUrl, apiAuthKey) {
data: {
token: response.resultatRetour.token,
num_abt: response.resultatRetour.num_abt,
- date_debut: startDate,
- date_fin: endDate,
+ date_debut: startDateString,
+ date_fin: endDateString,
},
}
@@ -356,8 +375,8 @@ async function getData(response, baseUrl, apiAuthKey) {
section: 'getData',
},
extra: {
- start: startDate,
- end: endDate,
+ start: startDateString,
+ end: endDateString,
},
})
@@ -403,14 +422,12 @@ function format(response) {
try {
return data.map(value => {
- const time = moment(value.DateReleve, moment.ISO_8601)
+ const time = DateTime.fromISO(value.DateReleve)
const processedLoad = value.ValeurIndex - refValue.ValeurIndex
if (processedLoad < 0) {
- const errorMessage = `Processing load error for day ${parseInt(
- time.format('D')
- )}/${parseInt(time.format('M'))}/${parseInt(
- time.format('YYYY')
+ const errorMessage = `Processing load error for day ${time.toLocaleString(
+ DateTime.DATE_SHORT
)}, value is: ${processedLoad}`
log('debug', errorMessage)
throw errors.VENDOR_DOWN
@@ -421,9 +438,9 @@ function format(response) {
return {
load: processedLoad,
- year: parseInt(time.format('YYYY')),
- month: parseInt(time.format('M')),
- day: parseInt(time.format('D')),
+ year: time.year,
+ month: time.month,
+ day: time.day,
hour: 0,
minute: 0,
type: value.TypeAgregat,
diff --git a/yarn.lock b/yarn.lock
index 99d097f..3f6650a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1435,13 +1435,6 @@
resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c"
integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==
-"@types/moment-timezone@^0.5.30":
- version "0.5.30"
- resolved "https://registry.yarnpkg.com/@types/moment-timezone/-/moment-timezone-0.5.30.tgz#340ed45fe3e715f4a011f5cfceb7cb52aad46fc7"
- integrity sha512-aDVfCsjYnAQaV/E9Qc24C5Njx1CoDjXsEgkxtp9NyXDpYu4CCbmclb6QhWloS9UTU/8YROUEEdEkWI0D7DxnKg==
- dependencies:
- moment-timezone "*"
-
"@types/node@*":
version "18.11.18"
resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f"
@@ -5659,6 +5652,11 @@ lru_map@^0.3.3:
resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd"
integrity sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ==
+luxon@^3.4.3:
+ version "3.4.3"
+ resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.4.3.tgz#8ddf0358a9492267ffec6a13675fbaab5551315d"
+ integrity sha512-tFWBiv3h7z+T/tDaoxA8rqTxy1CHV6gHS//QdaH4pulbq/JuBSGgQspQQqcgnwdAx6pNI7cmvz5Sv/addzHmUg==
+
macos-release@^2.2.0:
version "2.5.0"
resolved "https://registry.yarnpkg.com/macos-release/-/macos-release-2.5.0.tgz#067c2c88b5f3fb3c56a375b2ec93826220fa1ff2"
@@ -5943,18 +5941,6 @@ modify-values@^1.0.0:
resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022"
integrity sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw==
-moment-timezone@*, moment-timezone@^0.5.26:
- version "0.5.40"
- resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.40.tgz#c148f5149fd91dd3e29bf481abc8830ecba16b89"
- integrity sha512-tWfmNkRYmBkPJz5mr9GVDn9vRlVZOTe6yqY92rFxiOdWXbjaR0+9LwQnZGGuNR63X456NqmEkbskte8tWL5ePg==
- dependencies:
- moment ">= 2.9.0"
-
-"moment@>= 2.9.0", moment@^2.24.0:
- version "2.29.4"
- resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108"
- integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==
-
morgan@^1.9.1:
version "1.10.0"
resolved "https://registry.yarnpkg.com/morgan/-/morgan-1.10.0.tgz#091778abc1fc47cd3509824653dae1faab6b17d7"
--
GitLab