Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
egl_konnector
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
web-et-numerique
Factory
LLLE_Project
egl_konnector
Commits
20bad20a
Project 'web-et-numerique/llle_project/egl-konnector' was moved to 'web-et-numerique/factory/llle_project/egl-konnector'. Please update any links and bookmarks that may still have the old path.
Commit
20bad20a
authored
4 years ago
by
Hugo
Browse files
Options
Downloads
Patches
Plain Diff
fix
parent
9021edaa
No related branches found
No related tags found
1 merge request
!4
Dev
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
src/index.js
+122
-135
122 additions, 135 deletions
src/index.js
with
122 additions
and
135 deletions
src/index.js
+
122
−
135
View file @
20bad20a
...
...
@@ -5,38 +5,38 @@ const {
addData
,
hydrateAndFilter
,
cozyClient
}
=
require
(
'
cozy-konnector-libs
'
)
const
rp
=
require
(
'
request-promise
'
)
const
moment
=
require
(
'
moment
'
)
require
(
'
moment-timezone
'
)
}
=
require
(
"
cozy-konnector-libs
"
);
const
rp
=
require
(
"
request-promise
"
);
const
moment
=
require
(
"
moment
"
);
require
(
"
moment-timezone
"
);
moment
.
locale
(
'
fr
'
)
// set the language
moment
.
tz
.
setDefault
(
'
Europe/Paris
'
)
// set the timezone
moment
.
locale
(
"
fr
"
);
// set the language
moment
.
tz
.
setDefault
(
"
Europe/Paris
"
);
// set the timezone
const
startDate
=
moment
()
.
startOf
(
'
year
'
)
.
startOf
(
"
year
"
)
// .subtract(3, 'year') TODO PUT TERNARY EXPRESSION FOR MANUAL LAUNCH
.
subtract
(
1
,
'
year
'
)
.
subtract
(
1
,
'
day
'
)
.
format
(
'
MM/DD/YYYY
'
)
const
endDate
=
moment
().
format
(
'
MM/DD/YYYY
'
)
const
timeRange
=
[
'
day
'
,
'
month
'
,
'
year
'
]
.
subtract
(
1
,
"
year
"
)
.
subtract
(
1
,
"
day
"
)
.
format
(
"
MM/DD/YYYY
"
);
const
endDate
=
moment
().
format
(
"
MM/DD/YYYY
"
);
const
timeRange
=
[
"
day
"
,
"
month
"
,
"
year
"
];
const
rangeDate
=
{
day
:
{
doctype
:
'
com.grandlyon.egl.day
'
,
keys
:
[
'
year
'
,
'
month
'
,
'
day
'
]
doctype
:
"
com.grandlyon.egl.day
"
,
keys
:
[
"
year
"
,
"
month
"
,
"
day
"
]
},
month
:
{
doctype
:
'
com.grandlyon.egl.month
'
,
keys
:
[
'
year
'
,
'
month
'
]
doctype
:
"
com.grandlyon.egl.month
"
,
keys
:
[
"
year
"
,
"
month
"
]
},
year
:
{
doctype
:
'
com.grandlyon.egl.year
'
,
keys
:
[
'
year
'
]
doctype
:
"
com.grandlyon.egl.year
"
,
keys
:
[
"
year
"
]
}
}
}
;
module
.
exports
=
new
BaseKonnector
(
start
)
module
.
exports
=
new
BaseKonnector
(
start
)
;
// The start function is run by the BaseKonnector instance only when it got all the account
// information (fields). When you run this connector yourself in "standalone" mode or "dev" mode,
...
...
@@ -45,82 +45,83 @@ async function start(fields, cozyParameters) {
try
{
// resetting data for demo only
// await resetData()
const
baseUrl
=
cozyParameters
.
secret
.
eglBaseURL
const
apiAuthKey
=
cozyParameters
.
secret
.
eglAPIAuthKey
log
(
'
info
'
,
'
Authenticating ...
'
)
const
baseUrl
=
cozyParameters
.
secret
.
eglBaseURL
;
const
apiAuthKey
=
cozyParameters
.
secret
.
eglAPIAuthKey
;
log
(
"
info
"
,
"
Authenticating ...
"
);
const
response
=
await
authenticate
(
fields
.
login
,
fields
.
password
,
baseUrl
,
apiAuthKey
)
log
(
'
info
'
,
'
Successfully logged in
'
)
)
;
log
(
"
info
"
,
"
Successfully logged in
"
);
Promise
.
all
(
timeRange
.
map
(
timeStep
=>
processData
(
timeStep
,
response
,
baseUrl
,
apiAuthKey
)
)
)
)
;
}
catch
(
error
)
{
throw
new
Error
(
error
.
message
)
throw
new
Error
(
error
.
message
)
;
}
}
async
function
processData
(
timeStep
,
response
,
baseUrl
,
apiAuthKey
)
{
const
doctype
=
rangeDate
[
timeStep
].
doctype
log
(
'
info
'
,
'
Getting data TIMESTEP :
'
+
timeStep
)
const
loadProfile
=
await
getData
(
response
,
baseUrl
,
apiAuthKey
)
log
(
'
info
'
,
'
Saving data to Cozy
'
)
const
doctype
=
rangeDate
[
timeStep
].
doctype
;
log
(
"
info
"
,
"
Getting data TIMESTEP :
"
+
timeStep
);
const
loadProfile
=
await
getData
(
response
,
baseUrl
,
apiAuthKey
);
log
(
"
info
"
,
"
Saving data to Cozy
"
);
log
(
"
info
"
,
"
Response length :
"
+
loadProfile
.
length
);
if
(
doctype
===
rangeDate
.
day
.
doctype
)
{
log
(
'
info
'
,
'
Saving daily data
'
+
loadProfile
)
await
storeData
(
loadProfile
,
rangeDate
.
day
.
doctype
,
rangeDate
.
day
.
keys
)
log
(
"
info
"
,
"
Saving daily data
"
+
loadProfile
)
;
await
storeData
(
loadProfile
,
rangeDate
.
day
.
doctype
,
rangeDate
.
day
.
keys
)
;
}
else
if
(
doctype
===
rangeDate
.
month
.
doctype
)
{
await
resetInProgressAggregatedData
(
rangeDate
.
month
.
doctype
)
const
monthlyData
=
processMonthlyAggregation
(
loadProfile
,
rangeDate
.
month
)
log
(
'
info
'
,
'
Saving monthly data
'
+
monthlyData
)
await
storeData
(
monthlyData
,
rangeDate
.
month
.
doctype
,
rangeDate
.
month
.
keys
)
await
resetInProgressAggregatedData
(
rangeDate
.
month
.
doctype
)
;
const
monthlyData
=
processMonthlyAggregation
(
loadProfile
,
rangeDate
.
month
)
;
log
(
"
info
"
,
"
Saving monthly data
"
+
monthlyData
)
;
await
storeData
(
monthlyData
,
rangeDate
.
month
.
doctype
,
rangeDate
.
month
.
keys
)
;
}
else
if
(
doctype
===
rangeDate
.
year
.
doctype
)
{
await
resetInProgressAggregatedData
(
rangeDate
.
year
.
doctype
)
await
resetInProgressAggregatedData
(
rangeDate
.
year
.
doctype
)
;
const
yearlyData
=
processYearAggregation
(
loadProfile
,
rangeDate
.
year
.
doctype
)
log
(
'
info
'
,
'
Saving yearly data
'
+
yearlyData
)
await
storeData
(
yearlyData
,
rangeDate
.
year
.
doctype
,
rangeDate
.
year
.
keys
)
)
;
log
(
"
info
"
,
"
Saving yearly data
"
+
yearlyData
)
;
await
storeData
(
yearlyData
,
rangeDate
.
year
.
doctype
,
rangeDate
.
year
.
keys
)
;
}
else
{
throw
new
Error
(
'
Unkonw range type:
'
+
doctype
)
throw
new
Error
(
"
Unkonw range type:
"
+
doctype
)
;
}
}
async
function
authenticate
(
login
,
password
,
baseUrl
,
apiAuthKey
)
{
const
authRequest
=
{
method
:
'
POST
'
,
uri
:
baseUrl
+
'
/connect.aspx
'
,
method
:
"
POST
"
,
uri
:
baseUrl
+
"
/connect.aspx
"
,
headers
:
{
AuthKey
:
apiAuthKey
,
'
Content-Type
'
:
'
application/x-www-form-urlencoded
'
"
Content-Type
"
:
"
application/x-www-form-urlencoded
"
},
form
:
{
login
:
login
,
pass
:
password
},
json
:
true
}
const
response
=
await
rp
(
authRequest
)
}
;
const
response
=
await
rp
(
authRequest
)
;
if
(
response
.
codeRetour
===
100
)
{
return
response
return
response
;
}
else
{
throw
new
Error
(
errors
.
LOGIN_FAILED
)
throw
new
Error
(
errors
.
LOGIN_FAILED
)
;
}
}
async
function
getData
(
response
,
baseUrl
,
apiAuthKey
)
{
log
(
'
debug
'
,
'
Start date :
'
+
startDate
)
log
(
'
debug
'
,
'
End date :
'
+
endDate
)
log
(
"
debug
"
,
"
Start date :
"
+
startDate
)
;
log
(
"
debug
"
,
"
End date :
"
+
endDate
)
;
const
dataRequest
=
{
method
:
'
POST
'
,
uri
:
baseUrl
+
'
/getAllAgregatsByAbonnement.aspx
'
,
method
:
"
POST
"
,
uri
:
baseUrl
+
"
/getAllAgregatsByAbonnement.aspx
"
,
headers
:
{
AuthKey
:
apiAuthKey
,
'
Content-Type
'
:
'
application/x-www-form-urlencoded
'
"
Content-Type
"
:
"
application/x-www-form-urlencoded
"
},
form
:
{
token
:
response
.
resultatRetour
.
token
,
...
...
@@ -129,139 +130,125 @@ async function getData(response, baseUrl, apiAuthKey) {
date_fin
:
endDate
},
json
:
true
}
}
;
try
{
const
responseEgl
=
await
rp
(
dataRequest
)
const
responseEgl
=
await
rp
(
dataRequest
)
;
switch
(
responseEgl
.
codeRetour
)
{
case
100
:
return
format
(
responseEgl
)
return
format
(
responseEgl
)
;
case
-
2
:
throw
new
Error
(
errors
.
LOGIN_FAILED
)
throw
new
Error
(
errors
.
LOGIN_FAILED
)
;
case
-
1
:
throw
new
Error
(
errors
.
VENDOR_DOWN
)
throw
new
Error
(
errors
.
VENDOR_DOWN
)
;
default
:
throw
new
Error
(
errors
.
UNKNOWN_ERROR
)
throw
new
Error
(
errors
.
UNKNOWN_ERROR
)
;
}
}
catch
(
error
)
{
throw
new
Error
(
errors
.
VENDOR_DOWN
)
throw
new
Error
(
errors
.
VENDOR_DOWN
)
;
}
}
function
format
(
response
)
{
const
data
=
response
.
resultatRetour
.
slice
(
1
)
.
filter
(
value
=>
value
.
ValeurIndex
)
const
dataLen
=
data
.
length
data
.
map
((
value
,
index
)
=>
{
const
time
=
moment
(
value
.
DateReleve
,
moment
.
ISO_8601
)
.
filter
(
value
=>
value
.
ValeurIndex
)
;
const
dataLen
=
data
.
length
;
const
mapData
=
data
.
map
((
value
,
index
)
=>
{
const
time
=
moment
(
value
.
DateReleve
,
moment
.
ISO_8601
)
;
if
(
index
+
1
<
dataLen
)
{
log
(
'
info
'
,
'
date ->
'
+
"
info
"
,
"
date ->
"
+
value
.
DateReleve
+
'
SUBSTRACTING :
'
+
value
.
ValeurIndex
+
'
-
'
+
response
.
resultatRetour
[
index
]
.
ValeurIndex
+
'
\n
'
)
"
SUBSTRACTING :
"
+
data
[
index
+
1
]
.
ValeurIndex
+
"
-
"
+
value
.
ValeurIndex
+
'
==
'
+
data
[
index
+
1
].
ValeurIndex
-
value
.
ValeurIndex
+
"
\n
"
)
;
return
{
load
:
data
[
index
+
1
].
ValeurIndex
-
value
.
ValeurIndex
,
year
:
parseInt
(
time
.
format
(
'
YYYY
'
)),
month
:
parseInt
(
time
.
format
(
'
M
'
)),
day
:
parseInt
(
time
.
format
(
'
D
'
)),
year
:
parseInt
(
time
.
format
(
"
YYYY
"
)),
month
:
parseInt
(
time
.
format
(
"
M
"
)),
day
:
parseInt
(
time
.
format
(
"
D
"
)),
hour
:
0
,
minute
:
0
,
type
:
value
.
TypeAgregat
}
}
;
}
else
{
return
{
load
:
null
,
year
:
parseInt
(
time
.
format
(
'
YYYY
'
)),
month
:
parseInt
(
time
.
format
(
'
M
'
)),
day
:
parseInt
(
time
.
format
(
'
D
'
)),
year
:
parseInt
(
time
.
format
(
"
YYYY
"
)),
month
:
parseInt
(
time
.
format
(
"
M
"
)),
day
:
parseInt
(
time
.
format
(
"
D
"
)),
hour
:
0
,
minute
:
0
,
type
:
value
.
TypeAgregat
}
}
;
}
})
log
(
'
info
'
,
'
DATALOAD LIST
\n
'
)
data
.
map
(
value
=>
{
log
(
'
info
'
,
value
.
month
+
'
-
'
+
value
.
day
+
'
-
'
+
value
.
year
+
'
:
'
+
value
.
load
+
'
\n
'
)
})
return
data
});
return
mapData
;
}
function
processYearAggregation
(
data
,
doctype
)
{
log
(
'
info
'
,
'
Start aggregation for :
'
+
doctype
)
const
grouped
=
data
.
reduce
(
reduceYearFunction
,
{})
return
Object
.
values
(
grouped
)
log
(
"
info
"
,
"
Start aggregation for :
"
+
doctype
)
;
const
grouped
=
data
.
reduce
(
reduceYearFunction
,
{})
;
return
Object
.
values
(
grouped
)
;
}
function
processMonthlyAggregation
(
data
,
range
)
{
log
(
'
info
'
,
'
Start aggregation for :
'
+
range
.
doctype
)
log
(
"
info
"
,
"
Start aggregation for :
"
+
range
.
doctype
)
;
// Filter by year
const
tmpData
=
groupBy
(
data
,
'
year
'
)
const
keys
=
Object
.
keys
(
tmpData
)
var
dataToStore
=
[]
const
tmpData
=
groupBy
(
data
,
"
year
"
);
const
keys
=
Object
.
keys
(
tmpData
)
;
var
dataToStore
=
[]
;
// Monthly aggregation
for
(
const
index
in
keys
)
{
// Get daily data of a year
var
monthlyData
=
tmpData
[
keys
[
index
]]
var
monthlyData
=
tmpData
[
keys
[
index
]]
;
// Monthly aggregation
var
aggregatedData
=
monthlyData
.
reduce
(
reduceMonthFunction
,
{})
var
aggregatedData
=
monthlyData
.
reduce
(
reduceMonthFunction
,
{})
;
// Store it
dataToStore
=
dataToStore
.
concat
(
Object
.
values
(
aggregatedData
))
dataToStore
=
dataToStore
.
concat
(
Object
.
values
(
aggregatedData
))
;
}
return
dataToStore
return
dataToStore
;
}
function
groupBy
(
xs
,
key
)
{
return
xs
.
reduce
(
function
(
rv
,
x
)
{
;
(
rv
[
x
[
key
]]
=
rv
[
x
[
key
]]
||
[]).
push
(
x
)
return
rv
},
{})
(
rv
[
x
[
key
]]
=
rv
[
x
[
key
]]
||
[]).
push
(
x
)
;
return
rv
;
},
{})
;
}
function
reduceYearFunction
(
acc
,
x
)
{
var
id
=
acc
[
x
.
year
]
var
id
=
acc
[
x
.
year
]
;
if
(
id
)
{
id
.
load
+=
x
.
load
id
.
load
+=
x
.
load
;
}
else
{
acc
[
x
.
year
]
=
x
acc
[
x
.
year
]
=
x
;
}
return
acc
return
acc
;
}
function
reduceMonthFunction
(
acc
,
x
)
{
var
id
=
acc
[
x
.
month
]
var
id
=
acc
[
x
.
month
]
;
if
(
id
)
{
id
.
load
+=
x
.
load
id
.
load
+=
x
.
load
;
}
else
{
acc
[
x
.
month
]
=
x
acc
[
x
.
month
]
=
x
;
}
return
acc
return
acc
;
}
async
function
storeData
(
data
,
doctype
,
keys
)
{
log
(
'
debug
'
,
'
Store into
'
+
doctype
)
log
(
'
debug
'
,
keys
,
'
Store into keys
'
)
log
(
"
debug
"
,
"
Store into
"
+
doctype
)
;
log
(
"
debug
"
,
keys
,
"
Store into keys
"
);
return
hydrateAndFilter
(
data
,
doctype
,
{
keys
:
keys
}).
then
(
filteredDocuments
=>
{
addData
(
filteredDocuments
,
doctype
)
})
addData
(
filteredDocuments
,
doctype
)
;
})
;
}
/**
...
...
@@ -273,33 +260,33 @@ async function storeData(data, doctype, keys) {
*/
async
function
resetInProgressAggregatedData
(
doctype
)
{
// /!\ Warning: cannot use mongo queries because not supported for dev by cozy-konnectors-libs
log
(
'
debug
'
,
doctype
,
'
Remove aggregated data for
'
)
const
result
=
await
cozyClient
.
data
.
findAll
(
doctype
)
log
(
"
debug
"
,
doctype
,
"
Remove aggregated data for
"
);
const
result
=
await
cozyClient
.
data
.
findAll
(
doctype
)
;
if
(
result
.
error
||
result
.
length
<=
0
)
{
// eslint-disable-next-line no-console
console
.
warn
(
'
Error while fetching loads, doctype not found
'
)
console
.
warn
(
"
Error while fetching loads, doctype not found
"
);
}
else
{
const
currentDate
=
moment
()
const
currentDate
=
moment
()
;
// Filter data to remove
var
filtered
=
[]
var
filtered
=
[]
;
if
(
doctype
===
rangeDate
.
year
.
doctype
)
{
// Yearly case
filtered
=
result
.
filter
(
function
(
el
)
{
return
el
.
year
==
currentDate
.
year
()
})
return
el
.
year
==
currentDate
.
year
()
;
})
;
}
else
{
// Monthly case
filtered
=
result
.
filter
(
function
(
el
)
{
return
(
el
.
year
==
currentDate
.
year
()
&&
el
.
month
==
parseInt
(
moment
().
format
(
'
M
'
))
)
})
el
.
month
==
parseInt
(
moment
().
format
(
"
M
"
))
)
;
})
;
}
// Remove data
for
(
const
doc
of
filtered
)
{
log
(
'
debug
'
,
doc
,
'
Removing this entry for
'
+
doctype
)
await
cozyClient
.
data
.
delete
(
doctype
,
doc
)
log
(
"
debug
"
,
doc
,
"
Removing this entry for
"
+
doctype
)
;
await
cozyClient
.
data
.
delete
(
doctype
,
doc
)
;
}
}
}
This diff is collapsed.
Click to expand it.
Yoan VALLET
@yvallet
mentioned in commit
571e3a5c
·
4 years ago
mentioned in commit
571e3a5c
mentioned in commit 571e3a5c52d3c157496fa3871717a5f3233e8922
Toggle commit list
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment