Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
T
tracemob
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
POCS
cozy
tracemob
Commits
570f9bbb
Commit
570f9bbb
authored
4 years ago
by
paultranvan
Browse files
Options
Downloads
Patches
Plain Diff
refactor: Split into modules
parent
264fc91d
Branches
main
No related tags found
No related merge requests found
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
src/index.js
+2
-19
2 additions, 19 deletions
src/index.js
src/lib.js
+24
-104
24 additions, 104 deletions
src/lib.js
src/timeseries.js
+65
-0
65 additions, 0 deletions
src/timeseries.js
src/trace-requests.js
+73
-0
73 additions, 0 deletions
src/trace-requests.js
with
164 additions
and
123 deletions
src/index.js
+
2
−
19
View file @
570f9bbb
...
@@ -3,13 +3,8 @@ process.env.SENTRY_DSN =
...
@@ -3,13 +3,8 @@ process.env.SENTRY_DSN =
'
https://066ea9e6f9c84ef9aeb1f1592caff488@sentry.cozycloud.cc/147
'
'
https://066ea9e6f9c84ef9aeb1f1592caff488@sentry.cozycloud.cc/147
'
const
{
BaseKonnector
,
log
}
=
require
(
'
cozy-konnector-libs
'
)
const
{
BaseKonnector
,
log
}
=
require
(
'
cozy-konnector-libs
'
)
const
{
const
{
fetchAndSaveTrips
,
fetchAndSaveManualEntries
}
=
require
(
'
./lib
'
)
fetchAndSaveTrips
,
const
{
getFirstAndLastTripTimestamp
}
=
require
(
'
./trace-requests
'
)
fetchAndSaveManualEntries
,
getFirstAndLastTripTimestamp
}
=
require
(
'
./lib
'
)
// const timeseries = client.models.timeseries
module
.
exports
=
new
BaseKonnector
(
start
)
module
.
exports
=
new
BaseKonnector
(
start
)
...
@@ -85,15 +80,3 @@ async function start(fields) {
...
@@ -85,15 +80,3 @@ async function start(fields) {
log
(
'
error
'
,
e
)
log
(
'
error
'
,
e
)
}
}
}
}
/* async function findSavedTripByDates(startDate, endDate) {
console.log(`query from ${startDate} to ${endDate}`)
const trips = await timeseries.fetchTimeSeriesByIntervalAndSource(client, {
dataType: DATA_TYPE,
startDate,
endDate,
source: VENDOR
})
console.log('trips : ', trips)
return trips.data.length > 0 ? trips.data[0] : null
}*/
This diff is collapsed.
Click to expand it.
src/lib.js
+
24
−
104
View file @
570f9bbb
const
{
requestFactory
,
log
,
cozyClient
}
=
require
(
'
cozy-konnector-libs
'
)
const
{
log
}
=
require
(
'
cozy-konnector-libs
'
)
const
VENDOR
=
'
agremob.com
'
const
BASE_URL
=
'
https://trace.grfmap.com:8081
'
const
DATA_TYPE
=
'
geojson
'
const
TRIP_COLLECTION
=
'
analysis/cleaned_trip
'
const
TRIP_COLLECTION
=
'
analysis/cleaned_trip
'
const
PURPOSE_COLLECTION
=
'
manual/purpose_confirm
'
const
PURPOSE_COLLECTION
=
'
manual/purpose_confirm
'
const
MODE_COLLECTION
=
'
manual/mode_confirm
'
const
MODE_COLLECTION
=
'
manual/mode_confirm
'
const
request
=
requestFactory
({
cheerio
:
false
,
json
:
true
,
jar
:
false
})
const
client
=
cozyClient
.
new
const
{
getServerCollectionFromDate
,
getTripsForDay
}
=
require
(
'
./trace-requests
'
)
const
{
saveTrip
,
updateTripsWithManualEntries
}
=
require
(
'
./timeseries.js
'
)
function
filterTripsByDate
(
trips
,
tripStartDates
)
{
function
filterTripsByDate
(
trips
,
tripStartDates
)
{
return
trips
.
filter
(
trip
=>
{
return
trips
.
filter
(
trip
=>
{
...
@@ -19,6 +15,14 @@ function filterTripsByDate(trips, tripStartDates) {
...
@@ -19,6 +15,14 @@ function filterTripsByDate(trips, tripStartDates) {
})
})
}
}
/**
* Fetch and save trips from a trace server
*
* @param {string} token - The user token
* @param {Date} startDate - The starting date
* @param {options} - The options
* @returns {Date} The date of the last saved trip
*/
module
.
exports
.
fetchAndSaveTrips
=
async
function
(
module
.
exports
.
fetchAndSaveTrips
=
async
function
(
token
,
token
,
startDate
,
startDate
,
...
@@ -60,11 +64,19 @@ module.exports.fetchAndSaveTrips = async function(
...
@@ -60,11 +64,19 @@ module.exports.fetchAndSaveTrips = async function(
log
(
'
info
'
,
`Save
${
savePromises
.
length
}
trips`
)
log
(
'
info
'
,
`Save
${
savePromises
.
length
}
trips`
)
await
Promise
.
all
(
savePromises
)
await
Promise
.
all
(
savePromises
)
return
savePromises
.
length
>
1
return
savePromises
.
length
>
0
?
trips
[
trips
.
length
-
1
].
metadata
.
write_fmt_time
?
new
Date
(
trips
[
trips
.
length
-
1
].
metadata
.
write_fmt_time
).
toISOString
()
:
null
:
null
}
}
/**
* Fetch and save manual data from a trace server
*
* @param {string} token - The user token
* @param {Date} startManualDate - The starting date
* @param {options} - The options
* @returns {Date} The date of the last saved manual data
*/
module
.
exports
.
fetchAndSaveManualEntries
=
async
function
(
module
.
exports
.
fetchAndSaveManualEntries
=
async
function
(
token
,
token
,
startManualDate
,
startManualDate
,
...
@@ -111,95 +123,3 @@ module.exports.fetchAndSaveManualEntries = async function(
...
@@ -111,95 +123,3 @@ module.exports.fetchAndSaveManualEntries = async function(
}
}
return
null
return
null
}
}
module
.
exports
.
getFirstAndLastTripTimestamp
=
async
function
(
token
)
{
const
path
=
`
${
BASE_URL
}
/pipeline/get_range_ts`
const
body
=
{
user
:
token
}
return
request
(
path
,
{
method
:
'
POST
'
,
body
})
}
async
function
getTripsForDay
(
token
,
day
)
{
const
path
=
`
${
BASE_URL
}
/timeline/getTrips/
${
day
}
`
const
body
=
{
user
:
token
}
const
trips
=
await
request
(
path
,
{
method
:
'
POST
'
,
body
})
return
trips
.
timeline
}
async
function
getServerCollectionFromDate
(
token
,
startDate
,
collection
,
{
excludeFirst
=
true
}
=
{}
)
{
// Note the expected timestamp is in seconds
const
startTime
=
new
Date
(
startDate
).
getTime
()
/
1000
const
endTime
=
Date
.
now
()
/
1000
const
body
=
{
user
:
token
,
start_time
:
startTime
,
end_time
:
endTime
,
key_list
:
[
collection
]
}
const
path
=
`
${
BASE_URL
}
/datastreams/find_entries/timestamp`
const
results
=
await
request
(
path
,
{
method
:
'
POST
'
,
body
})
return
excludeFirst
?
results
.
phone_data
.
slice
(
1
)
:
results
.
phone_data
}
// TODO: use fetchTimeSeriesByIntervalAndSource from cozy-client models
async
function
findSavedTripByDates
(
startDate
,
endDate
)
{
const
doctype
=
`io.cozy.timeseries.
${
DATA_TYPE
}
`
const
query
=
client
.
find
(
doctype
)
.
where
({
source
:
VENDOR
,
startDate
:
{
$gte
:
startDate
},
endDate
:
{
$lte
:
endDate
}
})
.
indexFields
([
'
source
'
,
'
startDate
'
,
'
endDate
'
])
.
sortBy
([{
source
:
'
desc
'
},
{
startDate
:
'
desc
'
},
{
endDate
:
'
desc
'
}])
.
limitBy
(
1
)
const
trips
=
await
client
.
query
(
query
)
return
trips
.
data
.
length
>
0
?
trips
.
data
[
0
]
:
null
}
// TODO: use saveTimeSeries from cozy-client models
async
function
saveTrip
(
trip
)
{
const
startDate
=
trip
.
properties
.
start_fmt_time
const
endDate
=
trip
.
properties
.
end_fmt_time
const
timeserie
=
{
_type
:
`io.cozy.timeseries.
${
DATA_TYPE
}
`
,
series
:
[
trip
],
startDate
,
endDate
,
source
:
VENDOR
}
return
client
.
save
(
timeserie
)
}
// TODO: use saveTimeSeries from cozy-client models
async
function
updateTripsWithManualEntries
(
manualEntries
,
{
entryKey
})
{
for
(
const
entry
of
manualEntries
)
{
const
savedTrip
=
await
findSavedTripByDates
(
entry
.
data
.
start_fmt_time
,
entry
.
data
.
end_fmt_time
)
if
(
!
savedTrip
)
{
log
(
'
error
'
,
`No trip found for the manual entry from
${
entry
.
data
.
start_fmt_time
}
to
${
entry
.
data
.
end_fmt_time
}
`
)
continue
}
const
newTrip
=
{
...
savedTrip
}
newTrip
.
series
[
0
].
properties
[
entryKey
]
=
entry
.
data
.
label
await
client
.
save
(
newTrip
)
}
}
This diff is collapsed.
Click to expand it.
src/timeseries.js
0 → 100644
+
65
−
0
View file @
570f9bbb
const
{
log
,
cozyClient
}
=
require
(
'
cozy-konnector-libs
'
)
const
DATA_TYPE
=
'
geojson
'
const
VENDOR
=
'
agremob.com
'
const
client
=
cozyClient
.
new
// TODO: use fetchTimeSeriesByIntervalAndSource from cozy-client models
async
function
findSavedTripByDates
(
startDate
,
endDate
)
{
const
doctype
=
`io.cozy.timeseries.
${
DATA_TYPE
}
`
const
query
=
client
.
find
(
doctype
)
.
where
({
source
:
VENDOR
,
startDate
:
{
$gte
:
startDate
},
endDate
:
{
$lte
:
endDate
}
})
.
indexFields
([
'
source
'
,
'
startDate
'
,
'
endDate
'
])
.
sortBy
([{
source
:
'
desc
'
},
{
startDate
:
'
desc
'
},
{
endDate
:
'
desc
'
}])
.
limitBy
(
1
)
const
trips
=
await
client
.
query
(
query
)
return
trips
.
data
.
length
>
0
?
trips
.
data
[
0
]
:
null
}
// TODO: use saveTimeSeries from cozy-client models
const
saveTrip
=
async
function
(
trip
)
{
const
startDate
=
trip
.
properties
.
start_fmt_time
const
endDate
=
trip
.
properties
.
end_fmt_time
const
timeserie
=
{
_type
:
`io.cozy.timeseries.
${
DATA_TYPE
}
`
,
series
:
[
trip
],
startDate
,
endDate
,
source
:
VENDOR
}
return
client
.
save
(
timeserie
)
}
// TODO: use saveTimeSeries from cozy-client models
async
function
updateTripsWithManualEntries
(
manualEntries
,
{
entryKey
})
{
for
(
const
entry
of
manualEntries
)
{
const
savedTrip
=
await
findSavedTripByDates
(
entry
.
data
.
start_fmt_time
,
entry
.
data
.
end_fmt_time
)
if
(
!
savedTrip
)
{
log
(
'
error
'
,
`No trip found for the manual entry from
${
entry
.
data
.
start_fmt_time
}
to
${
entry
.
data
.
end_fmt_time
}
`
)
continue
}
const
newTrip
=
{
...
savedTrip
}
newTrip
.
series
[
0
].
properties
[
entryKey
]
=
entry
.
data
.
label
await
client
.
save
(
newTrip
)
}
}
module
.
exports
=
{
saveTrip
,
updateTripsWithManualEntries
}
This diff is collapsed.
Click to expand it.
src/trace-requests.js
0 → 100644
+
73
−
0
View file @
570f9bbb
const
BASE_URL
=
'
https://trace.grfmap.com:8081
'
const
{
requestFactory
}
=
require
(
'
cozy-konnector-libs
'
)
const
request
=
requestFactory
({
cheerio
:
false
,
json
:
true
,
jar
:
false
})
/**
* Get first and last trip dates
*
* @param {string} token - The user token
* @returns {object} - The first and last trip timestamps
*/
const
getFirstAndLastTripTimestamp
=
async
function
(
token
)
{
const
path
=
`
${
BASE_URL
}
/pipeline/get_range_ts`
const
body
=
{
user
:
token
}
return
request
(
path
,
{
method
:
'
POST
'
,
body
})
}
/**
* Get database collection on a time range
*
* @param {string} token - The user token
* @param {Date} startDate - The starting date
* @param {string} collection - The collection name
* @param {options} - The options
* @returns {object} - The request results
*/
const
getServerCollectionFromDate
=
async
function
(
token
,
startDate
,
collection
,
{
excludeFirst
=
true
}
=
{}
)
{
// Note the expected timestamp is in seconds
const
startTime
=
new
Date
(
startDate
).
getTime
()
/
1000
const
endTime
=
Date
.
now
()
/
1000
const
body
=
{
user
:
token
,
start_time
:
startTime
,
end_time
:
endTime
,
key_list
:
[
collection
]
}
const
path
=
`
${
BASE_URL
}
/datastreams/find_entries/timestamp`
const
results
=
await
request
(
path
,
{
method
:
'
POST
'
,
body
})
return
excludeFirst
?
results
.
phone_data
.
slice
(
1
)
:
results
.
phone_data
}
/**
* Get the full trips for a day
*
* @param {string} token - The user token
* @param {string} day - The day on a YYYY-MM-DD format
* @returns {object} - The full trips for this day
*/
const
getTripsForDay
=
async
function
(
token
,
day
)
{
const
path
=
`
${
BASE_URL
}
/timeline/getTrips/
${
day
}
`
const
body
=
{
user
:
token
}
const
trips
=
await
request
(
path
,
{
method
:
'
POST
'
,
body
})
return
trips
.
timeline
}
module
.
exports
=
{
getServerCollectionFromDate
,
getTripsForDay
,
getFirstAndLastTripTimestamp
}
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment