Skip to content
Snippets Groups Projects
Commit 690b373e authored by Rémi PAILHAREY's avatar Rémi PAILHAREY :fork_knife_plate:
Browse files

feat: simpler fluidPrices service

parent 8a816c71
No related branches found
No related tags found
1 merge request!1215feat: fluidsPrices no longer apply prices
...@@ -13,21 +13,6 @@ import FluidPricesService from './fluidsPrices.service' ...@@ -13,21 +13,6 @@ import FluidPricesService from './fluidsPrices.service'
describe('FluidPrices service', () => { describe('FluidPrices service', () => {
const fluidPricesService = new FluidPricesService(mockClient) const fluidPricesService = new FluidPricesService(mockClient)
describe('Fluid Prices - getAllPrices', () => {
it('should getAllPrices', async () => {
const mockQueryResult: QueryResult<FluidPrice[]> = {
data: fluidPrices,
bookmark: '',
next: false,
skip: 0,
}
mockClient.query.mockResolvedValueOnce(mockQueryResult)
const prices = await fluidPricesService.getAllPrices()
expect(prices).toBe(fluidPrices)
expect(mockClient.query).toHaveBeenCalled()
})
})
describe('Fluid Prices - getPrices', () => { describe('Fluid Prices - getPrices', () => {
it('should getPrices for elec', async () => { it('should getPrices for elec', async () => {
const mockQueryResult: QueryResult<FluidPrice[]> = { const mockQueryResult: QueryResult<FluidPrice[]> = {
...@@ -80,43 +65,6 @@ describe('FluidPrices service', () => { ...@@ -80,43 +65,6 @@ describe('FluidPrices service', () => {
}) })
}) })
describe('Fluid Prices - deleteAllFluidsPrices', () => {
it('should return true when fluidsPrices stored', async () => {
const mockQueryResult: QueryResult<FluidPrice[]> = {
data: fluidPrices,
bookmark: '',
next: false,
skip: 0,
}
mockClient.query.mockResolvedValueOnce(mockQueryResult)
const result = await fluidPricesService.deleteAllFluidsPrices()
expect(mockClient.destroy).toHaveBeenCalledTimes(6)
expect(result).toBe(true)
})
it('should return true when no fluidsPrices stored', async () => {
const mockQueryResult: QueryResult<FluidPrice[]> = {
data: [],
bookmark: '',
next: false,
skip: 0,
}
mockClient.query.mockResolvedValueOnce(mockQueryResult)
const result = await fluidPricesService.deleteAllFluidsPrices()
expect(result).toBe(true)
})
it('should return false when error happened on deletion', async () => {
const mockQueryResult: QueryResult<FluidPrice[]> = {
data: fluidPrices,
bookmark: '',
next: false,
skip: 0,
}
mockClient.destroy.mockRejectedValue(new Error())
mockClient.query.mockResolvedValueOnce(mockQueryResult)
const result = await fluidPricesService.deleteAllFluidsPrices()
expect(result).toBe(false)
})
})
it('should checkIfPriceExists and return it', async () => { it('should checkIfPriceExists and return it', async () => {
const mockQueryResult: QueryResult<FluidPrice[]> = { const mockQueryResult: QueryResult<FluidPrice[]> = {
data: [fluidPrices[0]], data: [fluidPrices[0]],
......
...@@ -17,17 +17,6 @@ export default class FluidPricesService { ...@@ -17,17 +17,6 @@ export default class FluidPricesService {
this._client = _client this._client = _client
} }
/**
* Get all prices available in database
*/
public async getAllPrices(): Promise<FluidPrice[]> {
const query = Q(FLUIDSPRICES_DOCTYPE).limitBy(900)
// TODO : handle case of 1000+ entries in doctype
const { data: fluidsPrices }: QueryResult<FluidPrice[]> =
await this._client.query(query)
return fluidsPrices
}
/** /**
* Get a price according to a fluidType and a data. This method return the nearest and valid price for the given date. * Get a price according to a fluidType and a data. This method return the nearest and valid price for the given date.
*/ */
...@@ -93,27 +82,6 @@ export default class FluidPricesService { ...@@ -93,27 +82,6 @@ export default class FluidPricesService {
return fluidsPrices return fluidsPrices
} }
/**
* Delete all fluidPrices entities from the db
* @returns {boolean} - true when deleted with success
* @throws {Error}
*/
public async deleteAllFluidsPrices(): Promise<boolean> {
const fluidsPrices = await this.getAllPrices()
try {
for (const price of fluidsPrices) {
await this._client.destroy(price)
}
return true
} catch (error) {
const errorMessage = `deleteAllFluidsPrices: ${JSON.stringify(error)}`
logStack('error', errorMessage)
logApp.error(errorMessage)
Sentry.captureException(error)
return false
}
}
/** /**
* Check if a fluidPrice exists in db * Check if a fluidPrice exists in db
* @returns {Promise<FluidPrice | null>} price or null * @returns {Promise<FluidPrice | null>} price or null
......
...@@ -63,15 +63,6 @@ jest.mock('./challenge.service', () => { ...@@ -63,15 +63,6 @@ jest.mock('./challenge.service', () => {
})) }))
}) })
const mockGetAllPrices = jest.fn()
const mockDeleteAllFluidsPrices = jest.fn()
jest.mock('./fluidsPrices.service', () => {
return jest.fn(() => ({
getAllPrices: mockGetAllPrices,
deleteAllFluidsPrices: mockDeleteAllFluidsPrices,
}))
})
const mockGetAllDuelEntities = jest.fn() const mockGetAllDuelEntities = jest.fn()
const mockDeleteAllDuelEntities = jest.fn() const mockDeleteAllDuelEntities = jest.fn()
jest.mock('./duel.service', () => { jest.mock('./duel.service', () => {
......
import * as Sentry from '@sentry/react'
import { Client } from 'cozy-client' import { Client } from 'cozy-client'
import logger from 'cozy-logger' import logger from 'cozy-logger'
import { import {
EGL_DAY_DOCTYPE,
ENEDIS_DAY_DOCTYPE,
GRDF_DAY_DOCTYPE,
REMOTE_ORG_ECOLYO_AGENT_PRICES, REMOTE_ORG_ECOLYO_AGENT_PRICES,
REMOTE_ORG_ECOLYO_AGENT_PRICES_REC, REMOTE_ORG_ECOLYO_AGENT_PRICES_REC,
} from 'doctypes' } from 'doctypes'
import { FluidType, TimeStep } from 'enums' import { FluidType } from 'enums'
import { DateTime } from 'luxon' import { FluidPrice } from 'models'
import { DataloadEntity, FluidPrice, TimePeriod } from 'models'
import ConsumptionDataManager from 'services/consumption.service'
import EnvironmentService from 'services/environment.service' import EnvironmentService from 'services/environment.service'
import FluidPricesService from 'services/fluidsPrices.service' import FluidPricesService from 'services/fluidsPrices.service'
import QueryRunner from 'services/queryRunner.service'
import { runService } from './service' import { runService } from './service'
const logStack = logger.namespace('fluidPrices') const logStack = logger.namespace('fluidPrices')
...@@ -33,35 +26,19 @@ const getRemotePricesByFluid = async ( ...@@ -33,35 +26,19 @@ const getRemotePricesByFluid = async (
return prices return prices
} }
/** const updatePrices = async ({ client }: { client: Client }) => {
* If a price has been updated, set the oldest startDate of the edited price so we can redo aggregation for (const fluidType of [
*/ FluidType.ELECTRICITY,
function updateFirstEditedPrice( FluidType.WATER,
firstEditedPrice: string | null, FluidType.GAS,
remotePrice: FluidPrice ]) {
): string { logStack('info', `Updating fluid prices for ${fluidType}...`)
return firstEditedPrice === null || firstEditedPrice >= remotePrice.startDate const fps = new FluidPricesService(client)
? remotePrice.startDate
: firstEditedPrice
}
/**
* Synchro the remote prices with database and returns a date where we have to relaunch aggregation if a price has been edited in backoffice
* @returns {string | null} the oldest startDate
*/
const synchroPricesToUpdate = async (
client: Client,
fluidType: FluidType
): Promise<string | null> => {
const fps = new FluidPricesService(client)
try {
const remotePrices = await getRemotePricesByFluid(client, fluidType) const remotePrices = await getRemotePricesByFluid(client, fluidType)
let firstEditedPrice: string | null = null
for (const remotePrice of remotePrices) { for (const remotePrice of remotePrices) {
const existingPrice = await fps.checkIfPriceExists(remotePrice) const existingPrice = await fps.checkIfPriceExists(remotePrice)
if (!existingPrice) { if (!existingPrice) {
logStack('debug', `Price doesn't exist in db, creating a new price`) logStack('debug', `Price doesn't exist in db, creating a new price`)
firstEditedPrice = updateFirstEditedPrice(firstEditedPrice, remotePrice)
await fps.createPrice(remotePrice) await fps.createPrice(remotePrice)
continue continue
} }
...@@ -72,7 +49,6 @@ const synchroPricesToUpdate = async ( ...@@ -72,7 +49,6 @@ const synchroPricesToUpdate = async (
} }
logStack('debug', `Price exists in db but not up to date, updating it`) logStack('debug', `Price exists in db but not up to date, updating it`)
// If a price has been updated, set the oldest startDate of the edited price so we can redo aggregation // If a price has been updated, set the oldest startDate of the edited price so we can redo aggregation
firstEditedPrice = updateFirstEditedPrice(firstEditedPrice, remotePrice)
await fps.updatePrice(existingPrice, { await fps.updatePrice(existingPrice, {
price: remotePrice.price, price: remotePrice.price,
...@@ -80,249 +56,9 @@ const synchroPricesToUpdate = async ( ...@@ -80,249 +56,9 @@ const synchroPricesToUpdate = async (
startDate: remotePrice.startDate, startDate: remotePrice.startDate,
endDate: remotePrice.endDate, endDate: remotePrice.endDate,
}) })
} logStack('info', `Price updated`)
return firstEditedPrice
} catch (error) {
logStack('error', `Error: ${error}`)
Sentry.captureException(error)
return null
}
}
const getTimePeriod = async (
timeStep: TimeStep,
date: DateTime
): Promise<TimePeriod> => {
switch (timeStep) {
case TimeStep.HALF_AN_HOUR:
return {
startDate: date,
endDate: date.plus({ day: 1 }).startOf('day'),
}
case TimeStep.DAY:
case TimeStep.MONTH:
return {
startDate: date.startOf('month'),
endDate: date.endOf('month'),
}
case TimeStep.YEAR:
return {
startDate: date.startOf('year'),
endDate: date.endOf('year'),
}
default:
logStack('error', 'Unhandled time period')
Sentry.captureException(
JSON.stringify({ error: 'Unhandled time period' })
)
throw Error('Unhandled time period')
}
}
const aggregatePrices = async (
qr: QueryRunner,
cdm: ConsumptionDataManager,
firstDate: DateTime,
today: DateTime,
fluidType: FluidType
) => {
const timeSteps = [TimeStep.MONTH, TimeStep.YEAR]
logStack(
'debug',
`Aggregation started for fluid: ${fluidType}, from ${firstDate} `
)
for (const timeStep of timeSteps) {
let aggregationDate = DateTime.fromObject({
year: firstDate.year,
month: firstDate.month,
day: firstDate.day,
})
try {
do {
const timePeriod = await getTimePeriod(timeStep, aggregationDate)
// Get doc for aggregation
const dayDocuments = await qr.fetchFluidRawDoctype(
timePeriod,
TimeStep.DAY,
fluidType
)
const docToUpdate = await qr.fetchFluidRawDoctype(
timePeriod,
timeStep,
fluidType
)
if (docToUpdate?.data && dayDocuments?.data) {
docToUpdate.data[0].price = dayDocuments.data
.map((item: DataloadEntity) => item.price || 0)
.reduce((a: number, b: number) => a + b)
}
await cdm.saveDocs(docToUpdate?.data)
// Update date according to timestep
if (timeStep === TimeStep.YEAR) {
aggregationDate = aggregationDate.plus({ year: 1 }).startOf('month')
} else {
aggregationDate = aggregationDate.plus({ month: 1 }).startOf('month')
}
} while (aggregationDate < today)
} catch (error) {
logStack('info', `Error : ${error}`)
Sentry.captureException(error)
}
}
logStack('debug', `Aggregation done`)
}
const getDoctypeTypeByFluid = (fluidType: FluidType): string => {
if (fluidType === FluidType.ELECTRICITY) {
return ENEDIS_DAY_DOCTYPE
}
if (fluidType === FluidType.GAS) {
return GRDF_DAY_DOCTYPE
}
if (fluidType === FluidType.WATER) {
return EGL_DAY_DOCTYPE
}
logStack('error', 'Unknown FluidType')
Sentry.captureException({ error: 'Unknown FluidType Doctype' })
throw new Error()
}
const getTimeStepsByFluid = (fluidType: FluidType): TimeStep[] => {
if (fluidType === FluidType.ELECTRICITY) {
return [TimeStep.DAY, TimeStep.HALF_AN_HOUR]
}
if (fluidType === FluidType.GAS || fluidType === FluidType.WATER) {
return [TimeStep.DAY]
}
logStack('error', 'Unknown FluidType')
Sentry.captureException({ error: 'Unknown FluidType' })
throw new Error()
}
const applyPrices = async (client: Client, fluidType: FluidType) => {
// If no doctypes exists, do nothing
const fluidsPricesService = new FluidPricesService(client)
const cdm = new ConsumptionDataManager(client)
const qr = new QueryRunner(client)
// Synchro db prices with remote prices
const firstEditedPriceDate = await synchroPricesToUpdate(client, fluidType)
const firstDataDate = await cdm.fetchAllFirstDateData([fluidType])
const prices = await fluidsPricesService.getAllPrices()
if (prices.length === 0) {
logStack('info', 'No fluidesPrices data')
return
}
logStack('debug', 'fluidPrices data found')
const firstMinuteData = await cdm.getFirstDataDateFromDoctypeWithPrice(
getDoctypeTypeByFluid(fluidType)
)
// If there is data, update hourly data and daily data
if (
!firstDataDate?.[0] ||
(!firstMinuteData && firstEditedPriceDate === null)
) {
logStack('info', `No data found for fluid ${fluidType}`)
return
}
const today = DateTime.now()
const timeSteps = getTimeStepsByFluid(fluidType)
let firstDate: DateTime
if (firstMinuteData && firstEditedPriceDate) {
// If there is first data without price and a price edited, set the smallest date
const firstMinuteDataDate = DateTime.fromObject({
year: firstMinuteData.year,
month: firstMinuteData.month,
day: firstMinuteData.day,
}).setZone('utc', {
keepLocalTime: true,
})
const formattedFirstEditedPrice = DateTime.fromISO(
firstEditedPriceDate
).setZone('utc', {
keepLocalTime: true,
})
// we want to exclude the period with no data if the edited date is smaller than the first data date
firstDate = DateTime.min(
DateTime.max(formattedFirstEditedPrice, firstDataDate[0]),
firstMinuteDataDate
)
} else if (firstMinuteData) {
firstDate = DateTime.fromObject({
year: firstMinuteData.year,
month: firstMinuteData.month,
day: firstMinuteData.day,
}).setZone('utc', {
keepLocalTime: true,
})
} else if (firstEditedPriceDate) {
firstDate = DateTime.max(
DateTime.fromISO(firstEditedPriceDate).setZone('utc', {
keepLocalTime: true,
}),
firstDataDate[0]
)
} else {
firstDate = today
}
// Hourly and daily prices
for (const timeStep of timeSteps) {
let date = DateTime.local().setZone('utc', {
keepLocalTime: true,
})
Object.assign(date, firstDate)
try {
do {
const priceData = await fluidsPricesService.getPrices(fluidType, date)
const timePeriod = await getTimePeriod(timeStep, date)
const data = await qr.fetchFluidRawDoctype(
timePeriod,
timeStep,
fluidType
)
// If lastItem has a price, skip this day (in order to save perf)
const lastItem = data?.data?.[data.data.length - 1]
if (lastItem && priceData) {
// if a price has been updated in backoffice re-calculates all price from the firstEditedPriceDate
data?.data.forEach((element: DataloadEntity) => {
element.price = element.load * priceData.price
})
await cdm.saveDocs(data.data)
}
// Update date
if (timeStep === TimeStep.HALF_AN_HOUR) {
date = date.plus({ days: 1 })
} else {
date = date.plus({ month: 1 }).startOf('month')
}
} while (date < today)
} catch (error) {
logStack('error', `ERROR : ${error} `)
Sentry.captureException(error)
} }
} }
await aggregatePrices(qr, cdm, firstDate, today, fluidType)
}
const processPrices = async ({ client }: { client: Client }) => {
logStack('info', `Processing electricity data...`)
await applyPrices(client, FluidType.ELECTRICITY)
logStack('info', `Electricity data done`)
logStack('info', `Processing gas data...`)
await applyPrices(client, FluidType.GAS)
logStack('info', `Gas data done`)
logStack('info', `Processing water data...`)
await applyPrices(client, FluidType.WATER)
logStack('info', `Water data done`)
logStack('info', `processPrices done`)
} }
runService(processPrices) runService(updatePrices)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment