diff --git a/src/configuration/config.ts b/src/configuration/config.ts
index 02caab30b42e7fec4430b2da891c8f79e60b6653..a0183191d28db8cb058061f6df19533c975d2f7d 100644
--- a/src/configuration/config.ts
+++ b/src/configuration/config.ts
@@ -25,6 +25,10 @@ export const config = {
ejs: 'adminStructureCreate.ejs',
json: 'adminStructureCreate.json',
},
+ adminStructureImport: {
+ ejs: 'adminStructureImport.ejs',
+ json: 'adminStructureImport.json',
+ },
adminUserCreate: {
ejs: 'adminUserCreate.ejs',
json: 'adminUserCreate.json',
diff --git a/src/mailer/mail-templates/adminStructureImport.ejs b/src/mailer/mail-templates/adminStructureImport.ejs
new file mode 100644
index 0000000000000000000000000000000000000000..e08c4612b9ae8e15bcac713678f160ea459e06a5
--- /dev/null
+++ b/src/mailer/mail-templates/adminStructureImport.ejs
@@ -0,0 +1,6 @@
+Bonjour,<br />
+<br />
+De nouvelles structures ont été importées:
+<a href="<%= config.protocol %>://<%= config.host %><%= config.port ? ':' + config.port : '' %>/admin/structure-list"
+ >cliquez ici pour les consulter.</a
+>
diff --git a/src/mailer/mail-templates/adminStructureImport.json b/src/mailer/mail-templates/adminStructureImport.json
new file mode 100644
index 0000000000000000000000000000000000000000..d5bec7768862a94dadebc2d9aaa63ca7a5872751
--- /dev/null
+++ b/src/mailer/mail-templates/adminStructureImport.json
@@ -0,0 +1,3 @@
+{
+ "subject": "Nouvelles structures importées"
+}
diff --git a/src/structures/services/structures-import.service.spec.ts b/src/structures/services/structures-import.service.spec.ts
index 28ef5f1bb6a542cfb7b77466be5f959908a8e2f1..60d5d6e736730d45e3fbc61bcde5506aa914cfd7 100644
--- a/src/structures/services/structures-import.service.spec.ts
+++ b/src/structures/services/structures-import.service.spec.ts
@@ -1,15 +1,21 @@
-import { HttpService } from '@nestjs/axios';
import { getModelToken } from '@nestjs/mongoose';
import { Test, TestingModule } from '@nestjs/testing';
-import { AxiosResponse } from 'axios';
import { of } from 'rxjs';
-import { mockGouvStructure, mockGouvStructureToResinFormat } from '../../../test/mock/data/gouvStructures.mock.data';
import { DataGouvStructure } from '../interfaces/data-gouv-structure.interface';
+import { HttpService } from '@nestjs/axios';
+import { AxiosResponse } from 'axios';
+import { mockGouvStructure, mockGouvStructureToResinFormat } from '../../../test/mock/data/gouvStructures.mock.data';
+import { StructuresSearchService } from './structures-search.service';
+import { StructuresService } from './structures.service';
+import { MailerService } from '../../mailer/mailer.service';
import { StructuresImportService } from './structures-import.service';
describe('StructuresImportService', () => {
let structuresImportService: StructuresImportService;
let httpService: HttpService;
+ let structuresSearchService: StructuresSearchService;
+ let structuresService: StructuresService;
+ let mailerService: MailerService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
@@ -21,6 +27,29 @@ describe('StructuresImportService', () => {
get: jest.fn(),
},
},
+ {
+ provide: StructuresSearchService,
+ useValue: {
+ indexStructure: jest.fn().mockResolvedValue(undefined),
+ },
+ },
+ {
+ provide: StructuresService,
+ useValue: {
+ updateDenormalizedFields: jest.fn().mockResolvedValue(undefined),
+ sendAdminNotificationAfterImportingStructure: jest.fn().mockImplementation(() => of(undefined)),
+ },
+ },
+ {
+ provide: MailerService,
+ useValue: {
+ config: {
+ templates: {
+ adminStructureImport: { ejs: null, json: null },
+ },
+ },
+ },
+ },
{
provide: getModelToken('Structure'),
useValue: {
@@ -34,6 +63,9 @@ describe('StructuresImportService', () => {
structuresImportService = module.get<StructuresImportService>(StructuresImportService);
httpService = module.get<HttpService>(HttpService);
+ structuresSearchService = module.get<StructuresSearchService>(StructuresSearchService);
+ structuresService = module.get<StructuresService>(StructuresService);
+ mailerService = module.get<MailerService>(MailerService);
});
describe('importDataGouvStructures', () => {
@@ -45,7 +77,10 @@ describe('StructuresImportService', () => {
headers: {},
config: {},
};
- const saveSpy = jest.fn();
+
+ const saveSpy = jest
+ .fn()
+ .mockResolvedValue({ id: 'b145231d-7388-4444-a54e-138298a4fff9|res-in-606715119f33ab0013a1cbad' });
const mockStructureModel = jest.fn().mockImplementation(() => ({ save: saveSpy }));
(structuresImportService as any).structureModel = mockStructureModel;
@@ -57,6 +92,10 @@ describe('StructuresImportService', () => {
.mockResolvedValueOnce(false);
jest.spyOn(httpService, 'get').mockImplementationOnce(() => of(mockedAxiosResponse));
+ jest.spyOn(structuresSearchService, 'indexStructure').mockResolvedValueOnce(undefined);
+
+ jest.spyOn(structuresService, 'updateDenormalizedFields').mockResolvedValueOnce(undefined);
+
await structuresImportService.importDataGouvStructures();
expect(httpService.get).toBeCalled();
@@ -72,4 +111,83 @@ describe('StructuresImportService', () => {
expect(result).toEqual(mockGouvStructureToResinFormat);
});
});
+
+ describe('convertFromOSMFormat', () => {
+ it('should convert simple weekday format', () => {
+ const osmFormat = 'Mo-Fr 08:30-12:30';
+ const result = structuresImportService['convertFromOSMFormat'](osmFormat);
+
+ expect(result.monday.open).toBe(true);
+ expect(result.monday.time).toEqual([{ opening: '08:30', closing: '12:30' }]);
+ expect(result.tuesday.time).toEqual([{ opening: '08:30', closing: '12:30' }]);
+ expect(result.wednesday.time).toEqual([{ opening: '08:30', closing: '12:30' }]);
+ expect(result.thursday.time).toEqual([{ opening: '08:30', closing: '12:30' }]);
+ expect(result.friday.time).toEqual([{ opening: '08:30', closing: '12:30' }]);
+ expect(result.saturday.open).toBe(false);
+ expect(result.sunday.open).toBe(false);
+ });
+
+ it('should handle a day marked as off', () => {
+ const osmFormat = 'Mo-Fr 08:30-12:30; We off';
+ const result = structuresImportService['convertFromOSMFormat'](osmFormat);
+
+ expect(result.wednesday.open).toBe(false);
+ });
+
+ it('should handle days marked as off', () => {
+ const osmFormat = 'Mo-Fr 08:30-12:30; We,Th off';
+ const result = structuresImportService['convertFromOSMFormat'](osmFormat);
+
+ expect(result.wednesday.open).toBe(false);
+ expect(result.thursday.open).toBe(false);
+ });
+
+ it('should handle multiple time ranges', () => {
+ const osmFormat = 'Mo-Fr 08:30-12:30,13:30-17:30';
+ const result = structuresImportService['convertFromOSMFormat'](osmFormat);
+
+ expect(result.monday.time).toEqual([
+ { opening: '08:30', closing: '12:30' },
+ { opening: '13:30', closing: '17:30' },
+ ]);
+ });
+
+ it('should handle individual days with multiple time ranges', () => {
+ const osmFormat = 'Mo,We 08:30-12:30,13:30-17:30';
+ const result = structuresImportService['convertFromOSMFormat'](osmFormat);
+
+ expect(result.monday.time).toEqual([
+ { opening: '08:30', closing: '12:30' },
+ { opening: '13:30', closing: '17:30' },
+ ]);
+ expect(result.wednesday.time).toEqual([
+ { opening: '08:30', closing: '12:30' },
+ { opening: '13:30', closing: '17:30' },
+ ]);
+ });
+
+ it('should handle complex OSM format', () => {
+ const osmFormat = 'Mo-Fr 08:30-12:30,13:30-17:30; We,Th off';
+ const result = structuresImportService['convertFromOSMFormat'](osmFormat);
+
+ expect(result.monday.time).toEqual([
+ { opening: '08:30', closing: '12:30' },
+ { opening: '13:30', closing: '17:30' },
+ ]);
+ expect(result.wednesday.open).toBe(false);
+ expect(result.thursday.open).toBe(false);
+ });
+
+ it('should handle a very complex OSM format', () => {
+ const osmFormat = 'Mo-Sa 08:30-11:45; Fr 08:30-11:45,14:30-16:45; We off ';
+ const result = structuresImportService['convertFromOSMFormat'](osmFormat);
+
+ expect(result.monday.time).toEqual([{ opening: '08:30', closing: '11:45' }]);
+ expect(result.wednesday.open).toBe(false);
+ expect(result.friday.time).toEqual([
+ { opening: '08:30', closing: '11:45' },
+ { opening: '14:30', closing: '16:45' },
+ ]);
+ });
+ });
});
diff --git a/src/structures/services/structures-import.service.ts b/src/structures/services/structures-import.service.ts
index 65f794483fce47d7126f0528fbcbf26a67e00625..13290a2eec926c5a4be44d3007d826a3a72bc122 100644
--- a/src/structures/services/structures-import.service.ts
+++ b/src/structures/services/structures-import.service.ts
@@ -1,7 +1,7 @@
import { Injectable, Logger } from '@nestjs/common';
import { StructureFormatted } from '../interfaces/structure-formatted.interface';
import { HttpService } from '@nestjs/axios';
-import { Model } from 'mongoose';
+import { Model, Types } from 'mongoose';
import { Structure, StructureDocument } from '../schemas/structure.schema';
import { InjectModel } from '@nestjs/mongoose';
import { Cron, CronExpression } from '@nestjs/schedule';
@@ -11,6 +11,9 @@ import { Week } from '../../shared/schemas/week.schema';
import { Day } from '../schemas/day.schema';
import { Time } from '../schemas/time.schema';
import { DataGouvStructure } from '../interfaces/data-gouv-structure.interface';
+import { StructuresSearchService } from './structures-search.service';
+import { StructuresService } from './structures.service';
+import { MailerService } from '../../mailer/mailer.service';
interface Condition {
address?: { street?: string; numero?: string };
contactMail?: string;
@@ -23,6 +26,9 @@ export class StructuresImportService {
private readonly logger = new Logger(StructuresImportService.name);
constructor(
private readonly httpService: HttpService,
+ private structuresSearchService: StructuresSearchService,
+ private structuresService: StructuresService,
+ private readonly mailerService: MailerService,
@InjectModel(Structure.name) private structureModel: Model<StructureDocument>
) {}
@@ -53,23 +59,50 @@ export class StructuresImportService {
}
private async processStructures(structures: DataGouvStructure[]): Promise<void> {
- let newOne = 0;
- let skipped = 0;
+ let newCount = 0;
+ let skippedCount = 0;
+ let errorCount = 0;
+
for (const structure of structures) {
- if (await this.doesAlreadyExist(structure)) {
- skipped++;
- } else {
- await this.createStructure(structure);
- newOne++;
+ try {
+ if (await this.doesAlreadyExist(structure)) {
+ skippedCount++;
+ } else {
+ await this.createStructure(structure);
+ newCount++;
+ }
+ } catch (error) {
+ errorCount++;
+ this.logger.error(
+ `Failed to process structure with ID ${structure.id || 'unknown'}: ${error.message}`,
+ error.stack
+ );
+ }
+ }
+
+ this.logger.log(
+ `Created ${newCount} new structures, skipped ${skippedCount} structures, and encountered errors in ${errorCount} structures...`
+ );
+
+ if (newCount > 0) {
+ try {
+ await this.structuresService.sendAdminNotificationAfterImportingStructure(
+ this.mailerService.config.templates.adminStructureImport.ejs,
+ this.mailerService.config.templates.adminStructureImport.json
+ );
+ } catch (notificationError) {
+ this.logger.error(`Failed to send admin notification: ${notificationError.message}`, notificationError.stack);
}
}
- this.logger.log(`Created ${newOne} new structures and skipped ${skipped} structures...`);
}
private async createStructure(structure: DataGouvStructure): Promise<void> {
const formattedStructure = await this.formatToResinSchema(structure);
const newStructure = new this.structureModel(formattedStructure);
- await newStructure.save();
+ const createdStructure = await newStructure.save();
+ // call indexStructure, updateDenormalizedFields accordingly when creating a new structure
+ await this.structuresSearchService.indexStructure(createdStructure);
+ await this.structuresService.updateDenormalizedFields(new Types.ObjectId(createdStructure._id));
}
private async doesAlreadyExist(structure: DataGouvStructure): Promise<boolean> {
@@ -118,7 +151,7 @@ export class StructuresImportService {
),
coord: [structure.longitude, structure.latitude],
description: structure.presentation_detail ?? null,
- contactPhone: structure.telephone ?? null,
+ contactPhone: structure.telephone?.replace(/^\+33/, '0') ?? null,
contactMail: structure.courriel ?? null,
hours: this.convertFromOSMFormat(structure.horaires),
accountVerified: true,
@@ -260,54 +293,73 @@ export class StructuresImportService {
week[day].time = [];
});
- if (osmHours === undefined) {
+ if (!osmHours) {
return week;
}
- const entries = osmHours.split(';');
-
+ // Normalize the osmHours string by removing any spaces after semicolons and then split it into individual time entries.
+ const entries = osmHours.replace(/;\s+/g, ';').split(';');
entries.forEach((entry) => {
- // split by ',' to get individual time ranges for each day(s)
- const timeRanges = entry.split(',');
-
- timeRanges.forEach((timeRange) => {
- // split by ' ' to separate days and time
- const [days, times] = timeRange.trim().split(' ');
-
- // check if days contains ',' then handle as individual days
- if (days.includes(',')) {
- const individualDays = days.split(',');
- individualDays.forEach((day) => {
- const startIndex = daysOfWeek.indexOf(day.toLowerCase());
- if (times) {
- const [opening, closing] = times.split('-');
- week[fullDaysOfWeek[startIndex]].open = true;
- week[fullDaysOfWeek[startIndex]].time.push({ opening, closing } as Time);
- }
- });
- } else {
- // split days by '-' to handle ranges (Tu-We)
- const [startDay, endDay] = days.toLowerCase().split('-');
+ const trimmedEntry = entry.trim();
- // find start and end indexes in the week for the range
- const startIndex = daysOfWeek.indexOf(startDay);
- let endIndex = daysOfWeek.indexOf(endDay);
+ // Check for "off" conditions first
+ if (trimmedEntry.includes('off')) {
+ const daysToClose = trimmedEntry.replace(' off', ''); // remove 'off'
- // if endDay is not provided, assume it's the same as startDay
- if (endIndex === -1) {
- endIndex = startIndex;
- }
+ if (daysToClose.includes('-')) {
+ // Range of days
+ const [startDay, endDay] = daysToClose.split('-').map((d) => d.trim());
+ const startIndex = daysOfWeek.indexOf(startDay.toLowerCase());
+ const endIndex = daysOfWeek.indexOf(endDay.toLowerCase());
- // iterate over the range of days and set the hours
- if (times) {
- const [opening, closing] = times.split('-');
- for (let i = startIndex; i <= endIndex; i++) {
- week[fullDaysOfWeek[i]].open = true;
+ for (let i = startIndex; i <= endIndex; i++) {
+ week[fullDaysOfWeek[i]].open = false;
+ week[fullDaysOfWeek[i]].time = [];
+ }
+ } else {
+ // Discrete days, split by comma
+ const splitDays = daysToClose.split(',');
+ splitDays.forEach((day) => {
+ const dayIndex = daysOfWeek.indexOf(day.trim().toLowerCase());
+ week[fullDaysOfWeek[dayIndex]].open = false;
+ week[fullDaysOfWeek[dayIndex]].time = [];
+ });
+ }
+ } else {
+ const [days, times] = trimmedEntry.split(' ');
+
+ // Handle day ranges like Mo-Fr or individual days
+ if (days.includes('-')) {
+ const [startDay, endDay] = days.split('-');
+ const startIndex = daysOfWeek.indexOf(startDay.toLowerCase());
+ const endIndex = daysOfWeek.indexOf(endDay.toLowerCase());
+
+ const timeIntervals = times.split(',');
+ for (let i = startIndex; i <= endIndex; i++) {
+ week[fullDaysOfWeek[i]].open = true;
+ timeIntervals.forEach((interval) => {
+ const [opening, closing] = interval.trim().split('-');
week[fullDaysOfWeek[i]].time.push({ opening, closing } as Time);
- }
+ });
}
+ } else {
+ // Discrete days, split by comma
+ const splitDays = days.split(',');
+ splitDays.forEach((day) => {
+ const dayIndex = daysOfWeek.indexOf(day.trim().toLowerCase());
+
+ // Clear previous timings for the day if they exist
+ week[fullDaysOfWeek[dayIndex]].time = [];
+
+ const timeSplits = times.split(',');
+ timeSplits.forEach((time) => {
+ const [opening, closing] = time.split('-');
+ week[fullDaysOfWeek[dayIndex]].open = true;
+ week[fullDaysOfWeek[dayIndex]].time.push({ opening, closing } as Time);
+ });
+ });
}
- });
+ }
});
return week;
diff --git a/src/structures/services/structures-search.service.ts b/src/structures/services/structures-search.service.ts
index 8d9b4bc9666c5e87782f29856f12c70f56bb2c10..2aedde7e642f0d0d5593ddd36fabf8335c415d4c 100644
--- a/src/structures/services/structures-search.service.ts
+++ b/src/structures/services/structures-search.service.ts
@@ -19,7 +19,7 @@ export class StructuresSearchService {
public async indexStructure(structure: StructureDocument): Promise<StructureDocument> {
this.logger.debug(`indexStructure`);
- this.elasticsearchService.index<StructureSearchBody>({
+ await this.elasticsearchService.index<StructureSearchBody>({
index: this.index,
id: structure._id,
document: this.formatIndexBody(structure),
diff --git a/src/structures/services/structures.service.ts b/src/structures/services/structures.service.ts
index 0161eefca9d6e1c9788cdcb1e0cd6976ec83fa88..e47a86538e5d5a7b98b5faf2311f92656182a1bc 100644
--- a/src/structures/services/structures.service.ts
+++ b/src/structures/services/structures.service.ts
@@ -761,6 +761,27 @@ export class StructuresService {
return structure;
}
+ public async sendAdminNotificationAfterImportingStructure(
+ templateLocation: string,
+ jsonConfigLocation: string,
+ user = null
+ ) {
+ const uniqueAdminEmails = [...new Set((await this.userService.getAdmins()).map((admin) => admin.email))].map(
+ (item) => {
+ return { email: item };
+ }
+ );
+
+ const config = this.mailerService.config;
+ const ejsPath = this.mailerService.getTemplateLocation(templateLocation);
+ const jsonConfig = this.mailerService.loadJsonConfig(jsonConfigLocation);
+ const html = await ejs.renderFile(ejsPath, {
+ config,
+ user: user,
+ });
+ this.mailerService.send(uniqueAdminEmails, jsonConfig.subject, html);
+ }
+
public async sendAdminStructureNotification(
structure: StructureDocument,
templateLocation: string,
diff --git a/src/structures/structures.controller.spec.ts b/src/structures/structures.controller.spec.ts
index 24472f46d8fcd3e30ed2a84746b7584649941af9..f58b6277f26da72f91365848f23031be21151199 100644
--- a/src/structures/structures.controller.spec.ts
+++ b/src/structures/structures.controller.spec.ts
@@ -1,31 +1,30 @@
import { HttpService } from '@nestjs/axios';
import { HttpStatus } from '@nestjs/common';
-import { getModelToken } from '@nestjs/mongoose';
import { Test, TestingModule } from '@nestjs/testing';
import { of } from 'rxjs';
import { PhotonResponseMock } from '../../test/mock/data/dataPhoton.mock.data';
import { structureDtoMock } from '../../test/mock/data/structure.mock.dto';
import { CategoriesServiceMock } from '../../test/mock/services/categories.mock.service';
-import {
- StructuresExportServiceMock,
- mockFormattedStructures,
-} from '../../test/mock/services/structures-export.mock.service';
import { UsersServiceMock } from '../../test/mock/services/user.mock.service';
import { CategoriesService } from '../categories/services/categories.service';
import { PersonalOffersService } from '../personal-offers/personal-offers.service';
import { TempUserService } from '../temp-user/temp-user.service';
import { UserRole } from '../users/enum/user-role.enum';
-import { JobDocument } from '../users/schemas/job.schema';
import { UsersService } from '../users/services/users.service';
import { CreateStructureDto } from './dto/create-structure.dto';
-import { StructureFormatted } from './interfaces/structure-formatted.interface';
-import { Structure } from './schemas/structure.schema';
-import { StructuresImportService } from './services/structures-import.service';
import { StructuresService } from './services/structures.service';
import { StructuresController } from './structures.controller';
import { mockDeletedStructure, mockStructure } from '../../test/mock/data/structures.mock.data';
import { mockUser } from '../../test/mock/data/users.mock.data';
import { StructuresExportService } from './services/structures-export.service';
+import {
+ StructuresExportServiceMock,
+ mockFormattedStructures,
+} from '../../test/mock/services/structures-export.mock.service';
+import { StructureFormatted } from './interfaces/structure-formatted.interface';
+import { JobDocument } from '../users/schemas/job.schema';
+import { getModelToken } from '@nestjs/mongoose';
+import { Structure } from './schemas/structure.schema';
describe('StructuresController', () => {
let structuresController: StructuresController;
@@ -105,7 +104,6 @@ describe('StructuresController', () => {
provide: StructuresExportService,
useClass: StructuresExportServiceMock,
},
- StructuresImportService,
{
provide: getModelToken(Structure.name),
useValue: {},
diff --git a/src/structures/structures.controller.ts b/src/structures/structures.controller.ts
index 503f4cdec7f01aca6700645e5f608cd0879f1886..c6af977f0d1e53c22ee22025cce6f0b29f61e203 100644
--- a/src/structures/structures.controller.ts
+++ b/src/structures/structures.controller.ts
@@ -43,7 +43,6 @@ import { Structure, StructureDocument } from './schemas/structure.schema';
import { StructuresExportService } from './services/structures-export.service';
import { StructuresService } from './services/structures.service';
import { StructureFormatted } from './interfaces/structure-formatted.interface';
-import { StructuresImportService } from './services/structures-import.service';
@ApiTags('structures')
@Controller('structures')
@@ -53,7 +52,6 @@ export class StructuresController {
private readonly httpService: HttpService,
private readonly structureService: StructuresService,
private readonly structuresExportService: StructuresExportService,
- private readonly structuresImportService: StructuresImportService,
private readonly userService: UsersService,
private readonly tempUserService: TempUserService,
private readonly categoriesService: CategoriesService,
@@ -130,13 +128,6 @@ export class StructuresController {
return this.structureService.updateAllDenormalizedFields();
}
- @Post('importDataGouv')
- @UseGuards(JwtAuthGuard, RolesGuard)
- @Roles('admin')
- public async importDataGouv(): Promise<void> {
- return this.structuresImportService.importDataGouvStructures();
- }
-
@Put('updateAfterOwnerVerify/:id')
public async updateAfterOwnerVerify(@Param('id') id: string): Promise<Structure> {
Logger.debug(`updateAfterOwnerVerify | structure ${id}`, StructuresController.name);
diff --git a/test/mock/data/gouvStructures.mock.data.ts b/test/mock/data/gouvStructures.mock.data.ts
index a78dc5cb570578672d7d75c1e31399810700c50e..f5a54c3a63b4ff8de6ac1f8a790c7c8ec09e5064 100644
--- a/test/mock/data/gouvStructures.mock.data.ts
+++ b/test/mock/data/gouvStructures.mock.data.ts
@@ -40,7 +40,7 @@ export const mockGouvStructureToResinFormat = new Structure({
postcode: '69290',
inseeCode: '69069',
},
- contactPhone: '+33478578285',
+ contactPhone: '0478578285',
contactMail: 'mediatheque@mairie-craponne.fr',
pmrAccess: false,
remoteAccompaniment: false,