From a51e8d7468d961fba1ded4728376b26bdf63d5bd Mon Sep 17 00:00:00 2001 From: SAFINE LAGET Anis <anis.safine@beta.gouv.fr> Date: Tue, 10 Dec 2024 16:37:39 +0000 Subject: [PATCH] =?UTF-8?q?chore(perimetres):=20corriger=20les=20p=C3=A9ri?= =?UTF-8?q?m=C3=A8tres=20qui=20ne=20sont=20pas=20valides=20(pub/pnm-public?= =?UTF-8?q?/camino!1487)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/api/package.json | 1 + packages/api/src/api/rest/etapes.ts | 2 +- .../titre-etape-heritage-props-find.test.ts | 7 + .../utils/titre-etape-heritage-props-find.ts | 27 ++- packages/api/src/scripts/check-perimetres.ts | 222 ++++++++++++++++++ 5 files changed, 249 insertions(+), 10 deletions(-) create mode 100644 packages/api/src/scripts/check-perimetres.ts diff --git a/packages/api/package.json b/packages/api/package.json index 93c7cb959..7b05be819 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -7,6 +7,7 @@ "type": "module", "scripts": { "build": "tsc --incremental", + "check-perimetres": "node --enable-source-maps --loader ts-node/esm/transpile-only ./src/scripts/check-perimetres.ts", "daily": "node --enable-source-maps --loader ts-node/esm/transpile-only ./src/scripts/daily.ts", "monthly": "node --enable-source-maps --loader ts-node/esm/transpile-only ./src/scripts/monthly.ts", "db:dump": "rm -rf ./backups/* && pg_dump --host=localhost --username=postgres --clean --if-exists --format=d --no-owner --no-privileges --dbname=camino --file=./backups/", diff --git a/packages/api/src/api/rest/etapes.ts b/packages/api/src/api/rest/etapes.ts index 1687280c9..59aab30db 100644 --- a/packages/api/src/api/rest/etapes.ts +++ b/packages/api/src/api/rest/etapes.ts @@ -315,7 +315,7 @@ type PerimetreInfos = { surface: KM2 | null } & Pick<GraphqlEtape, 'geojson4326Forages' | 'geojsonOrigineForages'> & Pick<GetGeojsonInformation, 'communes' | 'forets'> -const getPerimetreInfosInternal = ( +export const getPerimetreInfosInternal = ( pool: Pool, geojson4326Perimetre: GraphqlEtape['geojson4326Perimetre'], geojsonOriginePerimetre: GraphqlEtape['geojsonOriginePerimetre'], diff --git a/packages/api/src/business/utils/titre-etape-heritage-props-find.test.ts b/packages/api/src/business/utils/titre-etape-heritage-props-find.test.ts index 9f2aaf3a1..be4ae2eb6 100644 --- a/packages/api/src/business/utils/titre-etape-heritage-props-find.test.ts +++ b/packages/api/src/business/utils/titre-etape-heritage-props-find.test.ts @@ -204,6 +204,13 @@ describe('retourne l’étape en fonction de son héritage', () => { test('l’étape n’est pas modifiée si pas de changement sur le perimetre', () => { const titreEtapePrecedente = { id: 'titreEtapePrecedenteId', + geojson4326Forages: undefined, + geojson4326Points: undefined, + geojsonOriginePoints: undefined, + geojsonOriginePerimetre: undefined, + geojsonOrigineGeoSystemeId: undefined, + geojsonOrigineForages: undefined, + surface: 0, geojson4326Perimetre: { type: 'Feature', properties: {}, geometry: { type: 'MultiPolygon', coordinates: [[[[1, 2]]]] } }, heritageProps: ETAPE_HERITAGE_PROPS.reduce((acc, prop) => { acc[prop] = { actif: false, etapeId: null } diff --git a/packages/api/src/business/utils/titre-etape-heritage-props-find.ts b/packages/api/src/business/utils/titre-etape-heritage-props-find.ts index f1c888542..d708f264b 100644 --- a/packages/api/src/business/utils/titre-etape-heritage-props-find.ts +++ b/packages/api/src/business/utils/titre-etape-heritage-props-find.ts @@ -28,7 +28,7 @@ type IPropValueArray = undefined | null | IEntreprise[] | SubstanceLegaleId[] type IPropValue = number | string | IPropValueArray | FeatureMultiPolygon -const titreEtapePropCheck = (propId: string, oldValue?: IPropValue | null, newValue?: IPropValue | null) => { +const titreEtapePropCheck = (propId: string, oldValue?: IPropValue | null, newValue?: IPropValue | null): boolean => { if (['titulaires', 'amodiataires', 'substances'].includes(propId)) { return propertyArrayCheck(oldValue as IPropValueArray, newValue as IPropValueArray, propId) } @@ -78,26 +78,35 @@ export const titreEtapeHeritagePropsFind = ( const etapeId = prevHeritage?.etapeId && prevHeritage.actif ? prevHeritage.etapeId : prevTitreEtape?.id - const getEtapePropId = (heritagePropId: EtapeHeritageProps): keyof ITitreEtape => { + const getEtapePropId = (heritagePropId: EtapeHeritageProps): (keyof ITitreEtape)[] => { if (heritagePropId === 'perimetre') { - return 'geojson4326Perimetre' + return ['geojson4326Perimetre', 'geojson4326Points', 'geojsonOrigineGeoSystemeId', 'geojsonOriginePoints', 'geojsonOriginePerimetre', 'geojsonOrigineForages', 'geojson4326Forages', 'surface'] } if (heritagePropId === 'titulaires') { - return 'titulaireIds' + return ['titulaireIds'] } if (heritagePropId === 'amodiataires') { - return 'amodiataireIds' + return ['amodiataireIds'] } - return heritagePropId + return [heritagePropId] } if (heritage.actif) { if (prevTitreEtape) { - const oldValue = titreEtape[getEtapePropId(propId)] as IPropValue | undefined | null - const newValue = prevTitreEtape[getEtapePropId(propId)] as IPropValue | undefined | null + let newValue + let propChanged = false + for (let i = 0, propIds = getEtapePropId(propId); i < propIds.length; i += 1) { + const oldValue = titreEtape[propIds[i]] as IPropValue | undefined | null + newValue = prevTitreEtape[propIds[i]] as IPropValue | undefined | null + + if (!titreEtapePropCheck(propId, oldValue, newValue)) { + propChanged = true + break + } + } - if (!titreEtapePropCheck(propId, oldValue, newValue)) { + if (propChanged) { hasChanged = true newTitreEtape = objectClone(newTitreEtape) diff --git a/packages/api/src/scripts/check-perimetres.ts b/packages/api/src/scripts/check-perimetres.ts new file mode 100644 index 000000000..7e55e22b3 --- /dev/null +++ b/packages/api/src/scripts/check-perimetres.ts @@ -0,0 +1,222 @@ +import '../init' +import pg from 'pg' +import { config } from '../config/index' +import { Effect } from 'effect' +import { knex } from '../knex' +import { EtapeBrouillon, EtapeId } from 'camino-common/src/etape' +import { CaminoError } from 'camino-common/src/zod-tools' +import { getPerimetreInfosInternal } from '../api/rest/etapes' +import { TitreTypeId } from 'camino-common/src/static/titresTypes' +import { FeatureCollectionForages, FeatureCollectionPoints, FeatureMultiPolygon, MultiPolygon } from 'camino-common/src/perimetre' +import { GeoSystemeId } from 'camino-common/src/static/geoSystemes' +import { titreEtapeUpsert } from '../database/queries/titres-etapes' +import { userSuper } from '../database/user-super' +import { DemarcheId } from 'camino-common/src/demarche' +import { CaminoDate } from 'camino-common/src/date' +import { EtapeTypeId } from 'camino-common/src/static/etapesTypes' +import { EtapeStatutId } from 'camino-common/src/static/etapesStatuts' +import { TitreId } from 'camino-common/src/validators/titres' +import { callAndExit } from '../tools/fp-tools' + +// Le pool ne doit être qu'aux entrypoints : le daily, le monthly, et l'application. +const pool = new pg.Pool({ + host: config().PGHOST, + user: config().PGUSER, + password: config().PGPASSWORD, + database: config().PGDATABASE, +}) + +type QueryError = "Échec de récupération des ids d'étapes en BDD" +type EtapeError = 'Des étapes ont des périmètres invalides' +type PerimetreValidationError = "Échec de validation d'un périmètre" +type EtapeUploadError = "Échec du réupload d'un périmètre" +type PipelineError = CaminoError<QueryError | EtapeError | PerimetreValidationError | EtapeUploadError> + +type Perimetre = { + titreId: TitreId + titreTypeId: TitreTypeId + etapeId: EtapeId + demarcheId: DemarcheId + isBrouillon: EtapeBrouillon + date: CaminoDate + etapeTypeId: EtapeTypeId + etapeStatutId: EtapeStatutId + geojson4326Perimetre: MultiPolygon + geojsonOriginePerimetre: FeatureMultiPolygon | null + geojsonOriginePoints: FeatureCollectionPoints | null + geojsonOrigineGeoSystemeId: GeoSystemeId | null + geojsonOrigineForages: FeatureCollectionForages | null +} +function getAllPerimetres(): Effect.Effect<Perimetre[], CaminoError<QueryError>, never> { + return Effect.gen(function* () { + const { rows } = yield* Effect.tryPromise({ + try: () => + knex.raw<{ rows: Perimetre[] }>(` + SELECT + "titreId", + "titreTypeId", + "etapeId", + "demarcheId", + "isBrouillon", + "date", + "etapeTypeId", + "etapeStatutId", + ST_AsGeoJSON("geojson4326_perimetre", 40)::json AS "geojson4326Perimetre", + "geojsonOriginePerimetre", + "geojsonOriginePoints", + "geojsonOrigineGeoSystemeId", + "geojsonOrigineForages" + FROM ( + SELECT DISTINCT + t.id AS "titreId", + t.type_id AS "titreTypeId", + te.id AS "etapeId", + td.id AS "demarcheId", + te.is_brouillon AS "isBrouillon", + te.date, + te.type_id AS "etapeTypeId", + te.statut_id AS "etapeStatutId", + te.geojson4326_perimetre, + te.geojson_origine_perimetre AS "geojsonOriginePerimetre", + te.geojson_origine_points AS "geojsonOriginePoints", + te.geojson_origine_geo_systeme_id AS "geojsonOrigineGeoSystemeId", + te.geojson_origine_forages AS "geojsonOrigineForages" + FROM titres_etapes te + JOIN titres_demarches td ON td.id = te.titre_demarche_id + JOIN titres t ON t.id = td.titre_id + WHERE geojson4326_perimetre IS NOT NULL AND te.archive IS FALSE + ORDER BY te.date DESC + ) t + `), + catch: error => ({ + message: "Échec de récupération des ids d'étapes en BDD" as const, + extra: { error }, + }), + }) + + return rows + }) +} + +function updatePerimetre(perimetre: Perimetre): Effect.Effect<void, CaminoError<EtapeUploadError>, never> { + return Effect.tryPromise({ + try: async () => { + if (perimetre.geojsonOrigineGeoSystemeId !== '4326') { + throw new Error('Geosysteme invalide') + } + + await titreEtapeUpsert( + { + id: perimetre.etapeId, + typeId: perimetre.etapeTypeId, + statutId: perimetre.etapeStatutId, + date: perimetre.date, + isBrouillon: perimetre.isBrouillon, + titreDemarcheId: perimetre.demarcheId, + geojson4326Perimetre: { + type: 'Feature', + geometry: perimetre.geojson4326Perimetre, + properties: {}, + }, + geojsonOriginePerimetre: { + type: 'Feature', + geometry: perimetre.geojson4326Perimetre, + properties: {}, + }, + geojsonOriginePoints: perimetre.geojsonOriginePoints, + geojsonOrigineGeoSystemeId: perimetre.geojsonOrigineGeoSystemeId, + geojsonOrigineForages: perimetre.geojsonOrigineForages, + }, + userSuper, + perimetre.titreId + ) + }, + catch: error => ({ + message: "Échec du réupload d'un périmètre" as const, + extra: { + etapeId: perimetre.etapeId, + error, + }, + }), + }) +} + +type InvalidEtape = { id: EtapeId; errors: string[] } +function getInvalidEtapes(rows: Perimetre[]): Effect.Effect<InvalidEtape[], CaminoError<PerimetreValidationError | EtapeUploadError>, never> { + return Effect.gen(function* () { + const invalidEtapes: InvalidEtape[] = [] + + for (let i = 0; i < rows.length; i += 1) { + const errors = yield* getPerimetreErrors(rows[i]) + if (errors.length > 0) { + invalidEtapes.push({ id: rows[i].etapeId, errors }) + console.error(`(${Math.round((i / rows.length) * 10000) / 100}%) ${rows[i].etapeId} : ${errors.join(', ')}\n`) + + yield* updatePerimetre(rows[i]) + } + } + + return invalidEtapes + }) +} + +function getPerimetreErrors(perimetre: Perimetre): Effect.Effect<string[], CaminoError<PerimetreValidationError>, never> { + return Effect.tryPromise({ + try: async () => { + try { + await callAndExit( + getPerimetreInfosInternal( + pool, + { type: 'Feature', geometry: perimetre.geojson4326Perimetre, properties: {} }, + perimetre.geojsonOriginePerimetre, + perimetre.geojsonOriginePoints, + perimetre.titreTypeId, + perimetre.geojsonOrigineGeoSystemeId, + perimetre.geojsonOrigineForages + ) + ) + } catch (error) { + if (error instanceof Error) { + return [error.message] + } else if (typeof error === 'string') { + return [error] + } else { + return ['Périmètre invalide (raison inconnue)'] + } + } + + return [] + }, + catch: error => ({ + message: "Échec de validation d'un périmètre" as const, + extra: { + etapeId: perimetre.etapeId, + error: error instanceof Error ? error.message : error, + }, + }), + }) +} + +const pipeline: Effect.Effect<void, PipelineError, never> = Effect.gen(function* () { + console.time('PIPELINE') + const perimetres = yield* getAllPerimetres() + const invalidEtapes = yield* getInvalidEtapes(perimetres) + console.timeEnd('PIPELINE') + + if (invalidEtapes.length > 0) { + // eslint-disable-next-line no-console + console.log(`${invalidEtapes.length} ont des périmètres invalides : ${invalidEtapes.map(({ id }) => id).join(',')}`) + yield* Effect.fail({ + message: `Des étapes ont des périmètres invalides` as const, + extra: invalidEtapes.map(({ id, errors }) => `${id} : ${errors.join(', ')}`).join('\n'), + }) + } +}) + +try { + await Effect.runPromise(pipeline) + console.info('Script terminé : aucune erreur détectée') + process.exit() +} catch (error) { + process.exit(1) +} -- GitLab