Refactoring du bloc Node.js (jobs crawl + analysis)

.gitlab-ci.yml

@@ -21,9 +21,10 @@ crawl:
   image: registry.gitlab.com/gitlab-ci-utils/docker-puppeteer:node-18.18.2-bullseye-slim
   cache:
     paths:
-      - .npm/
+      - analysis/.npm/
   script:
     - mkdir -p target
+    - cd analysis
     - npm ci --cache .npm --prefer-offline
     - npm run crawl
   artifacts:
@@ -39,9 +40,10 @@ analysis:
   image: registry.gitlab.com/gitlab-ci-utils/docker-puppeteer:node-18.18.2-bullseye-slim
   cache:
     paths:
-      - .npm/
+      - analysis/.npm/
   script:
     - mkdir -p target
+    - cd analysis
     - npm ci --cache .npm --prefer-offline
     - npm run start
   artifacts:
@@ -57,9 +59,10 @@ analysis_polpublique1:
   image: registry.gitlab.com/gitlab-ci-utils/docker-puppeteer:node-18.18.2-bullseye-slim
   cache:
     paths:
-      - .npm/
+      - analysis/.npm/
   script:
     - mkdir -p target
+    - cd analysis
     - npm ci --cache .npm --prefer-offline
     - NUM=1 npm run polpublique
   artifacts:
@@ -75,9 +78,10 @@ analysis_polpublique2:
   image: registry.gitlab.com/gitlab-ci-utils/docker-puppeteer:node-18.18.2-bullseye-slim
   cache:
     paths:
-      - .npm/
+      - analysis/.npm/
   script:
     - mkdir -p target
+    - cd analysis
     - npm ci --cache .npm --prefer-offline
     - NUM=2 npm run polpublique
   artifacts:
@@ -93,9 +97,10 @@ analysis_polpublique3:
   image: registry.gitlab.com/gitlab-ci-utils/docker-puppeteer:node-18.18.2-bullseye-slim
   cache:
     paths:
-      - .npm/
+      - analysis/.npm/
   script:
     - mkdir -p target
+    - cd analysis
     - npm ci --cache .npm --prefer-offline
     - NUM=3 npm run polpublique
   artifacts:
@@ -111,9 +116,10 @@ analysis_polpublique4:
   image: registry.gitlab.com/gitlab-ci-utils/docker-puppeteer:node-18.18.2-bullseye-slim
   cache:
     paths:
-      - .npm/
+      - analysis/.npm/
   script:
     - mkdir -p target
+    - cd analysis
     - npm ci --cache .npm --prefer-offline
     - NUM=4 npm run polpublique
   artifacts:

analyser/analysis.js → analysis/analyser/analysis.js

@@ -13,9 +13,9 @@ const require = createRequire(import.meta.url);
 
 const cpt_timeout = 3;
 
-fs.access("./target", function (error) {
+fs.access("../target", function (error) {
   if (error) {
-    fs.mkdirSync('./target'); // create the directory target
+    fs.mkdirSync('../target'); // create the directory target
     console.log("Directory does not exist.") // Directory target
   }
 })
@@ -170,25 +170,25 @@ export async function newAnalysisCommun(lActu, lRdv, lDossiersPresse) {
 
   await browser.close();
 
-  csvExport('target/actu_stats.csv', actuStats);
-  csvExport('target/rv_stats.csv', rvStats);
-  imagesCsvExport('target/images_stats.csv', imageStats);
-  pdfCsvExport('target/dossiers_presse_stats.csv', dossiersPresseStats);
+  csvExport('../target/actu_stats.csv', actuStats);
+  csvExport('../target/rv_stats.csv', rvStats);
+  imagesCsvExport('../target/images_stats.csv', imageStats);
+  pdfCsvExport('../target/dossiers_presse_stats.csv', dossiersPresseStats);
 
-  csvToHtml('target/actu_stats.csv',
-    'target/actu_stats.html',
+  csvToHtml('../target/actu_stats.csv',
+    '../target/actu_stats.html',
     [4, 5], 4);
-  csvToHtml('target/rv_stats.csv',
-    'target/rv_stats.html',
+  csvToHtml('../target/rv_stats.csv',
+    '../target/rv_stats.html',
     [4, 5], 4);
   csvToHtml(
-    'target/images_stats.csv',
-    'target/images_stats.html',
+    '../target/images_stats.csv',
+    '../target/images_stats.html',
     [2],
     2,
   );
-  csvToHtml('target/dossiers_presse_stats.csv',
-    'target/dossiers_presse_stats.html',
+  csvToHtml('../target/dossiers_presse_stats.csv',
+    '../target/dossiers_presse_stats.html',
     [2],
     2,
   );
@@ -210,15 +210,15 @@ export async function newAnalysisPolPubliques(lfile, n) {
 
   await browser.close();
 
-  csvExport(`target/polp${n}_stats.csv`, polpStats);
-  imagesCsvExport(`target/polp${n}_images_stats.csv`, polp_imageStats);
+  csvExport(`../target/polp${n}_stats.csv`, polpStats);
+  imagesCsvExport(`../target/polp${n}_images_stats.csv`, polp_imageStats);
 
-  csvToHtml(`target/polp${n}_stats.csv`,
-    `target/polp${n}_stats.html`,
+  csvToHtml(`../target/polp${n}_stats.csv`,
+    `../target/polp${n}_stats.html`,
     [4, 5], 4);
   csvToHtml(
-    `target/polp${n}_images_stats.csv`,
-    `target/polp${n}_images_stats.html`,
+    `../target/polp${n}_images_stats.csv`,
+    `../target/polp${n}_images_stats.html`,
     [2],
     2,
   );
@@ -463,7 +463,7 @@ async function pageAnalysis(data, page, url) {
       }
     
       if (optionScreenshot) {
-        const path = `target/${url.substring(url.lastIndexOf('/') + 1)}.png`;
+        const path = `../target/${url.substring(url.lastIndexOf('/') + 1)}.png`;
         await page.screenshot({ path, fullPage: true });
       }
     

analyser/app.js → analysis/analyser/app.js

@@ -4,27 +4,27 @@ import fs from 'fs';
 if (process.env.DEBUG) {
   newAnalysisCommun(3);
 } else {
-  fs.access("./target/actu.json", function (error) {
+  fs.access("../target/actu.json", function (error) {
     if (error) {
       console.log("File does not exist.") // Directory target
       throw error;
     }
   })
-  const pathActu = './target/actu.json'
-  fs.access("./target/rdv.json", function (error) {
+  const pathActu = '../target/actu.json'
+  fs.access("../target/rdv.json", function (error) {
     if (error) {
       console.log("File does not exist.");
       throw error;
     }
   })
-  const pathRdv = './target/rdv.json';
-  fs.access("./target/dossiers_presse.json", function (error) {
+  const pathRdv = '../target/rdv.json';
+  fs.access("../target/dossiers_presse.json", function (error) {
     if (error) {
       console.log("File does not exist.")
       throw error;
     }
   })
-  const pathDossiersPresse = './target/dossiers_presse.json';
+  const pathDossiersPresse = '../target/dossiers_presse.json';
   newAnalysisCommun(await convertJsonToList(pathActu), await convertJsonToList(pathRdv), await convertJsonToList(pathDossiersPresse));
 }
 

analyser/app_pol.js → analysis/analyser/app_pol.js

@@ -2,7 +2,7 @@ import {convertJsonToList, newAnalysisPolPubliques} from './analysis.js';
 import fs from 'fs';
 
 const NUM = process.env.NUM || 0;
-const target = './target';
+const target = '../target';
 
 /**
  * Get polp files
@@ -27,6 +27,6 @@ if (process.env.DEBUG) {
     console.log(`Veuillez indiquer le numéro du fichier à analyser (${allPolPFiles.length} fichiers disponibles)`);
     process.exit(1);
   } else {
-    newAnalysisPolPubliques(await convertJsonToList(`target/polp${NUM}.json`), NUM);
+    newAnalysisPolPubliques(await convertJsonToList(`../target/polp${NUM}.json`), NUM);
   }
 }

analyser/crawler.js → analysis/analyser/crawler.js

@@ -7,7 +7,7 @@ const BASE_URL = process.env.BASE_URL ? process.env.BASE_URL: 'https://www.ecolo
 
 
 export async function getActuFile(actus) {
-  const path_file = 'target/actu.json';
+  const path_file = '../target/actu.json';
   fs.writeFile(path_file, JSON.stringify(actus), (err) => {
     if (err) throw err;
   })
@@ -73,7 +73,7 @@ let index = 0;
 }
 
 export async function getRdvFile(rdvs) {
-  const path_file = 'target/rdv.json'
+  const path_file = '../target/rdv.json'
   fs.writeFile(path_file, JSON.stringify(rdvs), (err) => {
     if (err) throw err;
   })
@@ -108,7 +108,7 @@ export async function getPolPublicFile(pols) {
   const uniquePols = [...new Map(pols.map(pol => [pol.url, pol])).values()]
   for (let i = 0; i < uniquePols.length; i += limit) {
     let pol = uniquePols.slice(i, (i + limit));
-    fs.writeFile(`target/polp${n}.json`, JSON.stringify(pol), (err) => {
+    fs.writeFile(`../target/polp${n}.json`, JSON.stringify(pol), (err) => {
       if (err) throw err;
     });
     n++;
@@ -147,7 +147,7 @@ export async function getPolPublics(nb) {
 }
 
 export async function getDossiersPresseFile(dossiers) {
-  const path_file = 'target/dossiers_presse.json';
+  const path_file = '../target/dossiers_presse.json';
   fs.writeFile(path_file, JSON.stringify(dossiers), (err) => {
     if (err) throw err;
   })