Ajout de retry dans la récupération des articles (latences)

analysis/analyser/crawler.js

@@ -121,7 +121,7 @@ export async function getPolPublics(nb) {
   const url_polpubl = `${BASE_URL}/sitemap.xml?page=6`;
   const response = await fetch(url_polpubl);
   const data = await response.text();
-  const parser = await libxmljs.parseXml(data);
+  const parser = libxmljs.parseXml(data);
   const obj_urls = parser.find("//*[contains(@href, '/politiques/')]");
   for (let link = 0; link < obj_urls.length; link++) {
     temp_urls.push(obj_urls[link].getAttribute('href').value());
@@ -221,62 +221,78 @@ async function getArticles({
   url,
   nb,
 }) {
+  const cpt_timeout = 3;
+  let cpt = 0;
   const urls = [];
   if (nb <= 0) {
     return urls;
   }
-  console.error(`Fetching ${url} ...`);
-  const response = await fetch(url);
-  const data = await response.text();
-
-  const dom = new JSDOM(data);
-  const document = dom.window.document;
-
-  const xpathResult = document.evaluate(
-    xpathNode,
-    document,
-    null,
-    dom.window.XPathResult.ORDERED_NODE_ITERATOR_TYPE,
-    null,
-  );
-
-  let node = xpathResult.iterateNext();
-  while (node) {
-    let date = document.evaluate(
-      xpathDate,
-      node,
-      null,
-      dom.window.XPathResult.STRING_TYPE,
-      null,
-    ).stringValue;
-    if (dateParser) {
-      date = dateParser(date);
-    }
-    urls.push({
-      url: toAbsoluteUrl(
-        document.evaluate(
-          xpathUrl,
+  while (true) {
+    try {
+      console.error(`Fetching ${url} ...`);
+      const response = await fetch(url);
+      const data = await response.text();
+    
+      const dom = new JSDOM(data);
+      const document = dom.window.document;
+    
+      const xpathResult = document.evaluate(
+        xpathNode,
+        document,
+        null,
+        dom.window.XPathResult.ORDERED_NODE_ITERATOR_TYPE,
+        null,
+      );
+    
+      let node = xpathResult.iterateNext();
+      while (node) {
+        let date = document.evaluate(
+          xpathDate,
           node,
           null,
           dom.window.XPathResult.STRING_TYPE,
           null,
-        ).stringValue,
-      ),
-      title: document.evaluate(
-        xpathTitle,
-        node,
-        null,
-        dom.window.XPathResult.STRING_TYPE,
-        null,
-      ).stringValue,
-      date,
-    });
-    if (urls.length === nb) {
+        ).stringValue;
+        if (dateParser) {
+          date = dateParser(date);
+        }
+        urls.push({
+          url: toAbsoluteUrl(
+            document.evaluate(
+              xpathUrl,
+              node,
+              null,
+              dom.window.XPathResult.STRING_TYPE,
+              null,
+            ).stringValue,
+          ),
+          title: document.evaluate(
+            xpathTitle,
+            node,
+            null,
+            dom.window.XPathResult.STRING_TYPE,
+            null,
+          ).stringValue,
+          date,
+        });
+        if (urls.length === nb) {
+          return urls;
+        }
+        node = xpathResult.iterateNext();
+      }
       return urls;
+    } 
+    catch(error) {
+      if (cpt < cpt_timeout) {
+        cpt += 1;
+        continue;
+      }
+      else {
+        throw new Error("Timeout Error: Navigation Timeout");
+        break;
+      }
     }
-    node = xpathResult.iterateNext();
   }
-  return urls;
 }
 
 /**