Skip to content
Snippets Groups Projects
Commit 9c60d12f authored by CANÉVET Cindy's avatar CANÉVET Cindy
Browse files

Correctifs cassure des urls affichés et filtrage des doublons d'articles

parent 7987c36c
No related branches found
No related tags found
No related merge requests found
......@@ -105,8 +105,9 @@ export async function getRendezvous(nbLinks) {
export async function getPolPublicFile(pols) {
let n = 1;
const limit = 250;
for (let i = 0; i < pols.length; i += limit) {
let pol = pols.slice(i, (i + limit));
const uniquePols = [...new Map(pols.map(pol => [pol.url, pol])).values()]
for (let i = 0; i < uniquePols.length; i += limit) {
let pol = uniquePols.slice(i, (i + limit));
fs.writeFile(`target/polp${n}.json`, JSON.stringify(pol), (err) => {
if (err) throw err;
});
......@@ -114,7 +115,6 @@ export async function getPolPublicFile(pols) {
}
return true;
}
export async function getPolPublics(nb) {
let temp_urls = [];
const all_urls = [];
......@@ -190,6 +190,18 @@ async function getArticles({
if (dateParser) {
date = dateParser(date);
}
const test = toAbsoluteUrl(
document.evaluate(
xpathUrl,
node,
null,
dom.window.XPathResult.STRING_TYPE,
null,
).stringValue,
);
if (test === 'https://www.ecologie.gouv.fr/achats-publics-durables') {
console.log(url)
}
urls.push({
url: toAbsoluteUrl(
document.evaluate(
......
table tr td:first-child a {
word-break: break-word;
}
\ No newline at end of file
......@@ -32,6 +32,7 @@ extra_javascript:
- https://cdn.jsdelivr.net/npm/vega-lite@5
- https://cdn.jsdelivr.net/npm/vega-embed@6
extra_css:
- css/main.css
- css/tablesort.css
theme:
name: null
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment