import prettyBytes from 'pretty-bytes';
import fs from 'fs';
import fetch from 'node-fetch';
import { parse } from 'csv-parse/sync';
import { stringify } from 'csv-stringify/sync';



// Define "require" for pupeeter
import { createRequire } from 'module';
import { getActu, getPolPublics, getRendezvous, getActuFile, getPolPublicFile, getRdvFile, getDossiersPresse, getDossiersPresseFile } from './crawler.js';
const require = createRequire(import.meta.url);

const cpt_timeout = 3;

fs.access("../target", function (error) {
  if (error) {
    fs.mkdirSync('../target'); // create the directory target
    console.log("Directory does not exist.") // Directory target
  }
})


const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
// eslint-disable-next-line new-cap
puppeteer.use(StealthPlugin());

const optionScroll = false;
const optionScreenshot = false;

const csvOptions = {
  delimiter: '|', // Utiliser | comme délimiteur de colonnes
  quote: '"', // Utiliser " comme séparateur de chaînes
};

// verify if target exist, if not create target
// for it to work, I had to type "mdir target" on terminal

/** Analysis Data */
class AnalysisData {
  /**
   * Empty data creation
   */
  constructor() {
    this.reset();
  }


  /**
   * Reset analysis data
   */
  reset() {
    this.dataReceived = 0;
    this.encodedDataReceived = 0;
    this.nbRequests = 0;
    this.requestsData = {};
    this.requestsUrls = new Set();
    this.domNbElements = 0;
  }
}

/**
 * On Data Received event
 * @param {any} data
 * @param {any} event
 */
async function onDataReceived(data, event) {
  if (data.requestsData[event.requestId]) {
    data.requestsData[event.requestId].size += event.dataLength;
    data.dataReceived += event.dataLength;
    data.encodedDataReceived += event.encodedDataLength;
  }
}


/**
 * Callback to init network data
 * @param {any} data
 * @param {any} event
 * @return {any}
 */
async function onSendRequest(data, event) {
  const url = event.request.url;
  if (url.startsWith('http')) {
    data.nbRequests++;
    const urlObject = new URL(url);
    data.requestsData[event.requestId] = {
      url: url,
      size: 0,
      type: '',
    };
    data.requestsUrls.add(urlObject.hostname);
  }
}

/**
 * Callback to get resource type
 * @param {any} data
 * @param {any} event
 * @return {any}
 */
async function onResponseReceived(data, event) {
  if (data.requestsData[event.requestId]) {
    data.requestsData[event.requestId].type = event.type;
  }
}

export async function convertJsonToList(jsonfile) {
  
  var data = fs.readFileSync(jsonfile,'utf8');
  let lfile = JSON.parse(data);
  return lfile;
}

export async function crawling(nb) {
  const actufile = await getActuFile(await getActu(nb));
  const rdvfile = await getRdvFile(await getRendezvous(nb));
  const polfile = await getPolPublicFile(await getPolPublics(nb));
  const dpfile = await getDossiersPresseFile(await getDossiersPresse(nb));
}

async function getBrowser() {
  const browser = await puppeteer.launch({
    headless: true,
    timeout: 90000,
    executablePath: require('puppeteer').executablePath(),
    args: ['--no-sandbox', '--disable-setuid-sandbox'],
  });
  return browser;
}

async function prepareBrowser(browser, data) {
  const page = await browser.newPage();
  await page.setDefaultNavigationTimeout(90000);
  page.setCacheEnabled(false);
  const client = await page.target().createCDPSession();
  await client.send('Network.enable');
  client.on('Network.dataReceived', onDataReceived.bind(null, data));
  client.on('Network.requestWillBeSent', onSendRequest.bind(null, data));
  client.on(
    'Network.responseReceived',
    onResponseReceived.bind(null, data),
  );

  await page.setViewport({
    width: 1200,
    height: 800,
    deviceScaleFactor: 1,
  });
  return page;
}

/**
 * Main analysis function
 * @param {*} nb
 */
export async function newAnalysisCommun(lActu, lRdv, lDossiersPresse) {
  const browser = await getBrowser();
  const data = new AnalysisData();
  const page = await prepareBrowser(browser, data);

  const imageStats = new Map();
  const actuStats = await articlesAnalysis(
    lActu, data, page, imageStats);
  const rvStats = await articlesAnalysis(
    lRdv, data, page, imageStats);
  const dossiersPresseStats = await pdfAnalysis(
    lDossiersPresse, data, page);

  await browser.close();

  csvExport('../target/actu_stats.csv', actuStats);
  csvExport('../target/rv_stats.csv', rvStats);
  imagesCsvExport('../target/images_stats.csv', imageStats);
  pdfCsvExport('../target/dossiers_presse_stats.csv', dossiersPresseStats);

  csvToHtml('../target/actu_stats.csv',
    '../target/actu_stats.html',
    [4, 5], 4);
  csvToHtml('../target/rv_stats.csv',
    '../target/rv_stats.html',
    [4, 5], 4);
  csvToHtml(
    '../target/images_stats.csv',
    '../target/images_stats.html',
    [2],
    2,
  );
  csvToHtml('../target/dossiers_presse_stats.csv',
    '../target/dossiers_presse_stats.html',
    [2],
    2,
  );
}

/**
 * Main analysis function
 * @param {*} lfile
 * @param {*} n
 */
export async function newAnalysisPolPubliques(lfile, n) {
  const browser = await getBrowser();
  const data = new AnalysisData();
  const page = await prepareBrowser(browser, data);

  const polp_imageStats = new Map();
  const polpStats = await articlesAnalysis(
    lfile, data, page, polp_imageStats);

  await browser.close();

  csvExport(`../target/polp${n}_stats.csv`, polpStats);
  imagesCsvExport(`../target/polp${n}_images_stats.csv`, polp_imageStats);

  csvToHtml(`../target/polp${n}_stats.csv`,
    `../target/polp${n}_stats.html`,
    [4, 5], 4);
  csvToHtml(
    `../target/polp${n}_images_stats.csv`,
    `../target/polp${n}_images_stats.html`,
    [2],
    2,
  );
}

/**
 * Analysis on a list of articles
 * @param {any} articles
 * @param {*} analysisData
 * @param {*} page
 * @param {*} imageStats
 */
async function articlesAnalysis(articles, analysisData, page, imageStats) {
  const stats = [];
  for (const article of articles) {
    const url = article.url;
    analysisData.reset();
    await pageAnalysis(analysisData, page, url);

    stats.push([
      article.title,
      article.date,
      url,
      analysisData.nbRequests,
      analysisData.encodedDataReceived,
      analysisData.dataReceived,
      analysisData.domNbElements,
    ]);

    displayStats(analysisData);

    for (const im of getImages(analysisData)) {
      imageStats.set(im.url, { url: url, size: im.size });
    }

    // Avoid flood detection
    await new Promise((resolve) => setTimeout(resolve, 50));
  }
  return stats;
}

/**
 * Analysis on a list of articles (pdf links isolation)
 * @param {any} articles
 * @param {*} analysisData
 * @param {*} page
 */
async function pdfAnalysis(articles, analysisData, page) {  
  const stats = [];
  analysisData.reset();
  let counter = 0;
  for (const article of articles) {
    counter++;
    const url = article.url;
    console.error(`Analyse de ${url} ...`)
    await page.goto(url);
    const pdfUrlList = await page.evaluate(() => Array.from(document.querySelectorAll('article a'), element => element.href));
    const uniquePdfUrlList = [...new Set(pdfUrlList)];
    for (const pdfUrl of uniquePdfUrlList) {
      const response = await fetch(pdfUrl);
      const contentType = response.headers.get('content-type');
      const contentLength = response.headers.get('content-length');

      if (contentType === 'application/pdf') {
        stats.push([
          pdfUrl,
          url,
          contentLength
        ])
      }

    }

    // Avoid flood detection
    await new Promise((resolve) => setTimeout(resolve, 50));
  }
  return stats;
}

/**
 * Convert CSV to HTML (DSFR)
 * @param {*} input
 * @param {*} output
 * @param {*} sizes
 * @param {*} sortCol
 */
export async function csvToHtml(input, output, sizes, sortCol) {
  const content = fs.readFileSync(input);
  const records = parse(content, csvOptions);
  const header = records.shift();
  htmlTableExport(output, header, records, sizes, sortCol);
}

/**
 * Export HTML table (DSFR)
 * @param {*} file
 * @param {*} header
 * @param {*} content
 * @param {*} sizes
 * @param {*} sortCol
 */
function htmlTableExport(file, header, content, sizes, sortCol) {
  const result = `<div class="fr-table"><table>
<thead><tr>${header
      .map((e, i) =>
        i == sortCol ? `<th aria-sort="descending">${e}</th>` : `<th>${e}</th>`,
      )
      .join('\n')}</tr></thead>
<tbody>${content
      .sort((a, b) => b[sortCol] - a[sortCol])
      .map(
        (r) =>
          `<tr>${r
            .map(
              (e, i) =>
                `<td>${sizes.includes(i) ? prettyBytes(parseInt(e)) : toUrl(e)
                }</td>`,
            )
            .join('\n')}</tr>`,
      )
      .join('\n')}</tbody>
</table>
</div>`;
  fs.writeFileSync(file, result);
}

/**
 * Convert URL to HTML (DSFR)
 * @param {*} e
 * @return {*} html code
 */
function toUrl(e) {
  if (e.startsWith('http')) {
    return `<a href="${e}" target="_blank">${e.substring(
      e.lastIndexOf('/') + 1,
    )}</a>`;
  } else {
    return e;
  }
}

/**
 * Export CSV file
 * @param {*} path
 * @param {*} stats
 */
function csvExport(path, stats) {
  rawCsvExport(
    path,
    [
      'Titre',
      'Date',
      'Url',
      'Nombre de requêtes',
      'Données transférées',
      'Données décodées',
      'Nombre d\'éléments',
    ],
    stats,
  );
}

/**
 * Export CSV file for images
 * @param {*} path
 * @param {*} stats
 */
function imagesCsvExport(path, stats) {
  const imageStatsContent = [...stats.entries()]
    .sort((a, b) => b[1].size - a[1].size)
    .map(([k, v]) => [k, v.url, v.size]);

  rawCsvExport(
    path,
    ['Image', 'Offre', 'Taille'],
    imageStatsContent,
  );
}

/**
 * Export CSV file for PDFs
 * @param {*} path
 * @param {*} stats
 */
function pdfCsvExport(path, stats) {
  rawCsvExport(
    path,
    ['PDF', 'Dossier de presse', 'Taille'],
    stats,
  );
}

/**
 * Low-level function for CSV export
 * @param {*} outputFilePath
 * @param {*} header
 * @param {*} bodyData
 */
function rawCsvExport(outputFilePath, header, bodyData) {
  const data = [header, ...bodyData];

  try {
    // Conversion synchrone des données en CSV
    const csvString = stringify(data, csvOptions);

    // Écrire le contenu CSV dans un fichier de manière synchrone
    fs.writeFileSync(outputFilePath, csvString);
    console.log('Fichier CSV exporté avec succès:', outputFilePath);
  } catch (err) {
    console.error('Erreur lors de l\'écriture du fichier CSV :', err);
  }
}

/**
 * Get images bigger than 100 ko
 * @param {any} data
 * @return {*}
 */
function getImages(data) {
  return Object.values(data.requestsData).filter(
    (r) => r.size > 100000 && r.type == 'Image',
  );
}

/**
 * Analysis of one page
 * @param {*} data
 * @param {*} page
 * @param {*} url
 */
async function pageAnalysis(data, page, url) {
  let cpt = 0;
  while (true) {
    try {
      console.log(`Analyse de ${url} ...`);
      await page.goto(url, {
        waitUntil: 'networkidle0',
      });

      if (optionScroll) {
        await autoScroll(page);
      }
    
      if (optionScreenshot) {
        const path = `../target/${url.substring(url.lastIndexOf('/') + 1)}.png`;
        await page.screenshot({ path, fullPage: true });
      }
    
      data.domNbElements = await domNbElements(page);
      break;

    }
    catch(error) {
      if (cpt < cpt_timeout) {
        cpt += 1;
        continue;
      }
      else {
        throw new Error("Timeout Error: Navigation Timeout");
        break;
      }
    }
  }
}

/**
 * Count number of DOM elements
 * @param {*} page
 * @return {*}
 */
async function domNbElements(page) {
  const rawSize = await page.evaluate(
    () => document.querySelectorAll('*').length,
  );
  const svgSize = await page.evaluate(
    () =>
      document.evaluate(
        '//*[local-name()=\'svg\']/*',
        document,
        null,
        XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE,
        null,
      ).snapshotLength,
  );
  return rawSize - svgSize;
}

/**
 * Emulate a human scrolling to the bottom of the page
 * @param {*} page
 */
async function autoScroll(page) {
  await page.evaluate(async () => {
    await new Promise((resolve, reject) => {
      let totalHeight = 0;
      const distance = 100;
      const timer = setInterval(() => {
        const scrollHeight = document.body.scrollHeight;
        window.scrollBy(0, distance);
        totalHeight += distance;

        if (totalHeight >= scrollHeight - window.innerHeight) {
          clearInterval(timer);
          resolve();
        }
      }, 400);
    });
  });
}

/**
 * Logging of page stats
 * @param {*} data
 */
function displayStats(data) {
  console.log(
    `${data.nbRequests} requêtes | ${prettyBytes(
      data.dataReceived,
    )} / ${prettyBytes(data.encodedDataReceived)} transférés | ${data.domNbElements
    } éléments DOM`,
  );
}
