Changed jobs analysis to pipelines analysis

src/cli.rs

 use std::fmt::Write;
-use std::process;
+use std::{panic, process};
 use std::time::Duration;
 
 use console::style;
-use indicatif::ProgressBar;
+use dialoguer::{Confirm, Input};
+use indicatif::{ProgressBar, ProgressStyle};
 use structopt::StructOpt;
 
 use crate::{Reportable, ReportPending, ReportStatus};
@@ -58,13 +59,14 @@ fn console_report_status(buffer: &mut String, report_status: &ReportStatus, inde
     };
 }
 
-fn console_report_statuses(report_statuses: &[ReportStatus]) -> String {
+pub fn console_report_statuses(report_statuses: &[ReportStatus], initial_indent: usize) -> String {
+    eprint!("\r");
     let mut result = String::new();
     if report_statuses.is_empty() {
         return result;
     }
     let mut statuses_iter = report_statuses.iter();
-    console_report_status(&mut result, statuses_iter.next().unwrap(), 0);
+    console_report_status(&mut result, statuses_iter.next().unwrap(), initial_indent);
     for report_status in statuses_iter {
         console_report_status(&mut result, report_status, 2);
     }
@@ -72,11 +74,55 @@ fn console_report_statuses(report_statuses: &[ReportStatus]) -> String {
 }
 
 pub fn display_report_pending<T: Reportable>(report_pending: ReportPending<T>) -> T {
-    let pb = ProgressBar::new_spinner();
-    pb.enable_steady_tick(Duration::from_millis(100));
-    pb.set_message(format!(" [*] {}", &report_pending.pending_msg));
-    let result = report_pending.job.join().unwrap();
-    pb.finish_with_message(console_report_statuses(&result.report()));
-    eprint!("\r");
-    result
+    let pb;
+    let initial_indent : usize;
+    if report_pending.progress.is_some() && report_pending.total.is_some() {
+        initial_indent = 2;
+        let sty = ProgressStyle::with_template(
+            "{msg} {bar:40.cyan/blue} {pos:>7}/{len:7} ({eta})")
+            .unwrap()
+            .progress_chars("#>-");
+
+        pb = ProgressBar::new(report_pending.total.unwrap() as u64);
+        pb.set_style(sty);
+        pb.set_message(report_pending.pending_msg.to_string());
+        let rx = report_pending.progress.unwrap();
+        while let Ok(received) = rx.recv() {
+            pb.set_position(received as u64);
+        }
+    } else {
+        initial_indent = 0;
+        pb = ProgressBar::new_spinner();
+        pb.enable_steady_tick(Duration::from_millis(100));
+        pb.set_message(format!(" [*] {}", &report_pending.pending_msg));
+    }
+
+    let result_join = report_pending.job.join();
+    match result_join {
+        Ok(result) => {
+            pb.finish_with_message(console_report_statuses(&result.report(), initial_indent));
+            eprint!("\r");
+            result
+        }
+        Err(e) => panic::resume_unwind(e)
+    }
 }
+
+pub fn input_clean_artifacts() -> Option<i64> {
+    if Confirm::new().with_prompt("Delete old pipelines ?").interact().unwrap_or(false) {
+        let input: i64 = Input::new()
+            .with_prompt("From which age in days ?")
+            .default("30".into())
+            .interact_text()
+            .unwrap_or_else(|_| "0".to_string())
+            .parse()
+            .unwrap_or(0);
+        if input > 0 {
+            Some(input)
+        } else {
+            None
+        }
+    } else {
+        None
+    }
+}
\ No newline at end of file

src/diagnosis.rs

 use std::sync::mpsc::Receiver;
 use std::thread::JoinHandle;
 
-pub mod artifact_size;
 pub mod gitlab_connection;
+pub mod pipeline_analysis;
+pub mod pipeline_clean;
 
 pub const STORAGE_LIMIT: u64 = 2_000_000_000;
 pub const REPO_LIMIT: u64 = 100_000_000;
@@ -12,6 +13,9 @@ pub const ARTIFACT_JOBS_DAYS_LIMIT: i64 = 30;
 pub const PACKAGE_REGISTRY_LIMIT: u64 = 1_000_000_000;
 pub const DOCKER_REGISTRY_LIMIT: u64 = 5_000_000_000;
 
+pub const GITLAB_403_ERROR: &str = "403 Forbidden";
+pub const GITLAB_SCOPE_ERROR: &str = "insufficient_scope";
+
 #[derive(Clone)]
 pub enum ReportStatus {
     OK(String),
@@ -23,7 +27,8 @@ pub enum ReportStatus {
 pub struct ReportPending<T> {
     pub pending_msg: String,
     pub job: JoinHandle<T>,
-    pub progress: Option<Receiver<u64>>,
+    pub progress: Option<Receiver<usize>>,
+    pub total: Option<usize>,
 }
 
 pub trait ReportJob {

src/diagnosis/artifact_size.rs

-use chrono::{DateTime, Duration, Local};
-use gitlab::api::{Pagination, projects, Query};
-use gitlab::api::paged;
-use gitlab::Gitlab;
-use human_bytes::human_bytes;
-use serde::Deserialize;
-
-use crate::{Reportable, ReportJob, ReportPending};
-use crate::diagnosis::{
-    ARTIFACT_JOBS_DAYS_LIMIT, ReportStatus,
-};
-use crate::diagnosis::gitlab_connection::{GitlabRepository, Project};
-
-#[derive(Debug, Deserialize)]
-pub struct Artifact {
-    pub size: u64,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct GitlabJob {
-    pub created_at: DateTime<Local>,
-    pub artifacts: Vec<Artifact>,
-}
-
-pub struct ArtifactSizeJob {
-    pub gitlab: Gitlab,
-    pub project: Project,
-}
-
-pub struct ArtifactReport {
-    pub gitlab_jobs: Vec<GitlabJob>,
-    pub report_status: Vec<ReportStatus>,
-    pub bytes_savable: u64,
-}
-
-impl Reportable for ArtifactReport {
-    fn report(&self) -> Vec<ReportStatus> {
-        self.report_status.clone()
-    }
-}
-
-impl ReportJob for ArtifactSizeJob {
-    type Diagnosis = ArtifactReport;
-
-    fn diagnose(self) -> ReportPending<Self::Diagnosis> {
-        ReportPending::<Self::Diagnosis> {
-            pending_msg: "Analysing Gitlab jobs...".to_string(),
-            job: {
-                std::thread::spawn(move || {
-                    if !self.project.jobs_enabled {
-                        return ArtifactReport {
-                            report_status: vec![
-                                ReportStatus::NA("No CI/CD configured on this project".to_string())],
-                            gitlab_jobs: vec![],
-                            bytes_savable: 0,
-                        };
-                    }
-                    let endpoint = projects::jobs::Jobs::builder()
-                        .project(self.project.id)
-                        .build()
-                        .unwrap();
-                    let jobs: Vec<GitlabJob> = paged(endpoint, Pagination::All).query(&self.gitlab).unwrap();
-                    let (report, bytes_savable) = ArtifactSizeJob::_number_jobs(&jobs);
-                    ArtifactReport {
-                        report_status: vec![report],
-                        gitlab_jobs: jobs,
-                        bytes_savable,
-                    }
-                })
-            },
-            progress: None,
-        }
-    }
-}
-
-impl ArtifactSizeJob {
-    pub fn from(gitlab: &GitlabRepository) -> ArtifactSizeJob {
-        ArtifactSizeJob {
-            gitlab: gitlab.gitlab.clone(),
-            project: gitlab.project.clone(),
-        }
-    }
-
-    fn _number_jobs(jobs: &[GitlabJob]) -> (ReportStatus, u64) {
-        let ref_date = Local::now() - Duration::days(ARTIFACT_JOBS_DAYS_LIMIT);
-        let mut old_count: usize = 0;
-        let mut old_size: u64 = 0;
-        for job in jobs.iter() {
-            let artifact_size: u64 = job.artifacts.iter().map(|a| a.size).sum();
-            if job.created_at.le(&ref_date) {
-                old_count += 1;
-                old_size += artifact_size;
-            }
-        }
-        (ReportStatus::NA(format!(
-            "{} jobs ({}) are older than {} days",
-            old_count,
-            human_bytes(old_size as f64),
-            ARTIFACT_JOBS_DAYS_LIMIT
-        )), old_size)
-    }
-}

src/diagnosis/gitlab_connection.rs

@@ -61,6 +61,7 @@ impl ReportJob for ConnectionJob {
                 })
             },
             progress: None,
+            total: None
         }
     }
 }

src/diagnosis/pipeline_analysis.rs

+use chrono::{DateTime, Duration, Local};
+use gitlab::api::{Pagination, Query};
+use gitlab::Gitlab;
+use serde::Deserialize;
+
+use crate::{Reportable, ReportJob, ReportPending, ReportStatus};
+use crate::diagnosis::ARTIFACT_JOBS_DAYS_LIMIT;
+use crate::diagnosis::gitlab_connection::{GitlabRepository, Project};
+
+#[derive(Debug, Deserialize)]
+pub struct GitlabPipeline {
+    pub id: u64,
+    pub created_at: DateTime<Local>,
+}
+
+pub struct PipelineAnalysisJob {
+    pub gitlab: Gitlab,
+    pub project: Project,
+}
+
+pub struct PipelineAnalysisReport {
+    pub gitlab: Gitlab,
+    pub project: Project,
+    pub pipelines: Vec<GitlabPipeline>,
+    pub report_status: Vec<ReportStatus>,
+}
+
+impl Reportable for PipelineAnalysisReport {
+    fn report(&self) -> Vec<ReportStatus> {
+        self.report_status.clone()
+    }
+}
+
+impl PipelineAnalysisJob {
+    fn to_report(self, report_status: Vec<ReportStatus>, pipelines: Vec<GitlabPipeline>)
+        -> PipelineAnalysisReport {
+        PipelineAnalysisReport {
+            gitlab: self.gitlab,
+            project: self.project,
+            pipelines,
+            report_status
+        }
+    }
+}
+impl ReportJob for PipelineAnalysisJob {
+    type Diagnosis = PipelineAnalysisReport;
+
+    fn diagnose(self) -> ReportPending<Self::Diagnosis> {
+        ReportPending::<Self::Diagnosis> {
+            pending_msg: "Analysis of pipelines...".to_string(),
+            job: std::thread::spawn(move || {
+                if !self.project.jobs_enabled {
+                    return self.to_report(
+                        vec![ReportStatus::NA("No CI/CD configured on this project".to_string())],
+                        vec![]);
+                }
+
+                let endpoint = gitlab::api::projects::pipelines::Pipelines::builder()
+                    .project(self.project.id)
+                    .build()
+                    .unwrap();
+                let query: Result<Vec<GitlabPipeline>, _> =
+                    gitlab::api::paged(endpoint, Pagination::All).query(&self.gitlab);
+                match query {
+                    Err(e) => {
+                        self.to_report(
+                            vec![ReportStatus::ERROR(format!("Error : {}", e.to_string()))],
+                            vec![]
+                        )
+                    }
+                    Ok(mut pipelines) => {
+                        let ref_date = Local::now() - Duration::days(ARTIFACT_JOBS_DAYS_LIMIT);
+                        pipelines.sort_by(|a, b| a.created_at.partial_cmp(&b.created_at).unwrap());
+                        self.to_report(
+                            vec![ReportStatus::NA(
+                                format!("{} pipelines. {} pipelines are older than {} days",
+                                    pipelines.len(),
+                                    pipelines.iter()
+                                        .position(|e| e.created_at > ref_date)
+                                        .unwrap_or(pipelines.len()),
+                                    ARTIFACT_JOBS_DAYS_LIMIT))],
+                                    pipelines)
+                    }
+                }
+            }),
+            progress: None,
+            total: None
+        }
+    }
+}
+
+impl PipelineAnalysisJob {
+    pub fn from(gitlab: &GitlabRepository) -> PipelineAnalysisJob {
+        PipelineAnalysisJob {
+            gitlab: gitlab.gitlab.clone(),
+            project: gitlab.project.clone(),
+        }
+    }
+}
\ No newline at end of file

src/diagnosis/pipeline_clean.rs

+use std::cmp::max;
+use std::sync::mpsc;
+
+use chrono::{Duration, Local};
+use gitlab::api::{ApiError, Query};
+use gitlab::Gitlab;
+use human_bytes::human_bytes;
+
+use crate::{Reportable, ReportPending, ReportStatus};
+use crate::diagnosis::{GITLAB_SCOPE_ERROR, RemedyJob};
+use crate::diagnosis::gitlab_connection::Project;
+use crate::diagnosis::pipeline_analysis::{GitlabPipeline, PipelineAnalysisReport};
+
+pub struct PipelineCleanJob {
+    pub pipeline_report: PipelineAnalysisReport,
+    pub days: i64,
+}
+
+pub struct PipelineCleanReport {
+    pub saved_bytes: u64,
+    pub deleted_pipelines: Vec<GitlabPipeline>,
+    pub report_status: Vec<ReportStatus>,
+}
+
+impl Reportable for PipelineCleanReport {
+    fn report(&self) -> Vec<ReportStatus> {
+        self.report_status.clone()
+    }
+}
+
+impl PipelineCleanReport {
+    fn fatal_error(id: u64, msg: &str) -> Self {
+        Self {
+            saved_bytes: 0,
+            deleted_pipelines: vec![],
+            report_status: vec![ReportStatus::ERROR(format!("Pipeline {} - Error : {}", id, msg))],
+        }
+    }
+}
+
+impl RemedyJob for PipelineCleanJob {
+    type Report = PipelineCleanReport;
+
+    fn remedy(self) -> ReportPending<Self::Report> {
+        let (tx, rx) = mpsc::channel();
+        let ref_date = Local::now() - Duration::days(self.days);
+        let count = self.pipeline_report.pipelines.iter()
+            .filter(|a| a.created_at <= ref_date)
+            .count();
+        ReportPending {
+            pending_msg: "Deleting old pipelines".to_string(),
+            job: std::thread::spawn(move || {
+                let mut deleted_pipelines = vec![];
+
+                for (i, pipeline) in self.pipeline_report.pipelines.into_iter().enumerate() {
+                    if pipeline.created_at > ref_date {
+                        break;
+                    }
+                    let mut retry = 0;
+                    loop {
+                        let endpoint = gitlab::api::projects::pipelines::DeletePipeline::builder()
+                            .project(self.pipeline_report.project.id)
+                            .pipeline(pipeline.id)
+                            .build()
+                            .unwrap();
+                        let query = gitlab::api::ignore(endpoint)
+                            .query(&self.pipeline_report.gitlab);
+                        match query {
+                            Ok(_) => {
+                                deleted_pipelines.push(pipeline);
+                                break;
+                            }
+                            Err(e) => {
+                                match e {
+                                    ApiError::Gitlab { msg } => {
+                                        return match msg.as_str() {
+                                            msg if msg.contains(GITLAB_SCOPE_ERROR) => {
+                                                PipelineCleanReport::fatal_error(
+                                                    pipeline.id,
+                                                    "Your token has insufficient privileges to delete pipelines")
+                                            }
+                                            other => {
+                                                PipelineCleanReport::fatal_error(
+                                                    pipeline.id,
+                                                    other)
+                                            }
+                                        }
+                                    }
+                                    ApiError::Client {source} => {
+                                        retry += 1;
+                                        if retry >= 3 {
+                                            return PipelineCleanReport::fatal_error(
+                                                pipeline.id,
+                                                source.to_string().as_str());
+                                        }
+                                    }
+                                    _ => {
+                                        return PipelineCleanReport::fatal_error(
+                                            pipeline.id,
+                                            e.to_string().as_str());
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    let _ = tx.send(i);
+                }
+                let saved_bytes = PipelineCleanJob::_compute_saved_bytes(
+                    &self.pipeline_report.gitlab,
+                    &self.pipeline_report.project);
+                PipelineCleanReport {
+                    saved_bytes,
+                    report_status: vec![ReportStatus::OK(format!("Deleted {} pipelines, {} saved.",
+                                                                 deleted_pipelines.len(),
+                                                                 human_bytes(saved_bytes as f64)))],
+                    deleted_pipelines,
+                }
+            }),
+            progress: Some(rx),
+            total: Some(count),
+        }
+    }
+}
+
+impl PipelineCleanJob {
+    pub fn from(pipeline_report: PipelineAnalysisReport, days: i64) -> Self {
+        if days < 0 {
+            panic!("Number of days must be 1 or superior")
+        }
+        Self {
+            pipeline_report,
+            days
+        }
+    }
+
+    fn _compute_saved_bytes(gitlab: &Gitlab, project: &Project) -> u64 {
+        let old_size = project.statistics.job_artifacts_size;
+        let endpoint = gitlab::api::projects::Project::builder()
+            .project(project.id)
+            .statistics(true)
+            .build()
+            .unwrap();
+
+        let new_size = endpoint.query(gitlab)
+            .map(|p: Project| p.statistics.job_artifacts_size)
+            .unwrap_or(old_size);
+        max(0, old_size - new_size)
+    }
+}
\ No newline at end of file

src/main.rs

-use std::sync::mpsc;
-use std::thread;
-use std::time::Duration;
-
-use dialoguer::Input;
-use indicatif::{ProgressBar, ProgressStyle};
 use structopt::StructOpt;
 
 use cli::Args;
 
-use crate::diagnosis::{Reportable, ReportJob, ReportPending};
-use crate::diagnosis::artifact_size::ArtifactSizeJob;
+use crate::diagnosis::{RemedyJob, Reportable, ReportJob, ReportPending};
 use crate::diagnosis::gitlab_connection::ConnectionJob;
+use crate::diagnosis::pipeline_analysis::PipelineAnalysisJob;
+use crate::diagnosis::pipeline_clean::PipelineCleanJob;
 use crate::diagnosis::ReportStatus;
 
 pub mod diagnosis;
@@ -36,7 +31,18 @@ fn main() {
     let connection_data = cli::fatal_if_none(connection.data, "Diagnosis stops here.");
 
     // Analysis of artifacts
-    let report_pending = ArtifactSizeJob::from(&connection_data).diagnose();
-    let _ = cli::display_report_pending(report_pending);
+    let report_pending = PipelineAnalysisJob::from(&connection_data).diagnose();
+    let pipeline_report = cli::display_report_pending(report_pending);
+    if !pipeline_report.pipelines.is_empty() {
+        if let Some(days) = cli::input_clean_artifacts() {
+            let report_pending = PipelineCleanJob::from(pipeline_report, days).remedy();
+            let _ = cli::display_report_pending(report_pending);
+        } else {
+            cli::console_report_statuses(
+                &[ReportStatus::WARNING("Jobs deletion cancelled".to_string())],
+                2);
+        }
+    }
+
 
 }