From 72414e4fe43c5e3f82b5afb77d9efe6d205378a9 Mon Sep 17 00:00:00 2001
From: Geoffrey Arthaud <geoffrey.arthaud@developpement-durable.gouv.fr>
Date: Fri, 26 Aug 2022 12:00:14 +0200
Subject: [PATCH] Changed jobs analysis to pipelines analysis
---
src/cli.rs | 68 ++++++++++---
src/diagnosis.rs | 9 +-
src/diagnosis/artifact_size.rs | 102 --------------------
src/diagnosis/gitlab_connection.rs | 1 +
src/diagnosis/pipeline_analysis.rs | 99 +++++++++++++++++++
src/diagnosis/pipeline_clean.rs | 149 +++++++++++++++++++++++++++++
src/main.rs | 26 +++--
7 files changed, 329 insertions(+), 125 deletions(-)
delete mode 100644 src/diagnosis/artifact_size.rs
create mode 100644 src/diagnosis/pipeline_analysis.rs
create mode 100644 src/diagnosis/pipeline_clean.rs
diff --git a/src/cli.rs b/src/cli.rs
index 9899e25..175a252 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -1,9 +1,10 @@
use std::fmt::Write;
-use std::process;
+use std::{panic, process};
use std::time::Duration;
use console::style;
-use indicatif::ProgressBar;
+use dialoguer::{Confirm, Input};
+use indicatif::{ProgressBar, ProgressStyle};
use structopt::StructOpt;
use crate::{Reportable, ReportPending, ReportStatus};
@@ -58,13 +59,14 @@ fn console_report_status(buffer: &mut String, report_status: &ReportStatus, inde
};
}
-fn console_report_statuses(report_statuses: &[ReportStatus]) -> String {
+pub fn console_report_statuses(report_statuses: &[ReportStatus], initial_indent: usize) -> String {
+ eprint!("\r");
let mut result = String::new();
if report_statuses.is_empty() {
return result;
}
let mut statuses_iter = report_statuses.iter();
- console_report_status(&mut result, statuses_iter.next().unwrap(), 0);
+ console_report_status(&mut result, statuses_iter.next().unwrap(), initial_indent);
for report_status in statuses_iter {
console_report_status(&mut result, report_status, 2);
}
@@ -72,11 +74,55 @@ fn console_report_statuses(report_statuses: &[ReportStatus]) -> String {
}
pub fn display_report_pending<T: Reportable>(report_pending: ReportPending<T>) -> T {
- let pb = ProgressBar::new_spinner();
- pb.enable_steady_tick(Duration::from_millis(100));
- pb.set_message(format!(" [*] {}", &report_pending.pending_msg));
- let result = report_pending.job.join().unwrap();
- pb.finish_with_message(console_report_statuses(&result.report()));
- eprint!("\r");
- result
+ let pb;
+ let initial_indent : usize;
+ if report_pending.progress.is_some() && report_pending.total.is_some() {
+ initial_indent = 2;
+ let sty = ProgressStyle::with_template(
+ "{msg} {bar:40.cyan/blue} {pos:>7}/{len:7} ({eta})")
+ .unwrap()
+ .progress_chars("#>-");
+
+ pb = ProgressBar::new(report_pending.total.unwrap() as u64);
+ pb.set_style(sty);
+ pb.set_message(report_pending.pending_msg.to_string());
+ let rx = report_pending.progress.unwrap();
+ while let Ok(received) = rx.recv() {
+ pb.set_position(received as u64);
+ }
+ } else {
+ initial_indent = 0;
+ pb = ProgressBar::new_spinner();
+ pb.enable_steady_tick(Duration::from_millis(100));
+ pb.set_message(format!(" [*] {}", &report_pending.pending_msg));
+ }
+
+ let result_join = report_pending.job.join();
+ match result_join {
+ Ok(result) => {
+ pb.finish_with_message(console_report_statuses(&result.report(), initial_indent));
+ eprint!("\r");
+ result
+ }
+ Err(e) => panic::resume_unwind(e)
+ }
}
+
+pub fn input_clean_artifacts() -> Option<i64> {
+ if Confirm::new().with_prompt("Delete old pipelines ?").interact().unwrap_or(false) {
+ let input: i64 = Input::new()
+ .with_prompt("From which age in days ?")
+ .default("30".into())
+ .interact_text()
+ .unwrap_or_else(|_| "0".to_string())
+ .parse()
+ .unwrap_or(0);
+ if input > 0 {
+ Some(input)
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+}
\ No newline at end of file
diff --git a/src/diagnosis.rs b/src/diagnosis.rs
index 269a3a7..fddef4e 100644
--- a/src/diagnosis.rs
+++ b/src/diagnosis.rs
@@ -1,8 +1,9 @@
use std::sync::mpsc::Receiver;
use std::thread::JoinHandle;
-pub mod artifact_size;
pub mod gitlab_connection;
+pub mod pipeline_analysis;
+pub mod pipeline_clean;
pub const STORAGE_LIMIT: u64 = 2_000_000_000;
pub const REPO_LIMIT: u64 = 100_000_000;
@@ -12,6 +13,9 @@ pub const ARTIFACT_JOBS_DAYS_LIMIT: i64 = 30;
pub const PACKAGE_REGISTRY_LIMIT: u64 = 1_000_000_000;
pub const DOCKER_REGISTRY_LIMIT: u64 = 5_000_000_000;
+pub const GITLAB_403_ERROR: &str = "403 Forbidden";
+pub const GITLAB_SCOPE_ERROR: &str = "insufficient_scope";
+
#[derive(Clone)]
pub enum ReportStatus {
OK(String),
@@ -23,7 +27,8 @@ pub enum ReportStatus {
pub struct ReportPending<T> {
pub pending_msg: String,
pub job: JoinHandle<T>,
- pub progress: Option<Receiver<u64>>,
+ pub progress: Option<Receiver<usize>>,
+ pub total: Option<usize>,
}
pub trait ReportJob {
diff --git a/src/diagnosis/artifact_size.rs b/src/diagnosis/artifact_size.rs
deleted file mode 100644
index 24ef91a..0000000
--- a/src/diagnosis/artifact_size.rs
+++ /dev/null
@@ -1,102 +0,0 @@
-use chrono::{DateTime, Duration, Local};
-use gitlab::api::{Pagination, projects, Query};
-use gitlab::api::paged;
-use gitlab::Gitlab;
-use human_bytes::human_bytes;
-use serde::Deserialize;
-
-use crate::{Reportable, ReportJob, ReportPending};
-use crate::diagnosis::{
- ARTIFACT_JOBS_DAYS_LIMIT, ReportStatus,
-};
-use crate::diagnosis::gitlab_connection::{GitlabRepository, Project};
-
-#[derive(Debug, Deserialize)]
-pub struct Artifact {
- pub size: u64,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct GitlabJob {
- pub created_at: DateTime<Local>,
- pub artifacts: Vec<Artifact>,
-}
-
-pub struct ArtifactSizeJob {
- pub gitlab: Gitlab,
- pub project: Project,
-}
-
-pub struct ArtifactReport {
- pub gitlab_jobs: Vec<GitlabJob>,
- pub report_status: Vec<ReportStatus>,
- pub bytes_savable: u64,
-}
-
-impl Reportable for ArtifactReport {
- fn report(&self) -> Vec<ReportStatus> {
- self.report_status.clone()
- }
-}
-
-impl ReportJob for ArtifactSizeJob {
- type Diagnosis = ArtifactReport;
-
- fn diagnose(self) -> ReportPending<Self::Diagnosis> {
- ReportPending::<Self::Diagnosis> {
- pending_msg: "Analysing Gitlab jobs...".to_string(),
- job: {
- std::thread::spawn(move || {
- if !self.project.jobs_enabled {
- return ArtifactReport {
- report_status: vec![
- ReportStatus::NA("No CI/CD configured on this project".to_string())],
- gitlab_jobs: vec![],
- bytes_savable: 0,
- };
- }
- let endpoint = projects::jobs::Jobs::builder()
- .project(self.project.id)
- .build()
- .unwrap();
- let jobs: Vec<GitlabJob> = paged(endpoint, Pagination::All).query(&self.gitlab).unwrap();
- let (report, bytes_savable) = ArtifactSizeJob::_number_jobs(&jobs);
- ArtifactReport {
- report_status: vec![report],
- gitlab_jobs: jobs,
- bytes_savable,
- }
- })
- },
- progress: None,
- }
- }
-}
-
-impl ArtifactSizeJob {
- pub fn from(gitlab: &GitlabRepository) -> ArtifactSizeJob {
- ArtifactSizeJob {
- gitlab: gitlab.gitlab.clone(),
- project: gitlab.project.clone(),
- }
- }
-
- fn _number_jobs(jobs: &[GitlabJob]) -> (ReportStatus, u64) {
- let ref_date = Local::now() - Duration::days(ARTIFACT_JOBS_DAYS_LIMIT);
- let mut old_count: usize = 0;
- let mut old_size: u64 = 0;
- for job in jobs.iter() {
- let artifact_size: u64 = job.artifacts.iter().map(|a| a.size).sum();
- if job.created_at.le(&ref_date) {
- old_count += 1;
- old_size += artifact_size;
- }
- }
- (ReportStatus::NA(format!(
- "{} jobs ({}) are older than {} days",
- old_count,
- human_bytes(old_size as f64),
- ARTIFACT_JOBS_DAYS_LIMIT
- )), old_size)
- }
-}
diff --git a/src/diagnosis/gitlab_connection.rs b/src/diagnosis/gitlab_connection.rs
index 6cd8f34..250ff04 100644
--- a/src/diagnosis/gitlab_connection.rs
+++ b/src/diagnosis/gitlab_connection.rs
@@ -61,6 +61,7 @@ impl ReportJob for ConnectionJob {
})
},
progress: None,
+ total: None
}
}
}
diff --git a/src/diagnosis/pipeline_analysis.rs b/src/diagnosis/pipeline_analysis.rs
new file mode 100644
index 0000000..88d0a82
--- /dev/null
+++ b/src/diagnosis/pipeline_analysis.rs
@@ -0,0 +1,99 @@
+use chrono::{DateTime, Duration, Local};
+use gitlab::api::{Pagination, Query};
+use gitlab::Gitlab;
+use serde::Deserialize;
+
+use crate::{Reportable, ReportJob, ReportPending, ReportStatus};
+use crate::diagnosis::ARTIFACT_JOBS_DAYS_LIMIT;
+use crate::diagnosis::gitlab_connection::{GitlabRepository, Project};
+
+#[derive(Debug, Deserialize)]
+pub struct GitlabPipeline {
+ pub id: u64,
+ pub created_at: DateTime<Local>,
+}
+
+pub struct PipelineAnalysisJob {
+ pub gitlab: Gitlab,
+ pub project: Project,
+}
+
+pub struct PipelineAnalysisReport {
+ pub gitlab: Gitlab,
+ pub project: Project,
+ pub pipelines: Vec<GitlabPipeline>,
+ pub report_status: Vec<ReportStatus>,
+}
+
+impl Reportable for PipelineAnalysisReport {
+ fn report(&self) -> Vec<ReportStatus> {
+ self.report_status.clone()
+ }
+}
+
+impl PipelineAnalysisJob {
+ fn to_report(self, report_status: Vec<ReportStatus>, pipelines: Vec<GitlabPipeline>)
+ -> PipelineAnalysisReport {
+ PipelineAnalysisReport {
+ gitlab: self.gitlab,
+ project: self.project,
+ pipelines,
+ report_status
+ }
+ }
+}
+impl ReportJob for PipelineAnalysisJob {
+ type Diagnosis = PipelineAnalysisReport;
+
+ fn diagnose(self) -> ReportPending<Self::Diagnosis> {
+ ReportPending::<Self::Diagnosis> {
+ pending_msg: "Analysis of pipelines...".to_string(),
+ job: std::thread::spawn(move || {
+ if !self.project.jobs_enabled {
+ return self.to_report(
+ vec![ReportStatus::NA("No CI/CD configured on this project".to_string())],
+ vec![]);
+ }
+
+ let endpoint = gitlab::api::projects::pipelines::Pipelines::builder()
+ .project(self.project.id)
+ .build()
+ .unwrap();
+ let query: Result<Vec<GitlabPipeline>, _> =
+ gitlab::api::paged(endpoint, Pagination::All).query(&self.gitlab);
+ match query {
+ Err(e) => {
+ self.to_report(
+ vec![ReportStatus::ERROR(format!("Error : {}", e.to_string()))],
+ vec![]
+ )
+ }
+ Ok(mut pipelines) => {
+ let ref_date = Local::now() - Duration::days(ARTIFACT_JOBS_DAYS_LIMIT);
+ pipelines.sort_by(|a, b| a.created_at.partial_cmp(&b.created_at).unwrap());
+ self.to_report(
+ vec![ReportStatus::NA(
+ format!("{} pipelines. {} pipelines are older than {} days",
+ pipelines.len(),
+ pipelines.iter()
+ .position(|e| e.created_at > ref_date)
+ .unwrap_or(pipelines.len()),
+ ARTIFACT_JOBS_DAYS_LIMIT))],
+ pipelines)
+ }
+ }
+ }),
+ progress: None,
+ total: None
+ }
+ }
+}
+
+impl PipelineAnalysisJob {
+ pub fn from(gitlab: &GitlabRepository) -> PipelineAnalysisJob {
+ PipelineAnalysisJob {
+ gitlab: gitlab.gitlab.clone(),
+ project: gitlab.project.clone(),
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/diagnosis/pipeline_clean.rs b/src/diagnosis/pipeline_clean.rs
new file mode 100644
index 0000000..622f9fb
--- /dev/null
+++ b/src/diagnosis/pipeline_clean.rs
@@ -0,0 +1,149 @@
+use std::cmp::max;
+use std::sync::mpsc;
+
+use chrono::{Duration, Local};
+use gitlab::api::{ApiError, Query};
+use gitlab::Gitlab;
+use human_bytes::human_bytes;
+
+use crate::{Reportable, ReportPending, ReportStatus};
+use crate::diagnosis::{GITLAB_SCOPE_ERROR, RemedyJob};
+use crate::diagnosis::gitlab_connection::Project;
+use crate::diagnosis::pipeline_analysis::{GitlabPipeline, PipelineAnalysisReport};
+
+pub struct PipelineCleanJob {
+ pub pipeline_report: PipelineAnalysisReport,
+ pub days: i64,
+}
+
+pub struct PipelineCleanReport {
+ pub saved_bytes: u64,
+ pub deleted_pipelines: Vec<GitlabPipeline>,
+ pub report_status: Vec<ReportStatus>,
+}
+
+impl Reportable for PipelineCleanReport {
+ fn report(&self) -> Vec<ReportStatus> {
+ self.report_status.clone()
+ }
+}
+
+impl PipelineCleanReport {
+ fn fatal_error(id: u64, msg: &str) -> Self {
+ Self {
+ saved_bytes: 0,
+ deleted_pipelines: vec![],
+ report_status: vec![ReportStatus::ERROR(format!("Pipeline {} - Error : {}", id, msg))],
+ }
+ }
+}
+
+impl RemedyJob for PipelineCleanJob {
+ type Report = PipelineCleanReport;
+
+ fn remedy(self) -> ReportPending<Self::Report> {
+ let (tx, rx) = mpsc::channel();
+ let ref_date = Local::now() - Duration::days(self.days);
+ let count = self.pipeline_report.pipelines.iter()
+ .filter(|a| a.created_at <= ref_date)
+ .count();
+ ReportPending {
+ pending_msg: "Deleting old pipelines".to_string(),
+ job: std::thread::spawn(move || {
+ let mut deleted_pipelines = vec![];
+
+ for (i, pipeline) in self.pipeline_report.pipelines.into_iter().enumerate() {
+ if pipeline.created_at > ref_date {
+ break;
+ }
+ let mut retry = 0;
+ loop {
+ let endpoint = gitlab::api::projects::pipelines::DeletePipeline::builder()
+ .project(self.pipeline_report.project.id)
+ .pipeline(pipeline.id)
+ .build()
+ .unwrap();
+ let query = gitlab::api::ignore(endpoint)
+ .query(&self.pipeline_report.gitlab);
+ match query {
+ Ok(_) => {
+ deleted_pipelines.push(pipeline);
+ break;
+ }
+ Err(e) => {
+ match e {
+ ApiError::Gitlab { msg } => {
+ return match msg.as_str() {
+ msg if msg.contains(GITLAB_SCOPE_ERROR) => {
+ PipelineCleanReport::fatal_error(
+ pipeline.id,
+ "Your token has insufficient privileges to delete pipelines")
+ }
+ other => {
+ PipelineCleanReport::fatal_error(
+ pipeline.id,
+ other)
+ }
+ }
+ }
+ ApiError::Client {source} => {
+ retry += 1;
+ if retry >= 3 {
+ return PipelineCleanReport::fatal_error(
+ pipeline.id,
+ source.to_string().as_str());
+ }
+ }
+ _ => {
+ return PipelineCleanReport::fatal_error(
+ pipeline.id,
+ e.to_string().as_str());
+ }
+ }
+ }
+ }
+ }
+ let _ = tx.send(i);
+ }
+ let saved_bytes = PipelineCleanJob::_compute_saved_bytes(
+ &self.pipeline_report.gitlab,
+ &self.pipeline_report.project);
+ PipelineCleanReport {
+ saved_bytes,
+ report_status: vec![ReportStatus::OK(format!("Deleted {} pipelines, {} saved.",
+ deleted_pipelines.len(),
+ human_bytes(saved_bytes as f64)))],
+ deleted_pipelines,
+ }
+ }),
+ progress: Some(rx),
+ total: Some(count),
+ }
+ }
+}
+
+impl PipelineCleanJob {
+ pub fn from(pipeline_report: PipelineAnalysisReport, days: i64) -> Self {
+ if days < 0 {
+ panic!("Number of days must be 1 or superior")
+ }
+ Self {
+ pipeline_report,
+ days
+ }
+ }
+
+ fn _compute_saved_bytes(gitlab: &Gitlab, project: &Project) -> u64 {
+ let old_size = project.statistics.job_artifacts_size;
+ let endpoint = gitlab::api::projects::Project::builder()
+ .project(project.id)
+ .statistics(true)
+ .build()
+ .unwrap();
+
+ let new_size = endpoint.query(gitlab)
+ .map(|p: Project| p.statistics.job_artifacts_size)
+ .unwrap_or(old_size);
+ max(0, old_size - new_size)
+ }
+}
\ No newline at end of file
diff --git a/src/main.rs b/src/main.rs
index 7699a9b..d4b445a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,16 +1,11 @@
-use std::sync::mpsc;
-use std::thread;
-use std::time::Duration;
-
-use dialoguer::Input;
-use indicatif::{ProgressBar, ProgressStyle};
use structopt::StructOpt;
use cli::Args;
-use crate::diagnosis::{Reportable, ReportJob, ReportPending};
-use crate::diagnosis::artifact_size::ArtifactSizeJob;
+use crate::diagnosis::{RemedyJob, Reportable, ReportJob, ReportPending};
use crate::diagnosis::gitlab_connection::ConnectionJob;
+use crate::diagnosis::pipeline_analysis::PipelineAnalysisJob;
+use crate::diagnosis::pipeline_clean::PipelineCleanJob;
use crate::diagnosis::ReportStatus;
pub mod diagnosis;
@@ -36,7 +31,18 @@ fn main() {
let connection_data = cli::fatal_if_none(connection.data, "Diagnosis stops here.");
// Analysis of artifacts
- let report_pending = ArtifactSizeJob::from(&connection_data).diagnose();
- let _ = cli::display_report_pending(report_pending);
+ let report_pending = PipelineAnalysisJob::from(&connection_data).diagnose();
+ let pipeline_report = cli::display_report_pending(report_pending);
+ if !pipeline_report.pipelines.is_empty() {
+ if let Some(days) = cli::input_clean_artifacts() {
+ let report_pending = PipelineCleanJob::from(pipeline_report, days).remedy();
+ let _ = cli::display_report_pending(report_pending);
+ } else {
+ cli::console_report_statuses(
+ &[ReportStatus::WARNING("Jobs deletion cancelled".to_string())],
+ 2);
+ }
+ }
+
}
--
GitLab