From b4ab07aaf30aa2d3c71e68056d03892794c0399d Mon Sep 17 00:00:00 2001 From: Geoffrey Arthaud <geoffrey.arthaud@developpement-durable.gouv.fr> Date: Thu, 29 Feb 2024 22:08:56 +0100 Subject: [PATCH] Improve token handling #1 --- .gitlab-ci.yml | 8 ++-- i18n/en-GB/gitlab_project_doctor.ftl | 67 +++++++++++++-------------- i18n/fr-FR/gitlab_project_doctor.ftl | 68 ++++++++++++++-------------- src/cli.rs | 18 +++++--- src/diagnosis/gitlab_connection.rs | 49 +++++++++++++++----- src/main.rs | 18 ++++---- 6 files changed, 132 insertions(+), 96 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 122cb08..e0594c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ variables: RUST_VERSION: "1.76" # slim or alpine not adapted because of openSSL dependency TARGET_ARCH: default CARGO_HOME: .cargo + TEST_OPTIONS: '-ab .' default: cache: @@ -43,6 +44,7 @@ test-rust-current: image: rust:$RUST_VERSION script: - cargo test --verbose + - cargo run -- $TEST_OPTIONS rules: - if: $CI_COMMIT_REF_NAME == 'main' - if: $CI_PIPELINE_SOURCE == "merge_request_event" @@ -59,7 +61,7 @@ test-rust-nightly: script: - rustup target add $TARGET_ARCH - cargo build $CARGO_OPTS --target $TARGET_ARCH --release - - if [ -z "$NO_POSTPROCESS" ]; then strip $TARGET; $LDD_CMD $TARGET; $TARGET --help; fi + - if [ -z "$NO_POSTPROCESS" ]; then strip $TARGET; $LDD_CMD $TARGET; $TARGET $TEST_OPTIONS; fi - 'if [ -z "$DEBUG" ]; then curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file $TARGET "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/$TARGET_ARCH/${CI_COMMIT_TAG:-latest}/$APP_NAME"; fi' artifacts: paths: @@ -115,7 +117,7 @@ docker_build: retry: 2 needs: - release-linux - cache: [] + cache: [ ] rules: - if: $CI_COMMIT_REF_NAME == 'main' @@ -128,6 +130,6 @@ docker_build-prod: retry: 2 needs: - release-linux - cache: [] + cache: [ ] rules: - if: $CI_COMMIT_TAG \ No newline at end of file diff --git a/i18n/en-GB/gitlab_project_doctor.ftl b/i18n/en-GB/gitlab_project_doctor.ftl index bac24ce..13bbbf2 100644 --- a/i18n/en-GB/gitlab_project_doctor.ftl +++ b/i18n/en-GB/gitlab_project_doctor.ftl @@ -1,42 +1,43 @@ +ask-age-days = From which age in days ? +ask-delete-files = Delete obsolete files ? +ask-delete-pipelines = Delete old pipelines ? +conf-analysing = Analysis of package configuration +conf-fix = Fix this : {$url} connecting-to-gitlab = Connecting to Gitlab -gitlab-repo = Gitlab repository : {$repo} -error-gl-token = GL_TOKEN environment variable must contain a valid Gitlab private token -error-not-gitlab-repo = This URL is not a gitlab repository -error-not-git-repo = This dir is not a Git repository +container-analysing = Analysis of container registry +container-policy-disabled = The option "Clean up image tags" is disabled. +container-policy-enabled = The option "Clean up image tags" is enabled +container-report = {$image_count} images in container registry. {$old_image_count} are older than {$nb_days} days +container-summary = Container registry size: {$registry_size} +duplicate-assets-option-error = Cannot get the number of duplicate assets to keep option +duplicate-assets-option-onepackage = The number of duplicate assets to keep is 1 +duplicate-assets-option-warn = The number of duplicate assets to keep is NOT 1 +error = Error: +error-gl-token = No token found. Please set the GL_TOKEN variable or the command -t argument with a valid private token or run the program in a Gilab CI job +error-insufficient-privileges = Your token has insufficient privileges error-no-gitlab-remote = This dir does not contain a gitlab remote -size-storage = Storage size: -size-git-repo = Git repository size: -size-artifacts = Artifact jobs size: -size-packages = Package registry size: -package-analysing = Analysis of packages +error-not-git-repo = This dir is not a Git repository +error-not-gitlab-repo = This URL is not a gitlab repository +gitlab-repo = Gitlab repository : {$repo} +help-analysis = Analysis mode : Output a detailed analysis in JSON format. No cleaning. +help-batch = Batch mode : No questions, no progress bar, ideal for CI environment +help-days = Number of days from which an element is considered "old", 30 by default +help-git-path = Analyze the project from a local path of a Git repository. Ignored if url option is specified +help-token = (Not recommended) Gitlab private token for authentication, maintainer role is needed for objects deletion. It is strongly advised to use the environment variable GL_TOKEN instead of this argument, for security reasons. +help-url = Analyze the project from the URL of Gitlab repository no-cicd = No CI/CD configured for this project -error = Error: -package-report = {$nb_packages} packages. {$nb_files} files are obsolete ({$size}) +package-analysing = Analysis of packages +package-clean-report = Deleted {$nb_packages} packages, {$size} saved. package-deleting = Deleting obsolete packages files package-no-deletion = No package has been deleted -error-insufficient-privileges = Your token has insufficient privileges -package-clean-report = Deleted {$nb_packages} packages, {$size} saved. +package-report = {$nb_packages} packages. {$nb_files} files are obsolete ({$size}) pipeline-analysing = Analysis of pipelines -pipeline-report = {$total_pipelines} pipelines. {$old_pipelines} pipelines are older than {$nb_days} days -pipeline-deleting = Deleting old pipelines pipeline-clean-report = Deleted {$nb_pipelines} pipelines, {$size} saved. +pipeline-deleting = Deleting old pipelines pipeline-last-notdeleted = Latest pipeline is not deleted. pipeline-no-deletion = No pipeline has been deleted -ask-delete-pipelines = Delete old pipelines ? -ask-delete-files = Delete obsolete files ? -ask-age-days = From which age in days ? -help-url = Analyze the project from the URL of Gitlab repository -help-git-path = Analyze the project from a local path of a Git repository. Ignored if url option is specified -help-batch = Batch mode : No questions, no progress bar, ideal for CI environment -help-days = Number of days from which an element is considered "old", 30 by default -help-analysis = Analysis mode : Output a detailed analysis in JSON format. No cleaning. -container-policy-enabled = The option "Clean up image tags" is enabled -container-policy-disabled = The option "Clean up image tags" is disabled. -conf-analysing = Analysis of package configuration -duplicate-assets-option-onepackage = The number of duplicate assets to keep is 1 -duplicate-assets-option-warn = The number of duplicate assets to keep is NOT 1 -duplicate-assets-option-error = Cannot get the number of duplicate assets to keep option -conf-fix = Fix this : {$url} -container-analysing = Analysis of container registry -container-report = {$image_count} images in container registry. {$old_image_count} are older than {$nb_days} days -container-summary = Container registry size: {$registry_size} \ No newline at end of file +pipeline-report = {$total_pipelines} pipelines. {$old_pipelines} pipelines are older than {$nb_days} days +size-artifacts = Artifact jobs size: +size-git-repo = Git repository size: +size-packages = Package registry size: +size-storage = Storage size: \ No newline at end of file diff --git a/i18n/fr-FR/gitlab_project_doctor.ftl b/i18n/fr-FR/gitlab_project_doctor.ftl index 98d4720..7eb428c 100644 --- a/i18n/fr-FR/gitlab_project_doctor.ftl +++ b/i18n/fr-FR/gitlab_project_doctor.ftl @@ -1,42 +1,44 @@ +ask-age-days = Datant de plus de combien de jours ? +ask-delete-files = Supprimer fichiers obsolètes ? +ask-delete-pipelines = Supprimer anciens pipelines ? +conf-analysing = Analyse de la configuration des packages +conf-fix = Pour corriger : {$url} connecting-to-gitlab = Connexion à Gitlab -gitlab-repo = Dépôt Gitlab : {$repo} -error-gl-token = GL_TOKEN environment variable must contain a valid Gitlab private tokenerror-not-gitlab-repo = "This URL is not a gitlab repository" -error-not-git-repo = "Ce dossier n'est pas un dépôt Git" +container-analysing = Analyse du container registry +container-policy-disabled = L'option "Clean up image tags" est désactivée. +container-policy-enabled = L'option "Clean up image tags" est activée +container-report = {$image_count} images dans le container registry. {$old_image_count} datent de plus de {$nb_days} jours +container-summary = Taille du container registry : {$registry_size} +duplicate-assets-option-error = Cannot get the number of duplicate assets to keep option +duplicate-assets-option-onepackage = L'option "The number of duplicate assets to keep" vaut 1 +duplicate-assets-option-warn = L'option "The number of duplicate assets to keep" ne vaut PAS 1 +error = Erreur : +error-gl-token = "Aucun token trouvé. Veuillez renseigner la variable GL_TOKEN contenant un token privée valide ou bien exécutez le programme dans un job Gilab CI". +error-insufficient-privileges = Le token utilisé n'a pas les permissions requises error-no-gitlab-remote = Ce dépôt Git n'est pas lié à un dépôt Gitlab -size-storage = Taille globale : -size-git-repo = Taille du dépôt Git : -size-artifacts = Taille des artefacts de jobs : -size-packages = Taille du package registry : -size-packages = Package registry size: -package-analysing = Analyse du package registry +error-not-git-repo = "Ce dossier n'est pas un dépôt Git" +error-not-gitlab-repo = "This URL is not a gitlab repository" +gitlab-repo = Dépôt Gitlab : {$repo} +help-analysis = Mode analyse : Analyse détaillé au format JSON. Pas de nettoyage +help-batch = Mode batch : pas de questions, pas de barre de progression, idéal pour du CI +help-days = Nombre de jours d'ancienneté à partir duquel un élément est considéré "ancien" +help-git-path = Analyse du projet à partir d'un chemin vers un dépôt Git. Ignoré si l'option url est spécifiée +help-token = (Non recommandé) Token privé Gitlab d'authentification, avec le rôle maintainer nécessaire pour la suppression. Il est conseillé d'utiliser la variable GL_TOKEN à la place de cet argument, pour des raisons de sécurité. +help-url = Analyse du projet à partir d'une URL Gitlab no-cicd = Aucun CI/CD configuré pour ce projet -error = Erreur : -package-report = {$nb_packages} packages. {$nb_files} fichiers sont obsolètes ({$size}) +package-analysing = Analyse du package registry +package-clean-report = {$nb_packages} packages supprimés, {$size} récupérés. package-deleting = Suppression des fichiers de package obsolètes package-no-deletion = Aucun package n'a été supprimé -error-insufficient-privileges = Le token utilisé n'a pas les permissions requises -package-clean-report = {$nb_packages} packages supprimés, {$size} récupérés. +package-report = {$nb_packages} packages. {$nb_files} fichiers sont obsolètes ({$size}) pipeline-analysing = Analyse des pipelines -pipeline-report = {$total_pipelines} pipelines. {$old_pipelines} pipelines datent de plus de {$nb_days} jours -pipeline-deleting = Suppression des anciens pipelines pipeline-clean-report = {$nb_pipelines} pipelines supprimés, {$size} économisés. +pipeline-deleting = Suppression des anciens pipelines pipeline-last-notdeleted = Le dernier pipeline n'est pas supprimé. pipeline-no-deletion = Aucun pipeline n'a été supprimé. -ask-delete-pipelines = Supprimer anciens pipelines ? -ask-delete-files = Supprimer fichiers obsolètes ? -ask-age-days = Datant de plus de combien de jours ? -help-url = Analyse du projet à partir d'une URL Gitlab -help-git-path = Analyse du projet à partir d'un chemin vers un dépôt Git. Ignoré si l'option url est spécifiée -help-batch = Mode batch : pas de questions, pas de barre de progression, idéal pour du CI -help-days = Nombre de jours d'ancienneté à partir duquel un élément est considéré "ancien" -help-analysis = Mode analyse : Analyse détaillé au format JSON. Pas de nettoyage -container-policy-enabled = L'option "Clean up image tags" est activée -container-policy-disabled = L'option "Clean up image tags" est désactivée. -conf-analysing = Analyse de la configuration des packages -duplicate-assets-option-onepackage = L'option "The number of duplicate assets to keep" vaut 1 -duplicate-assets-option-warn = L'option "The number of duplicate assets to keep" ne vaut PAS 1 -duplicate-assets-option-error = Cannot get the number of duplicate assets to keep option -conf-fix = Pour corriger : {$url} -container-analysing = Analyse du container registry -container-report = {$image_count} images dans le container registry. {$old_image_count} datent de plus de {$nb_days} jours -container-summary = Taille du container registry : {$registry_size} \ No newline at end of file +pipeline-report = {$total_pipelines} pipelines. {$old_pipelines} pipelines datent de plus de {$nb_days} jours +size-artifacts = Taille des artefacts de jobs : +size-git-repo = Taille du dépôt Git : +size-packages = Package registry size: +size-packages = Taille du package registry : +size-storage = Taille globale : \ No newline at end of file diff --git a/src/cli.rs b/src/cli.rs index 8a68ecc..24816f0 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,15 +1,16 @@ -use atty::Stream; use std::fmt::Write; use std::time::Duration; use std::{panic, process}; -use crate::{fl, ReportPending, ReportStatus, Reportable}; +use atty::Stream; use console::style; use dialoguer::{Confirm, Input}; use indicatif::{ProgressBar, ProgressStyle}; use lazy_static::lazy_static; use structopt::StructOpt; +use crate::{fl, ReportPending, ReportStatus, Reportable}; + pub fn fatal_if_none<T>(result: Option<T>, msg: &str) -> T { match result { Some(x) => x, @@ -23,21 +24,24 @@ lazy_static! { static ref HELP_URL: String = fl!("help-url"); static ref HELP_GIT_PATH: String = fl!("help-git-path"); static ref HELP_BATCH_MODE: String = fl!("help-batch"); + static ref HELP_TOKEN: String = fl!("help-token"); static ref HELP_DAYS: String = fl!("help-days"); static ref HELP_ANALYSIS: String = fl!("help-analysis"); } #[derive(StructOpt)] pub struct Args { - #[structopt(name = "url", long, help = &HELP_URL)] + #[structopt(name = "url", long, help = & HELP_URL)] pub url: Option<String>, - #[structopt(name = "git_path", required_unless = "url", help = &HELP_GIT_PATH)] + #[structopt(long = "token", short = "t", help = & HELP_TOKEN)] + pub token: Option<String>, + #[structopt(name = "git_path", required_unless = "url", help = & HELP_GIT_PATH)] pub git_path: Option<String>, - #[structopt(long = "batch", short = "b", help = &HELP_BATCH_MODE)] + #[structopt(long = "batch", short = "b", help = & HELP_BATCH_MODE)] pub batch_mode: bool, - #[structopt(long = "days", short = "d", default_value = "30", help = &HELP_DAYS)] + #[structopt(long = "days", short = "d", default_value = "30", help = & HELP_DAYS)] pub days: usize, - #[structopt(long = "analysis", short = "a", help = &HELP_ANALYSIS)] + #[structopt(long = "analysis", short = "a", help = & HELP_ANALYSIS)] pub analysis_mode: bool, } diff --git a/src/diagnosis/gitlab_connection.rs b/src/diagnosis/gitlab_connection.rs index 06affba..42fa301 100644 --- a/src/diagnosis/gitlab_connection.rs +++ b/src/diagnosis/gitlab_connection.rs @@ -16,6 +16,13 @@ use crate::fl; type Result<T> = std::result::Result<T, Box<dyn error::Error>>; +// An enum of a Gitlab token type : private or job token +#[derive(Debug, Deserialize, Clone)] +pub enum GitlabToken { + PrivateToken(String), + JobToken(String), +} + #[derive(Debug, Deserialize, Clone, Serialize)] pub struct Statistics { pub commit_count: u64, @@ -24,6 +31,7 @@ pub struct Statistics { pub job_artifacts_size: u64, pub packages_size: u64, } + #[derive(Debug, Deserialize, Clone)] pub struct ContainerExpirationPolicy { pub enabled: bool, @@ -54,8 +62,8 @@ pub struct ConnectionReport { } pub enum ConnectionJob { - FromUrl(String), - FromPath(String), + FromUrl(String, Option<String>), + FromPath(String, Option<String>), } impl ReportJob for ConnectionJob { @@ -67,8 +75,10 @@ impl ReportJob for ConnectionJob { job: { std::thread::spawn(|| { ConnectionJob::_to_report_status(match self { - ConnectionJob::FromUrl(url) => ConnectionJob::_from_url(&url), - ConnectionJob::FromPath(path) => ConnectionJob::_from_git_path(&path), + ConnectionJob::FromUrl(url, token) => ConnectionJob::_from_url(&url, token), + ConnectionJob::FromPath(path, token) => { + ConnectionJob::_from_git_path(&path, token) + } }) }) }, @@ -103,9 +113,15 @@ impl ConnectionJob { }, } } - fn _gitlab_project(server: &str, path: &str) -> Result<(Gitlab, Project)> { - let token = env::var("GL_TOKEN").map_err(|_| fl!("error-gl-token"))?; - let client = Gitlab::new(server, token)?; + fn _gitlab_project( + server: &str, + path: &str, + token: Option<String>, + ) -> Result<(Gitlab, Project)> { + let client = match Self::_env_token(token)? { + GitlabToken::PrivateToken(token) => Gitlab::new(server, token)?, + GitlabToken::JobToken(token) => Gitlab::new_job_token(server, token)?, + }; let endpoint = projects::Project::builder() .project(path) .statistics(true) @@ -116,9 +132,20 @@ impl ConnectionJob { Ok((client, project)) } - fn _from_url(url: &str) -> Result<GitlabRepository> { + fn _env_token(token: Option<String>) -> Result<GitlabToken> { + let private_token_option = token.or(env::var("GL_TOKEN").ok()); + if let Some(private_token) = private_token_option { + Ok(GitlabToken::PrivateToken(private_token)) + } else { + Ok(GitlabToken::JobToken( + env::var("CI_JOB_TOKEN").map_err(|_| fl!("error-gl-token"))?, + )) + } + } + + fn _from_url(url: &str, token: Option<String>) -> Result<GitlabRepository> { let (server, path) = path_from_git_url(url).ok_or_else(|| fl!("error-not-gitlab-repo"))?; - let (gitlab, project) = ConnectionJob::_gitlab_project(server, path)?; + let (gitlab, project) = ConnectionJob::_gitlab_project(server, path, token)?; Ok(GitlabRepository { url: String::from(path), gitlab, @@ -127,10 +154,10 @@ impl ConnectionJob { }) } - fn _from_git_path(path: &str) -> Result<GitlabRepository> { + fn _from_git_path(path: &str, token: Option<String>) -> Result<GitlabRepository> { let repo = Repository::open(path).map_err(|_| fl!("error-not-git-repo"))?; let (server, url_path) = gitlab_url(&repo).ok_or_else(|| fl!("error-no-gitlab-remote"))?; - let (gitlab, project) = ConnectionJob::_gitlab_project(&server, &url_path)?; + let (gitlab, project) = ConnectionJob::_gitlab_project(&server, &url_path, token)?; Ok(GitlabRepository { url: url_path, gitlab, diff --git a/src/main.rs b/src/main.rs index 7c182ec..ace5709 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,10 @@ +use i18n_embed::{ + fluent::{fluent_language_loader, FluentLanguageLoader}, + DesktopLanguageRequester, +}; +use lazy_static::lazy_static; +use rust_embed::RustEmbed; +use serde::Serialize; use structopt::StructOpt; use cli::Args; @@ -12,13 +19,6 @@ use crate::diagnosis::pipeline_analysis::{PipelineAnalysisJob, PipelineAnalysisR use crate::diagnosis::pipeline_clean::PipelineCleanJob; use crate::diagnosis::ReportStatus; use crate::diagnosis::{RemedyJob, ReportJob, ReportPending, Reportable}; -use i18n_embed::{ - fluent::{fluent_language_loader, FluentLanguageLoader}, - DesktopLanguageRequester, -}; -use lazy_static::lazy_static; -use rust_embed::RustEmbed; -use serde::Serialize; pub mod api; pub mod cli; @@ -119,11 +119,11 @@ impl AnalysisReport { fn _connect_to_gitlab(args: &Args) -> GitlabRepository { let connection_job = { if args.url.is_some() { - ConnectionJob::FromUrl(args.url.as_ref().unwrap().clone()) + ConnectionJob::FromUrl(args.url.as_ref().unwrap().clone(), args.token.clone()) } else { let default_path = String::from("."); let path: &str = args.git_path.as_ref().unwrap_or(&default_path); - ConnectionJob::FromPath(path.to_string()) + ConnectionJob::FromPath(path.to_string(), args.token.clone()) } }; -- GitLab