Resolve "Ajouter les projets de Gitlab (classique)"

Closes #16 (closed)

dev/script_chargement_rdata.R

@@ -6,18 +6,294 @@ library(jsonlite)
 library(utils)
 library(lubridate)
 library(bizdays)
+# gitlab_forge ----
+##Chargement 1 ----
+gitlab_url2 <- "https://gitlab-forge.din.developpement-durable.gouv.fr"
 
-#Chargement 1 ----
-gitlab_url <- "https://gitlab-forge.din.developpement-durable.gouv.fr"
-token <- Sys.getenv("GITLAB_COM_TOKEN")
+gitlabr::set_gitlab_connection(
+  gitlab_url = gitlab_url2, # adresse internet
+  private_token = Sys.getenv("GITLAB_COM_TOKEN")
+) # Token personelle pour connexion
+
+# attention j'ai du modifier le filtre puisque les variable ne correspondait plus a ce de la veille on va peut etre partire sur le select()
+group2 <- gitlabr::gl_list_group_projects(group_id = 1013, max_page = 10)
+group2 <- group2 %>% dplyr::select(id, name, created_at, star_count, last_activity_at, open_issues_count, updated_at, description, dplyr::contains("members"), path) # _links.members
+# premier filtre pour les groupes
+groupf2 <- (group2[as.numeric(group2$open_issues_count) != 0, ])
+groupf2 <- groupf2 %>%
+  dplyr::mutate(
+    name_encoded = utils::URLencode(name), # Encode les espaces et autres caractères spéciaux
+    analyse = paste0(
+      "<a href='https://gitlab-forge.din.developpement-durable.gouv.fr/dreal-pdl/csd/",
+      name_encoded,
+      "/-/value_stream_analytics' target='_blank'>GitLab_analytics</a>"
+    )
+  ) %>%
+  dplyr::select(id, name, created_at, star_count, last_activity_at, open_issues_count, updated_at, description, analyse, dplyr::contains("members"))
+
+groupf2$open_issues_count <- as.numeric(groupf2$open_issues_count)
+# choix par identifiant de projet
+choice2 <- groupf2$id %>% stats::setNames(groupf2$name)
+
+## creation du df pour le journal d'activité
+
+filan_table02 <- purrr::map_dfr(groupf2$id, gitlabr::gl_list_issues)
+
+final_table2 <- filan_table02 %>%
+  dplyr::mutate(id = project_id) %>%
+  dplyr::select(id, project_id, title, description, state, created_at, updated_at, labels, labels1, labels2, labels3, assignees.username, assignees.state, author.username, type, confidential, references.short, closed_at, closed_by.username) %>%
+  dplyr::left_join(dplyr::select(groupf2, id, name), by = "id") %>%
+  dplyr::select(name, project_id, title, description, state, created_at, updated_at, labels, labels1, labels2, labels3, assignees.username, assignees.state, author.username, type, confidential, references.short, closed_at, closed_by.username)
+
+## fonction API
+# Fonction pour récupérer les événements GitLab pour un projet
+get_gitlab_events <- function(project_id, event_type = NULL) {
+  url <- paste0(gitlab_url2, "/api/v4/projects/", project_id, "/events")
+  headers <- httr::add_headers(`PRIVATE-TOKEN` = Sys.getenv("GITLAB_COM_TOKEN"))
+
+  # Construire la requête en fonction du type d'événement souhaité
+  params <- list(per_page = 1500) # Limite à 1500 événements par requête
+  if (!is.null(event_type)) {
+    params[["action"]] <- event_type
+  }
+
+  response <- httr::GET(url, headers, query = params)
+  httr::stop_for_status(response)
+
+  if (httr::status_code(response) != 200) {
+    stop("Échec de la récupération des données pour le projet ", project_id, " : ", httr::content(response, "text"))
+  }
+
+  events <- jsonlite::fromJSON(httr::content(response, "text"), flatten = TRUE)
+  events$project_id <- project_id # Ajouter l'identifiant du projet pour le suivi
+  return(events)
+}
+
+# Fonction pour récupérer les événements pour plusieurs projets
+get_events_for_projects <- function(project_ids, event_type = NULL) {
+  all_events2 <- purrr::map(project_ids, ~ {
+    cat("Récupération des événements pour le projet : ", .x, "\n")
+    tryCatch(
+      {
+        get_gitlab_events(.x, event_type)
+      },
+      error = function(e) {
+        message("Erreur lors de la récupération des événements pour le projet ", .x, " : ", e$message)
+        return(NULL) # Retourner NULL en cas d'erreur pour ce projet
+      }
+    )
+  }) %>%
+    purrr::set_names(project_ids) %>%
+    purrr::discard(is.null)
+
+  all_events_df2 <- dplyr::bind_rows(all_events2, .id = "project_id")
+  return(all_events_df2)
+}
+
+# Liste des identifiants de projets
+project_ids <- as.list(groupf2$id) # Remplacez par vos identifiants de projets
+
+# Récupérer tous les événements pour tous les projets
+all_events2 <- get_events_for_projects(project_ids)
+# Convertir les données en un DataFrame
+all_events_df2 <- dplyr::as_tibble(all_events2)
+
+## Chargement 2 -----
+# Function to fetch all data from GitLab API for a single project
+get_gitlab_project_data <- function(project_id, per_page = 100) {
+  api_url <- paste0(gitlab_url2, "/api/v4/projects/", project_id, "/events")
+
+  # Initial request to get total number of pages
+  initial_response <- httr::GET(api_url, query = list(per_page = per_page, page = 1), httr::add_headers(`PRIVATE-TOKEN` = Sys.getenv("GITLAB_COM_TOKEN")))
+
+  if (httr::status_code(initial_response) == 404) {
+    warning(paste("Project ID", project_id, "not found (404). Skipping."))
+    return(NULL)
+  } else if (httr::status_code(initial_response) != 200) {
+    stop("Failed to fetch initial data for project ID ", project_id, " with status code ", httr::status_code(initial_response))
+  }
+
+  initial_content <- jsonlite::fromJSON(httr::content(initial_response, "text"))
+  total_pages <- as.numeric(headers(initial_response)$`x-total-pages`)
+
+  # Function to fetch data from a specific page
+  fetch_page <- function(page) {
+    response <- httr::GET(api_url, query = list(per_page = per_page, page = page), httr::add_headers(`PRIVATE-TOKEN` = Sys.getenv("GITLAB_COM_TOKEN")))
+    if (httr::status_code(response) == 200) {
+      return(jsonlite::fromJSON(httr::content(response, "text")))
+    } else {
+      warning(paste("Failed to fetch page", page, "with status code", httr::status_code(response), "for project ID", project_id))
+      return(NULL)
+    }
+  }
+
+  all_data <- purrr::map(1:total_pages, fetch_page)
+  all_data <- purrr::compact(all_data)
+  combined_data <- dplyr::bind_rows(all_data)
+  return(combined_data)
+}
+
+# Function to fetch GitLab events for a project with additional variables
+get_gitlab_events <- function(project_id, event_type = NULL) {
+  api_url <- paste0(gitlab_url2, "/api/v4/projects/", project_id, "/events")
+  headers <- httr::add_headers(`PRIVATE-TOKEN` = Sys.getenv("GITLAB_COM_TOKEN"))
+
+  params <- list(per_page = 100)
+  if (!is.null(event_type)) {
+    params[["action"]] <- event_type
+  }
+
+  response <- httr::GET(api_url, headers, query = params)
+  httr::stop_for_status(response)
+
+  if (httr::status_code(response) != 200) {
+    stop("Failed to fetch data for project ", project_id, ": ", httr::content(response, "text"))
+  }
+
+  events <- jsonlite::fromJSON(httr::content(response, "text"), flatten = TRUE)
+  events$project_id <- as.character(project_id) # Convert to character
+  return(events)
+}
+
+# Function to fetch events for multiple projects
+get_events_for_projects <- function(project_ids, event_type = NULL) {
+  all_events2 <- purrr::map(project_ids, ~ {
+    cat("Fetching events for project: ", .x, "\n")
+    tryCatch(
+      {
+        get_gitlab_events(.x, event_type)
+      },
+      error = function(e) {
+        message("Error fetching events for project ", .x, ": ", e$message)
+        return(NULL)
+      }
+    )
+  }) %>%
+    purrr::set_names(project_ids) %>%
+    purrr::discard(is.null)
+
+  all_events_df2 <- dplyr::bind_rows(all_events2, .id = "project_id")
+  return(all_events_df2)
+}
+
+# Function to fetch data for a list of projects
+get_gitlab_data_for_projects <- function(project_ids, per_page = 100) {
+  all_projects_data <- purrr::map(project_ids, ~purrr::safely(get_gitlab_project_data)(.x, per_page)$result)
+  all_projects_data <- purrr::compact(all_projects_data)
+  combined_projects_data <- dplyr::bind_rows(all_projects_data)
+  combined_projects_data$project_id <- as.character(combined_projects_data$project_id) # Convert to character
+  return(combined_projects_data)
+}
+
+# Example usage
+all_events2 <- get_events_for_projects(project_ids)
+gitlab_data2 <- get_gitlab_data_for_projects(project_ids, per_page = 200)
+
+# Merge data
+merged_data2 <- dplyr::left_join(all_events2, gitlab_data2, by = "project_id")
+
+# Select important variables if they exist
+columns_to_select <- c("project_id", "action_name", "push_data.commit_title")
+existing_columns <- dplyr::intersect(columns_to_select, names(merged_data2))
+final_data2 <- merged_data2 %>%
+  dplyr::select(dplyr::any_of(existing_columns), dplyr::everything())
+
+
+
+
+
+
+## pour le graphique temps/projet gitlab forge----
+# traduction des type
+# traduction <- c(
+#   "created" = "Création",
+#   "DiffNote" = "Note d'information",
+#   "DiscussionNote" = "Note de discussion",
+#   "Issue" = "Ticket",
+#   "joined" = "Rejoint",
+#   "MergeRequest" = "Demande de fusion",
+#   "pushed to" = "poussé vers",
+#   "WikiPage::Meta" = "Page Wiki"
+# )
+all_events_df2 <- dplyr::as_tibble(all_events2)
+traduction <- c(
+  "created" = "Code",
+  "DiffNote"= "Documentation",
+  "DiscussionNote" = "Documentation" ,
+  "Issue" = "Gestion de projet" ,
+  "joined" = "Code" ,
+  "MergeRequest" = "Gestion de projet",
+  "pushed to" = "Gestion de projet",
+  "WikiPage::Meta" = "Documentation",
+  "Milestone" = "Gestion de projet",
+  "Note" = "Documentation"
+)
+
+bizdays::create.calendar("MyCalendar", weekdays = c("saturday", "sunday"))
+
+
+data_chronol2 <- all_events_df2 %>%
+  dplyr::mutate(
+    id = project_id,
+    type = dplyr::coalesce(target_type, action_name),
+    type = as.factor(type),
+    temps = lubridate::as_datetime(created_at),
+    info = dplyr::coalesce(push_data.commit_title, target_title)
+  ) %>%
+  dplyr::select(id, type, temps, info) %>%
+  dplyr::left_join(dplyr::select(groupf2, id, name), by = "id")
+# au cas ou il reste des valeur manquante
+data_chronol2 $info[is.na(data_chronol2 $info)] <- ""
+data_chronol2 <- dplyr::distinct(data_chronol2 )
+
+
+# Traduire les niveaux de facteur
+
+
+bizdays::create.calendar("MyCalendar", weekdays = c("saturday", "sunday"))
+
+# Créer un vecteur logique pour identifier les jours ouvrables
+is_workday <- bizdays::is.bizday(data_chronol2 $temps, cal = "MyCalendar")
+
+# Filtrer les données pour exclure les week-ends
+data_chronol2 <- data_chronol2 [is_workday, ]
+
+# Créer un vecteur logique pour identifier les jours ouvrables
+is_workday <- bizdays::is.bizday(data_chronol2$temps, cal = "MyCalendar")
+
+# Filtrer les données pour exclure les week-ends
+gitlab_data_chronol_workdays <- data_chronol2[is_workday, ]
+
+data_gitlab_chonol2 <- final_data2 %>%
+  dplyr::mutate(
+    id = project_id,
+    type = dplyr::coalesce(target_type.y,action_name.y),
+    type = as.factor(type),
+    temps = lubridate::as_datetime(created_at.y),
+    info = dplyr::coalesce(push_data.commit_title, target_title.y)
+  ) %>%
+  dplyr::select(id, type, temps, info) %>%
+  dplyr::left_join(dplyr::select(groupf2, id, name), by = "id")
+
+gitlab_chronol2<-dplyr::distinct(dplyr::bind_rows(data_chronol2,data_gitlab_chonol2))
+
+gitlab_chronol2 <- gitlab_chronol2 %>%
+  dplyr::mutate(type = dplyr::recode(type, !!!traduction))
+gitlab_chronol2 <- gitlab_chronol2 %>% dplyr::filter(!type %in% c("deleted", "pushed new"))
+
+
+#gitlab classique
+
+##Chargement 1 ----
+gitlab_url <- "https://gitlab.com"
 
 gitlabr::set_gitlab_connection(
   gitlab_url = gitlab_url, # adresse internet
-  private_token = token
+  private_token = Sys.getenv("GITLAB_PAT")
 ) # Token personelle pour connexion
 
 # attention j'ai du modifier le filtre puisque les variable ne correspondait plus a ce de la veille on va peut etre partire sur le select()
-group <- gitlabr::gl_list_group_projects(group_id = 1013, max_page = 10)
+group <- gitlabr::gl_list_group_projects(group_id = 6567080, max_page = 10)
 group <- group %>% dplyr::select(id, name, created_at, star_count, last_activity_at, open_issues_count, updated_at, description, dplyr::contains("members"), path) # _links.members
 # premier filtre pour les groupes
 groupf <- (group[as.numeric(group$open_issues_count) != 0, ])
@@ -25,7 +301,7 @@ groupf <- groupf %>%
   dplyr::mutate(
     name_encoded = utils::URLencode(name), # Encode les espaces et autres caractères spéciaux
     analyse = paste0(
-      "<a href='https://gitlab-forge.din.developpement-durable.gouv.fr/dreal-pdl/csd/",
+      "<a href='https://gitlab.com",
       name_encoded,
       "/-/value_stream_analytics' target='_blank'>GitLab_analytics</a>"
     )
@@ -36,7 +312,7 @@ groupf$open_issues_count <- as.numeric(groupf$open_issues_count)
 # choix par identifiant de projet
 choice <- groupf$id %>% stats::setNames(groupf$name)
 
-## creation du df pour le journal d'activité ----
+## creation du df pour le journal d'activité
 
 final_table0 <- purrr::map_dfr(groupf$id, gitlabr::gl_list_issues)
 
@@ -46,11 +322,11 @@ final_table <- final_table0 %>%
   dplyr::left_join(dplyr::select(groupf, id, name), by = "id") %>%
   dplyr::select(name, project_id, title, description, state, created_at, updated_at, labels, labels1, labels2, labels3, assignees.username, assignees.state, author.username, type, confidential, references.short, closed_at, closed_by.username)
 
-## fonction API -----
+## fonction API
 # Fonction pour récupérer les événements GitLab pour un projet
 get_gitlab_events <- function(project_id, event_type = NULL) {
   url <- paste0(gitlab_url, "/api/v4/projects/", project_id, "/events")
-  headers <- httr::add_headers(`PRIVATE-TOKEN` = token)
+  headers <- httr::add_headers(`PRIVATE-TOKEN` = Sys.getenv("GITLAB_PAT"))
 
   # Construire la requête en fonction du type d'événement souhaité
   params <- list(per_page = 1500) # Limite à 1500 événements par requête
@@ -99,13 +375,13 @@ all_events <- get_events_for_projects(project_ids)
 # Convertir les données en un DataFrame
 all_events_df <- dplyr::as_tibble(all_events)
 
-# Chargement 2 -----
+## Chargement 2 -----
 # Function to fetch all data from GitLab API for a single project
 get_gitlab_project_data <- function(project_id, per_page = 100) {
   api_url <- paste0(gitlab_url, "/api/v4/projects/", project_id, "/events")
 
   # Initial request to get total number of pages
-  initial_response <- httr::GET(api_url, query = list(per_page = per_page, page = 1), httr::add_headers(`PRIVATE-TOKEN` = token))
+  initial_response <- httr::GET(api_url, query = list(per_page = per_page, page = 1), httr::add_headers(`PRIVATE-TOKEN` = Sys.getenv("GITLAB_PAT")))
 
   if (httr::status_code(initial_response) == 404) {
     warning(paste("Project ID", project_id, "not found (404). Skipping."))
@@ -119,7 +395,7 @@ get_gitlab_project_data <- function(project_id, per_page = 100) {
 
   # Function to fetch data from a specific page
   fetch_page <- function(page) {
-    response <- httr::GET(api_url, query = list(per_page = per_page, page = page), httr::add_headers(`PRIVATE-TOKEN` = token))
+    response <- httr::GET(api_url, query = list(per_page = per_page, page = page), httr::add_headers(`PRIVATE-TOKEN` = Sys.getenv("GITLAB_PAT")))
     if (httr::status_code(response) == 200) {
       return(jsonlite::fromJSON(httr::content(response, "text")))
     } else {
@@ -137,7 +413,7 @@ get_gitlab_project_data <- function(project_id, per_page = 100) {
 # Function to fetch GitLab events for a project with additional variables
 get_gitlab_events <- function(project_id, event_type = NULL) {
   api_url <- paste0(gitlab_url, "/api/v4/projects/", project_id, "/events")
-  headers <- httr::add_headers(`PRIVATE-TOKEN` = token)
+  headers <- httr::add_headers(`PRIVATE-TOKEN` = Sys.getenv("GITLAB_PAT"))
 
   params <- list(per_page = 100)
   if (!is.null(event_type)) {
@@ -207,7 +483,7 @@ final_data <- merged_data %>%
 
 
 
-## pour le graphique temps/projet ----
+## pour le graphique temps/projet gitlab classqiue----
 # traduction des type
 # traduction <- c(
 #   "created" = "Création",
@@ -284,6 +560,13 @@ gitlab_chronol<-dplyr::distinct(dplyr::bind_rows(data_chronol,data_gitlab_chonol
 gitlab_chronol <- gitlab_chronol %>%
   dplyr::mutate(type = dplyr::recode(type, !!!traduction))
 gitlab_chronol <- gitlab_chronol %>% dplyr::filter(!type %in% c("deleted", "pushed new"))
+
+
+# concaténer les2 -----
+choice <- append(choice, choice2)
+final_table <- dplyr::union(final_table,final_table2)
+groupf <- dplyr::union(groupf,groupf2)
+gitlab_chronol <- dplyr::union(gitlab_chronol,gitlab_chronol2)
 save.image("datamart_gitlabr.RData")
 
 # deploiement vers le sserveur interne de dataviz----------------------------------------