les groupes, les topics sont automatiques et le noms des projets aussi ,...

les groupes, les topics sont automatiques et le noms des projets aussi , attention les auteurs sont encore sous formes de mail

dev/gitlab_classique.R

@@ -9,6 +9,32 @@ set_gitlab_connection(
   gitlab_url = "https://gitlab.com" ,
   private_token = Sys.getenv("GITLAB_COM_TOKEN")
 )
+response <- data.frame()
+# on charge les projets du groupe CSD
+buildres <- function(i){
+  res_proj100 <- GET(
+    "https://gitlab.com/api/v4/groups/6567080/projects",
+    add_headers(`PRIVATE-TOKEN` = Sys.getenv("GITLAB_COM_TOKEN")),
+    query = list(
+      include_subgroups = "true",
+      per_page = 100,
+      page = i
+    ),
+    timeout(120)  # Timeout augmenté pour les projets
+  )
+  res <- content(res_proj100, as = "text", encoding = "UTF-8") %>% fromJSON()
+  return(res)
+}
+#on réalise une boucle si plus de 100 projets
+x <- 1
+repeat {
+  res <- buildres(x)
+  if (length(res) == 0) {
+    break
+  }
+  response <- bind_rows(response, res)
+  x <- x + 1
+}
 project_ids <- c(41600697, 19859695, 49118792, 21138017, 44145525, 18441361, 17610613,18439010)
 
 get_project_name <- function(base_url, project_id, private_token) {
@@ -103,10 +129,7 @@ get_all_pages <- function(url, private_token, max_retries = 3, max_pages = 159)
 
 
 
-# Fonction pour extraire le nom du projet à partir de l'URL
-extract_project_name <- function(web_url) {
-  str_extract(web_url, "(?<=/)[^/]+(?=/-/)")
-}
+
 
 # Fonction pour récupérer les issues d'un projet GitLab
 get_gitlab_issues <- function(base_url, project_id, private_token) {
@@ -114,10 +137,10 @@ get_gitlab_issues <- function(base_url, project_id, private_token) {
   data <- get_all_pages(api_url, private_token)
   data <- data %>%
     mutate(
-      project_name = extract_project_name(web_url),
+      project_name = as.character(project_id),
       type = "issue",
       message = title,
-      author = author.username
+      author = paste0(author.username,"@developpement-durable.gouv.fr")
     )
   return(data %>% select(project_name, type, message, updated_at, author))
 }
@@ -148,11 +171,9 @@ get_gitlab_events <- function(base_url, project_id, private_token) {
       ),
 
       updated_at = created_at,
-      author = author.username
-    ) %>%
-    left_join(name_gitlab, by = c("project_name" = "id")) %>%
-    mutate(project_name = name) %>%
-    select(-name)
+      author = paste0(author.username,"@developpement-durable.gouv.fr")
+    )
+
   return(data %>% select(project_name, type, message, updated_at, author))
 }
 
@@ -171,30 +192,6 @@ get_data_from_multiple_projects <- function(base_url, project_ids, private_token
   return(bind_rows(all_data))
 }
 
-# Topics
-get_project_topics <- function(base_url, project_id, private_token) {
-  api_url <- paste0(base_url, "/api/v4/projects/", project_id)
-  response <- GET(api_url, add_headers("PRIVATE-TOKEN" = private_token))
-
-  if (status_code(response) == 200) {
-    project_info <- fromJSON(content(response, "text", encoding = "UTF-8"), flatten = TRUE)
-    return(data.frame(
-      id = as.character(project_info$name),
-      topics = paste(project_info$topics, collapse = ", "),  # Combine les topics en une chaîne de caractères
-      stringsAsFactors = FALSE
-    ))
-  } else {
-    warning("Impossible de récupérer les topics pour le projet ID: ", project_id, " - Code: ", status_code(response))
-    return(data.frame(
-      id = project_id,
-      topics = NA,
-      stringsAsFactors = FALSE
-    ))
-  }
-}
-extract_before_at <- function(email) {
-  sub("@.*", "", email)
-}
 ############################################################### -----
 # Fonction générique
 process_projects <- function(project_ids) {
@@ -225,16 +222,12 @@ process_projects <- function(project_ids) {
 
 final_result <- process_projects(project_ids)
 final_result$project_id <- as.character(final_result$project_id)
-final_result <- final_result %>% left_join(name_gitlab, by = c("project_id" = "id")) %>%
-  mutate(project_name = name, type = "commit") %>%
-  select(-name)
-final_result <- final_result %>% mutate(author = extract_before_at(committer_email))
-final_result <- final_result %>% rename( "updated_at" = 'committed_date' )
+
+final_result <- final_result %>% mutate(author = committer_email, type = "commit")
+final_result <- final_result %>% rename( "updated_at" = 'committed_date',"project_name" = 'project_id' )
 fg <- final_result %>% filter(is_duplicate == FALSE) %>% select(project_name,type , message ,updated_at,author)
 ###############################################################
 
-# Récupérer les topics pour tous les projets
-projects_topics <- bind_rows(lapply(project_ids, get_project_topics, base_url = base_url, private_token = private_token))
 
 # Récupérer les données pour tous les projets
 all_data_gitlab <- get_data_from_multiple_projects(base_url, project_ids, private_token)
@@ -242,9 +235,21 @@ all_data_gitlab <- all_data_gitlab %>%
   filter(!(is.na(all_data_gitlab$type) | is.na(all_data_gitlab$message)) | all_data_gitlab$type == "joined" | all_data_gitlab$type == "WikiPage::Meta")
 all_data_gitlab <- bind_rows(fg,all_data_gitlab)
 all_data_gitlab <- all_data_gitlab %>%
-  mutate(origine ="Gitlab",
-         groupe = "RDES_DREAL")
-all_data_gitlab <- left_join(all_data_gitlab, projects_topics, by = c("project_name" = "id"))
+  mutate(origine ="Gitlab")
+
+#data.frame avec les groupes et le nom pour left join-----
+response_df <- data.frame(
+  id = as.character(response$id),
+  name = response$name,
+  groupe = response$namespace$name,
+  topics = sapply(response$topics, function(x) paste(x, collapse = ", ")) # on degroupe et regroupe ?? mais ca fonctionne
+)
+# Joindre les deux jeux de données
+all_data_gitlab <- all_data_gitlab %>%
+  left_join(response_df, by = c("project_name" = "id")) # Faire un premier left_join() sur la colonne name
+all_data_gitlab$author <-  tolower(all_data_gitlab$author)
 
+# sauvegarde du jeu de donnée-----
+all_data_gitlab <- all_data_gitlab %>% select(name , type, message , updated_at , author , topics, origine,groupe)
 # save.image("gitlab.RData")
 save(all_data_gitlab, file = "gitlab.RData")