feat: add concurrency

cmd/gogitlabber/git.go

@@ -5,11 +5,27 @@ import (
 	"log"
 	"os/exec"
 	"strings"
+	"sync"
 )
 
-func checkoutRepositories(repositories []Repository) {
+var mu sync.Mutex
+func checkoutRepositories(repositories []Repository, concurrency int) {
+
+  var wg sync.WaitGroup
+  semaphore := make(chan struct{}, concurrency)
+
 	for _, repo := range repositories {
 
+    wg.Add(1)
+    semaphore <- struct{}{}
+
+    go func(repo Repository) {
+
+      defer func() {
+        <-semaphore
+        wg.Done()
+      }()
+
       // get repository name + create repo destination
       repoName := string(repo.PathWithNamespace)
       repoDestination := repoDestinationPre + repoName
@@ -47,7 +63,9 @@ func checkoutRepositories(repositories []Repository) {
         if err != nil {
           log.Printf("ERROR: %v\n", err)
         }
-			clonedCount = clonedCount + 1
+        mu.Lock()
+        clonedCount++
+        mu.Unlock()
         progressBarAdd(1)
 
       // pull the latest
@@ -58,10 +76,14 @@ func checkoutRepositories(repositories []Repository) {
       default:
         log.Printf("ERROR: decided not to clone or pull repository %v\n", repoName)
         log.Printf("ERROR: this is why: %v\n", repoStatus)
-			errorCount = errorCount + 1
+        mu.Lock()
+        errorCount++
+        mu.Unlock()
         progressBarAdd(1)
       }
+    }(repo)
   }
+  wg.Wait()
 }
 
 func pullRepository(repoName string, repoDestination string) {

cmd/gogitlabber/input.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"log"
 	"os"
+	"strconv"
 	"strings"
 )
 
@@ -11,6 +12,7 @@ func manageArguments() {
 
 	// configuration vars
 	var archivedFlag = flag.String("archived", "excluded", "To include archived repositories (any|excluded|exclusive)\n  example: -archived=any\nenv = GOGITLABBER_ARCHIVED\n")
+	var concurrencyFlag = flag.Int("concurrency", 15, "Specify repository concurrency\n  example: -concurrency=15\nenv = GOGITLABBER_CONCURRENCY\n")
 	var destinationFlag = flag.String("destination", "$HOME/Documents", "Specify where to check the repositories out\n  example: -destination=$HOME/repos\nenv = GOGITLABBER_DESTINATION\n")
 	var hostFlag = flag.String("gitlab-url", "gitlab.com", "Specify GitLab host\n  example: -gitlab-url=gitlab.com\nenv = GITLAB_URL\n")
 	var tokenFlag = flag.String("gitlab-api-token", "", "Specify GitLab API token\n  example: -gitlab-api=glpat-xxxx\nenv = GITLAB_API_TOKEN\n")
@@ -18,10 +20,11 @@ func manageArguments() {
 	flag.Parse()
 
 	// assign the parsed values to your variables
+	concurrency = *concurrencyFlag
+	gitlabHost = *hostFlag
+	gitlabToken = *tokenFlag
 	includeArchived = *archivedFlag
 	repoDestinationPre = *destinationFlag
-	gitlabToken = *tokenFlag
-	gitlabHost = *hostFlag
 
 	// manage gitlab api option
 	switch envToken := os.Getenv("GITLAB_API_TOKEN"); {
@@ -56,6 +59,21 @@ func manageArguments() {
 		repoDestinationPre += "/"
 	}
 
+	// manage concurrency option
+  switch envConcurrency := os.Getenv("GOGITLABBER_CONCURRENCY"); {
+  case envConcurrency == "":
+    concurrency = 15
+  case envConcurrency != "":
+		concurrencyValue, err := strconv.Atoi(envConcurrency)
+		if err != nil {
+      log.Fatalf("FATAL: invalid concurrency value in environment: %v", err)
+		}
+		concurrency = concurrencyValue
+  default:
+    flag.Usage()
+    log.Fatalf("FATAL: config; concurrency not found\n")
+  }
+
 	// manage archived option
 	switch envArchived := os.Getenv("GOGITLABBER_ARCHIVED"); {
 	case envArchived == "":

cmd/gogitlabber/main.go

@@ -3,10 +3,11 @@ package main
 import "log"
 
 // userdata
-var repoDestinationPre string
+var concurrency int
-var includeArchived string
-var gitlabToken string
 var gitlabHost string
+var gitlabToken string
+var includeArchived string
+var repoDestinationPre string
 
 // keep count 🧛
 var clonedCount int
@@ -36,7 +37,7 @@ func main() {
 
 	// manage found repositories
 	progressBar(repositories)
-	checkoutRepositories(repositories)
+	checkoutRepositories(repositories, concurrency)
 	printSummary()
 	printPullError(pullErrorMsg)
 }