feat: add concurrency

cmd/gogitlabber/git.go

@@ -5,63 +5,85 @@ import (
 	"log"
 	"os/exec"
 	"strings"
+	"sync"
 )
 
-func checkoutRepositories(repositories []Repository) {
+var mu sync.Mutex
+func checkoutRepositories(repositories []Repository, concurrency int) {
+
+  var wg sync.WaitGroup
+  semaphore := make(chan struct{}, concurrency)
+
 	for _, repo := range repositories {
 
-		// get repository name + create repo destination
+    wg.Add(1)
-		repoName := string(repo.PathWithNamespace)
+    semaphore <- struct{}{}
-		repoDestination := repoDestinationPre + repoName
 
-		// make gitlab url
+    go func(repo Repository) {
-		url := fmt.Sprintf("https://gitlab-token:%s@%s/%s.git", gitlabToken, gitlabHost, repoName)
 
-		// check current status of repoDestination
+      defer func() {
-		checkRepo := func(repoDestination string) string {
+        <-semaphore
-			checkCmd := exec.Command("git", "-C", repoDestination, "remote", "-v")
+        wg.Done()
-			checkOutput, _ := checkCmd.CombinedOutput()
+      }()
 
-			return string(checkOutput)
+      // get repository name + create repo destination
-		}
+      repoName := string(repo.PathWithNamespace)
-		repoStatus := checkRepo(repoDestination)
+      repoDestination := repoDestinationPre + repoName
 
-		// report error if not cloned or pulled repository
+      // make gitlab url
-		// clone repository if it does not exist
+      url := fmt.Sprintf("https://gitlab-token:%s@%s/%s.git", gitlabToken, gitlabHost, repoName)
-		switch {
-		case strings.Contains(string(repoStatus), "No such file or directory"):
 
-			// update the progress bar
+      // check current status of repoDestination
-			descriptionPrefixPre := "Cloning repository "
+      checkRepo := func(repoDestination string) string {
-			descriptionPrefix := descriptionPrefixPre + repoName + " ..."
+        checkCmd := exec.Command("git", "-C", repoDestination, "remote", "-v")
-			bar.Describe(descriptionPrefix)
+        checkOutput, _ := checkCmd.CombinedOutput()
 
-			// clone the repo
+        return string(checkOutput)
-			cloneRepository := func(repoDestination string, url string) (string, error) {
+      }
-				cloneCmd := exec.Command("git", "clone", url, repoDestination)
+      repoStatus := checkRepo(repoDestination)
-				cloneOutput, err := cloneCmd.CombinedOutput()
 
-				return string(cloneOutput), err
+      // report error if not cloned or pulled repository
-			}
+      // clone repository if it does not exist
-			_, err := cloneRepository(repoDestination, url)
+      switch {
-			if err != nil {
+      case strings.Contains(string(repoStatus), "No such file or directory"):
-				log.Printf("ERROR: %v\n", err)
-			}
-			clonedCount = clonedCount + 1
-			progressBarAdd(1)
 
-		// pull the latest
+        // update the progress bar
-		case strings.Contains(string(repoStatus), url):
+        descriptionPrefixPre := "Cloning repository "
-			pullRepository(repoName, repoDestination)
+        descriptionPrefix := descriptionPrefixPre + repoName + " ..."
-			progressBarAdd(1)
+        bar.Describe(descriptionPrefix)
 
-		default:
+        // clone the repo
-			log.Printf("ERROR: decided not to clone or pull repository %v\n", repoName)
+        cloneRepository := func(repoDestination string, url string) (string, error) {
-			log.Printf("ERROR: this is why: %v\n", repoStatus)
+          cloneCmd := exec.Command("git", "clone", url, repoDestination)
-			errorCount = errorCount + 1
+          cloneOutput, err := cloneCmd.CombinedOutput()
-			progressBarAdd(1)
+
-		}
+          return string(cloneOutput), err
-	}
+        }
+        _, err := cloneRepository(repoDestination, url)
+        if err != nil {
+          log.Printf("ERROR: %v\n", err)
+        }
+        mu.Lock()
+        clonedCount++
+        mu.Unlock()
+        progressBarAdd(1)
+
+      // pull the latest
+      case strings.Contains(string(repoStatus), url):
+        pullRepository(repoName, repoDestination)
+        progressBarAdd(1)
+
+      default:
+        log.Printf("ERROR: decided not to clone or pull repository %v\n", repoName)
+        log.Printf("ERROR: this is why: %v\n", repoStatus)
+        mu.Lock()
+        errorCount++
+        mu.Unlock()
+        progressBarAdd(1)
+      }
+    }(repo)
+  }
+  wg.Wait()
 }
 
 func pullRepository(repoName string, repoDestination string) {

cmd/gogitlabber/input.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"log"
 	"os"
+	"strconv"
 	"strings"
 )
 
@@ -11,6 +12,7 @@ func manageArguments() {
 
 	// configuration vars
 	var archivedFlag = flag.String("archived", "excluded", "To include archived repositories (any|excluded|exclusive)\n  example: -archived=any\nenv = GOGITLABBER_ARCHIVED\n")
+	var concurrencyFlag = flag.Int("concurrency", 15, "Specify repository concurrency\n  example: -concurrency=15\nenv = GOGITLABBER_CONCURRENCY\n")
 	var destinationFlag = flag.String("destination", "$HOME/Documents", "Specify where to check the repositories out\n  example: -destination=$HOME/repos\nenv = GOGITLABBER_DESTINATION\n")
 	var hostFlag = flag.String("gitlab-url", "gitlab.com", "Specify GitLab host\n  example: -gitlab-url=gitlab.com\nenv = GITLAB_URL\n")
 	var tokenFlag = flag.String("gitlab-api-token", "", "Specify GitLab API token\n  example: -gitlab-api=glpat-xxxx\nenv = GITLAB_API_TOKEN\n")
@@ -18,10 +20,11 @@ func manageArguments() {
 	flag.Parse()
 
 	// assign the parsed values to your variables
+	concurrency = *concurrencyFlag
+	gitlabHost = *hostFlag
+	gitlabToken = *tokenFlag
 	includeArchived = *archivedFlag
 	repoDestinationPre = *destinationFlag
-	gitlabToken = *tokenFlag
-	gitlabHost = *hostFlag
 
 	// manage gitlab api option
 	switch envToken := os.Getenv("GITLAB_API_TOKEN"); {
@@ -29,7 +32,7 @@ func manageArguments() {
 		gitlabToken = envToken
 	default:
 		flag.Usage()
-    log.Printf("FATAL: config; gitlab api token not found\n")
+		log.Printf("FATAL: config; gitlab api token not found\n")
 	}
 
 	// manage gitlab url option
@@ -38,7 +41,7 @@ func manageArguments() {
 		gitlabHost = envHost
 	default:
 		flag.Usage()
-    log.Fatalf("FATAL: config; gitlab host not found\n")
+		log.Fatalf("FATAL: config; gitlab host not found\n")
 	}
 
 	// manage destination option
@@ -47,7 +50,7 @@ func manageArguments() {
 		repoDestinationPre = envRepoDest
 	default:
 		flag.Usage()
-    log.Fatalf("FATAL: config; destination not found\n")
+		log.Fatalf("FATAL: config; destination not found\n")
 	}
 
 	// add slash 🎩🎸 if not provided
@@ -56,6 +59,21 @@ func manageArguments() {
 		repoDestinationPre += "/"
 	}
 
+	// manage concurrency option
+  switch envConcurrency := os.Getenv("GOGITLABBER_CONCURRENCY"); {
+  case envConcurrency == "":
+    concurrency = 15
+  case envConcurrency != "":
+		concurrencyValue, err := strconv.Atoi(envConcurrency)
+		if err != nil {
+      log.Fatalf("FATAL: invalid concurrency value in environment: %v", err)
+		}
+		concurrency = concurrencyValue
+  default:
+    flag.Usage()
+    log.Fatalf("FATAL: config; concurrency not found\n")
+  }
+
 	// manage archived option
 	switch envArchived := os.Getenv("GOGITLABBER_ARCHIVED"); {
 	case envArchived == "":
@@ -72,6 +90,6 @@ func manageArguments() {
 
 	default:
 		flag.Usage()
-    log.Fatalf("FATAL: config; no or wrong archive option found\n")
+		log.Fatalf("FATAL: config; no or wrong archive option found\n")
 	}
 }

cmd/gogitlabber/main.go

@@ -3,10 +3,11 @@ package main
 import "log"
 
 // userdata
-var repoDestinationPre string
+var concurrency int
-var includeArchived string
-var gitlabToken string
 var gitlabHost string
+var gitlabToken string
+var includeArchived string
+var repoDestinationPre string
 
 // keep count 🧛
 var clonedCount int
@@ -36,7 +37,7 @@ func main() {
 
 	// manage found repositories
 	progressBar(repositories)
-	checkoutRepositories(repositories)
+	checkoutRepositories(repositories, concurrency)
 	printSummary()
 	printPullError(pullErrorMsg)
 }