diff --git a/cmd/gogitlabber/git.go b/cmd/gogitlabber/git.go index 2f4ccb6..e708962 100644 --- a/cmd/gogitlabber/git.go +++ b/cmd/gogitlabber/git.go @@ -5,63 +5,85 @@ import ( "log" "os/exec" "strings" + "sync" ) -func checkoutRepositories(repositories []Repository) { +var mu sync.Mutex +func checkoutRepositories(repositories []Repository, concurrency int) { + + var wg sync.WaitGroup + semaphore := make(chan struct{}, concurrency) + for _, repo := range repositories { - // get repository name + create repo destination - repoName := string(repo.PathWithNamespace) - repoDestination := repoDestinationPre + repoName + wg.Add(1) + semaphore <- struct{}{} - // make gitlab url - url := fmt.Sprintf("https://gitlab-token:%s@%s/%s.git", gitlabToken, gitlabHost, repoName) + go func(repo Repository) { - // check current status of repoDestination - checkRepo := func(repoDestination string) string { - checkCmd := exec.Command("git", "-C", repoDestination, "remote", "-v") - checkOutput, _ := checkCmd.CombinedOutput() + defer func() { + <-semaphore + wg.Done() + }() - return string(checkOutput) - } - repoStatus := checkRepo(repoDestination) + // get repository name + create repo destination + repoName := string(repo.PathWithNamespace) + repoDestination := repoDestinationPre + repoName - // report error if not cloned or pulled repository - // clone repository if it does not exist - switch { - case strings.Contains(string(repoStatus), "No such file or directory"): + // make gitlab url + url := fmt.Sprintf("https://gitlab-token:%s@%s/%s.git", gitlabToken, gitlabHost, repoName) - // update the progress bar - descriptionPrefixPre := "Cloning repository " - descriptionPrefix := descriptionPrefixPre + repoName + " ..." - bar.Describe(descriptionPrefix) + // check current status of repoDestination + checkRepo := func(repoDestination string) string { + checkCmd := exec.Command("git", "-C", repoDestination, "remote", "-v") + checkOutput, _ := checkCmd.CombinedOutput() - // clone the repo - cloneRepository := func(repoDestination string, url string) (string, error) { - cloneCmd := exec.Command("git", "clone", url, repoDestination) - cloneOutput, err := cloneCmd.CombinedOutput() + return string(checkOutput) + } + repoStatus := checkRepo(repoDestination) - return string(cloneOutput), err - } - _, err := cloneRepository(repoDestination, url) - if err != nil { - log.Printf("ERROR: %v\n", err) - } - clonedCount = clonedCount + 1 - progressBarAdd(1) + // report error if not cloned or pulled repository + // clone repository if it does not exist + switch { + case strings.Contains(string(repoStatus), "No such file or directory"): - // pull the latest - case strings.Contains(string(repoStatus), url): - pullRepository(repoName, repoDestination) - progressBarAdd(1) + // update the progress bar + descriptionPrefixPre := "Cloning repository " + descriptionPrefix := descriptionPrefixPre + repoName + " ..." + bar.Describe(descriptionPrefix) - default: - log.Printf("ERROR: decided not to clone or pull repository %v\n", repoName) - log.Printf("ERROR: this is why: %v\n", repoStatus) - errorCount = errorCount + 1 - progressBarAdd(1) - } - } + // clone the repo + cloneRepository := func(repoDestination string, url string) (string, error) { + cloneCmd := exec.Command("git", "clone", url, repoDestination) + cloneOutput, err := cloneCmd.CombinedOutput() + + return string(cloneOutput), err + } + _, err := cloneRepository(repoDestination, url) + if err != nil { + log.Printf("ERROR: %v\n", err) + } + mu.Lock() + clonedCount++ + mu.Unlock() + progressBarAdd(1) + + // pull the latest + case strings.Contains(string(repoStatus), url): + pullRepository(repoName, repoDestination) + progressBarAdd(1) + + default: + log.Printf("ERROR: decided not to clone or pull repository %v\n", repoName) + log.Printf("ERROR: this is why: %v\n", repoStatus) + mu.Lock() + errorCount++ + mu.Unlock() + progressBarAdd(1) + } + }(repo) + } + wg.Wait() } func pullRepository(repoName string, repoDestination string) { diff --git a/cmd/gogitlabber/input.go b/cmd/gogitlabber/input.go index 58a29ad..b37ec78 100644 --- a/cmd/gogitlabber/input.go +++ b/cmd/gogitlabber/input.go @@ -4,6 +4,7 @@ import ( "flag" "log" "os" + "strconv" "strings" ) @@ -11,6 +12,7 @@ func manageArguments() { // configuration vars var archivedFlag = flag.String("archived", "excluded", "To include archived repositories (any|excluded|exclusive)\n example: -archived=any\nenv = GOGITLABBER_ARCHIVED\n") + var concurrencyFlag = flag.Int("concurrency", 15, "Specify repository concurrency\n example: -concurrency=15\nenv = GOGITLABBER_CONCURRENCY\n") var destinationFlag = flag.String("destination", "$HOME/Documents", "Specify where to check the repositories out\n example: -destination=$HOME/repos\nenv = GOGITLABBER_DESTINATION\n") var hostFlag = flag.String("gitlab-url", "gitlab.com", "Specify GitLab host\n example: -gitlab-url=gitlab.com\nenv = GITLAB_URL\n") var tokenFlag = flag.String("gitlab-api-token", "", "Specify GitLab API token\n example: -gitlab-api=glpat-xxxx\nenv = GITLAB_API_TOKEN\n") @@ -18,10 +20,11 @@ func manageArguments() { flag.Parse() // assign the parsed values to your variables + concurrency = *concurrencyFlag + gitlabHost = *hostFlag + gitlabToken = *tokenFlag includeArchived = *archivedFlag repoDestinationPre = *destinationFlag - gitlabToken = *tokenFlag - gitlabHost = *hostFlag // manage gitlab api option switch envToken := os.Getenv("GITLAB_API_TOKEN"); { @@ -29,7 +32,7 @@ func manageArguments() { gitlabToken = envToken default: flag.Usage() - log.Printf("FATAL: config; gitlab api token not found\n") + log.Printf("FATAL: config; gitlab api token not found\n") } // manage gitlab url option @@ -38,7 +41,7 @@ func manageArguments() { gitlabHost = envHost default: flag.Usage() - log.Fatalf("FATAL: config; gitlab host not found\n") + log.Fatalf("FATAL: config; gitlab host not found\n") } // manage destination option @@ -47,7 +50,7 @@ func manageArguments() { repoDestinationPre = envRepoDest default: flag.Usage() - log.Fatalf("FATAL: config; destination not found\n") + log.Fatalf("FATAL: config; destination not found\n") } // add slash 🎩🎸 if not provided @@ -56,6 +59,21 @@ func manageArguments() { repoDestinationPre += "/" } + // manage concurrency option + switch envConcurrency := os.Getenv("GOGITLABBER_CONCURRENCY"); { + case envConcurrency == "": + concurrency = 15 + case envConcurrency != "": + concurrencyValue, err := strconv.Atoi(envConcurrency) + if err != nil { + log.Fatalf("FATAL: invalid concurrency value in environment: %v", err) + } + concurrency = concurrencyValue + default: + flag.Usage() + log.Fatalf("FATAL: config; concurrency not found\n") + } + // manage archived option switch envArchived := os.Getenv("GOGITLABBER_ARCHIVED"); { case envArchived == "": @@ -72,6 +90,6 @@ func manageArguments() { default: flag.Usage() - log.Fatalf("FATAL: config; no or wrong archive option found\n") + log.Fatalf("FATAL: config; no or wrong archive option found\n") } } diff --git a/cmd/gogitlabber/main.go b/cmd/gogitlabber/main.go index c095f32..4c21cee 100644 --- a/cmd/gogitlabber/main.go +++ b/cmd/gogitlabber/main.go @@ -3,10 +3,11 @@ package main import "log" // userdata -var repoDestinationPre string -var includeArchived string -var gitlabToken string +var concurrency int var gitlabHost string +var gitlabToken string +var includeArchived string +var repoDestinationPre string // keep count 🧛 var clonedCount int @@ -36,7 +37,7 @@ func main() { // manage found repositories progressBar(repositories) - checkoutRepositories(repositories) + checkoutRepositories(repositories, concurrency) printSummary() printPullError(pullErrorMsg) }