feat: add concurrency
This commit is contained in:
parent
0e388f04ae
commit
501e74967c
3 changed files with 103 additions and 62 deletions
|
|
@ -5,63 +5,85 @@ import (
|
|||
"log"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func checkoutRepositories(repositories []Repository) {
|
||||
var mu sync.Mutex
|
||||
func checkoutRepositories(repositories []Repository, concurrency int) {
|
||||
|
||||
var wg sync.WaitGroup
|
||||
semaphore := make(chan struct{}, concurrency)
|
||||
|
||||
for _, repo := range repositories {
|
||||
|
||||
// get repository name + create repo destination
|
||||
repoName := string(repo.PathWithNamespace)
|
||||
repoDestination := repoDestinationPre + repoName
|
||||
wg.Add(1)
|
||||
semaphore <- struct{}{}
|
||||
|
||||
// make gitlab url
|
||||
url := fmt.Sprintf("https://gitlab-token:%s@%s/%s.git", gitlabToken, gitlabHost, repoName)
|
||||
go func(repo Repository) {
|
||||
|
||||
// check current status of repoDestination
|
||||
checkRepo := func(repoDestination string) string {
|
||||
checkCmd := exec.Command("git", "-C", repoDestination, "remote", "-v")
|
||||
checkOutput, _ := checkCmd.CombinedOutput()
|
||||
defer func() {
|
||||
<-semaphore
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
return string(checkOutput)
|
||||
}
|
||||
repoStatus := checkRepo(repoDestination)
|
||||
// get repository name + create repo destination
|
||||
repoName := string(repo.PathWithNamespace)
|
||||
repoDestination := repoDestinationPre + repoName
|
||||
|
||||
// report error if not cloned or pulled repository
|
||||
// clone repository if it does not exist
|
||||
switch {
|
||||
case strings.Contains(string(repoStatus), "No such file or directory"):
|
||||
// make gitlab url
|
||||
url := fmt.Sprintf("https://gitlab-token:%s@%s/%s.git", gitlabToken, gitlabHost, repoName)
|
||||
|
||||
// update the progress bar
|
||||
descriptionPrefixPre := "Cloning repository "
|
||||
descriptionPrefix := descriptionPrefixPre + repoName + " ..."
|
||||
bar.Describe(descriptionPrefix)
|
||||
// check current status of repoDestination
|
||||
checkRepo := func(repoDestination string) string {
|
||||
checkCmd := exec.Command("git", "-C", repoDestination, "remote", "-v")
|
||||
checkOutput, _ := checkCmd.CombinedOutput()
|
||||
|
||||
// clone the repo
|
||||
cloneRepository := func(repoDestination string, url string) (string, error) {
|
||||
cloneCmd := exec.Command("git", "clone", url, repoDestination)
|
||||
cloneOutput, err := cloneCmd.CombinedOutput()
|
||||
return string(checkOutput)
|
||||
}
|
||||
repoStatus := checkRepo(repoDestination)
|
||||
|
||||
return string(cloneOutput), err
|
||||
}
|
||||
_, err := cloneRepository(repoDestination, url)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: %v\n", err)
|
||||
}
|
||||
clonedCount = clonedCount + 1
|
||||
progressBarAdd(1)
|
||||
// report error if not cloned or pulled repository
|
||||
// clone repository if it does not exist
|
||||
switch {
|
||||
case strings.Contains(string(repoStatus), "No such file or directory"):
|
||||
|
||||
// pull the latest
|
||||
case strings.Contains(string(repoStatus), url):
|
||||
pullRepository(repoName, repoDestination)
|
||||
progressBarAdd(1)
|
||||
// update the progress bar
|
||||
descriptionPrefixPre := "Cloning repository "
|
||||
descriptionPrefix := descriptionPrefixPre + repoName + " ..."
|
||||
bar.Describe(descriptionPrefix)
|
||||
|
||||
default:
|
||||
log.Printf("ERROR: decided not to clone or pull repository %v\n", repoName)
|
||||
log.Printf("ERROR: this is why: %v\n", repoStatus)
|
||||
errorCount = errorCount + 1
|
||||
progressBarAdd(1)
|
||||
}
|
||||
}
|
||||
// clone the repo
|
||||
cloneRepository := func(repoDestination string, url string) (string, error) {
|
||||
cloneCmd := exec.Command("git", "clone", url, repoDestination)
|
||||
cloneOutput, err := cloneCmd.CombinedOutput()
|
||||
|
||||
return string(cloneOutput), err
|
||||
}
|
||||
_, err := cloneRepository(repoDestination, url)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: %v\n", err)
|
||||
}
|
||||
mu.Lock()
|
||||
clonedCount++
|
||||
mu.Unlock()
|
||||
progressBarAdd(1)
|
||||
|
||||
// pull the latest
|
||||
case strings.Contains(string(repoStatus), url):
|
||||
pullRepository(repoName, repoDestination)
|
||||
progressBarAdd(1)
|
||||
|
||||
default:
|
||||
log.Printf("ERROR: decided not to clone or pull repository %v\n", repoName)
|
||||
log.Printf("ERROR: this is why: %v\n", repoStatus)
|
||||
mu.Lock()
|
||||
errorCount++
|
||||
mu.Unlock()
|
||||
progressBarAdd(1)
|
||||
}
|
||||
}(repo)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func pullRepository(repoName string, repoDestination string) {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
|
|
@ -11,6 +12,7 @@ func manageArguments() {
|
|||
|
||||
// configuration vars
|
||||
var archivedFlag = flag.String("archived", "excluded", "To include archived repositories (any|excluded|exclusive)\n example: -archived=any\nenv = GOGITLABBER_ARCHIVED\n")
|
||||
var concurrencyFlag = flag.Int("concurrency", 15, "Specify repository concurrency\n example: -concurrency=15\nenv = GOGITLABBER_CONCURRENCY\n")
|
||||
var destinationFlag = flag.String("destination", "$HOME/Documents", "Specify where to check the repositories out\n example: -destination=$HOME/repos\nenv = GOGITLABBER_DESTINATION\n")
|
||||
var hostFlag = flag.String("gitlab-url", "gitlab.com", "Specify GitLab host\n example: -gitlab-url=gitlab.com\nenv = GITLAB_URL\n")
|
||||
var tokenFlag = flag.String("gitlab-api-token", "", "Specify GitLab API token\n example: -gitlab-api=glpat-xxxx\nenv = GITLAB_API_TOKEN\n")
|
||||
|
|
@ -18,10 +20,11 @@ func manageArguments() {
|
|||
flag.Parse()
|
||||
|
||||
// assign the parsed values to your variables
|
||||
concurrency = *concurrencyFlag
|
||||
gitlabHost = *hostFlag
|
||||
gitlabToken = *tokenFlag
|
||||
includeArchived = *archivedFlag
|
||||
repoDestinationPre = *destinationFlag
|
||||
gitlabToken = *tokenFlag
|
||||
gitlabHost = *hostFlag
|
||||
|
||||
// manage gitlab api option
|
||||
switch envToken := os.Getenv("GITLAB_API_TOKEN"); {
|
||||
|
|
@ -29,7 +32,7 @@ func manageArguments() {
|
|||
gitlabToken = envToken
|
||||
default:
|
||||
flag.Usage()
|
||||
log.Printf("FATAL: config; gitlab api token not found\n")
|
||||
log.Printf("FATAL: config; gitlab api token not found\n")
|
||||
}
|
||||
|
||||
// manage gitlab url option
|
||||
|
|
@ -38,7 +41,7 @@ func manageArguments() {
|
|||
gitlabHost = envHost
|
||||
default:
|
||||
flag.Usage()
|
||||
log.Fatalf("FATAL: config; gitlab host not found\n")
|
||||
log.Fatalf("FATAL: config; gitlab host not found\n")
|
||||
}
|
||||
|
||||
// manage destination option
|
||||
|
|
@ -47,7 +50,7 @@ func manageArguments() {
|
|||
repoDestinationPre = envRepoDest
|
||||
default:
|
||||
flag.Usage()
|
||||
log.Fatalf("FATAL: config; destination not found\n")
|
||||
log.Fatalf("FATAL: config; destination not found\n")
|
||||
}
|
||||
|
||||
// add slash 🎩🎸 if not provided
|
||||
|
|
@ -56,6 +59,21 @@ func manageArguments() {
|
|||
repoDestinationPre += "/"
|
||||
}
|
||||
|
||||
// manage concurrency option
|
||||
switch envConcurrency := os.Getenv("GOGITLABBER_CONCURRENCY"); {
|
||||
case envConcurrency == "":
|
||||
concurrency = 15
|
||||
case envConcurrency != "":
|
||||
concurrencyValue, err := strconv.Atoi(envConcurrency)
|
||||
if err != nil {
|
||||
log.Fatalf("FATAL: invalid concurrency value in environment: %v", err)
|
||||
}
|
||||
concurrency = concurrencyValue
|
||||
default:
|
||||
flag.Usage()
|
||||
log.Fatalf("FATAL: config; concurrency not found\n")
|
||||
}
|
||||
|
||||
// manage archived option
|
||||
switch envArchived := os.Getenv("GOGITLABBER_ARCHIVED"); {
|
||||
case envArchived == "":
|
||||
|
|
@ -72,6 +90,6 @@ func manageArguments() {
|
|||
|
||||
default:
|
||||
flag.Usage()
|
||||
log.Fatalf("FATAL: config; no or wrong archive option found\n")
|
||||
log.Fatalf("FATAL: config; no or wrong archive option found\n")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,10 +3,11 @@ package main
|
|||
import "log"
|
||||
|
||||
// userdata
|
||||
var repoDestinationPre string
|
||||
var includeArchived string
|
||||
var gitlabToken string
|
||||
var concurrency int
|
||||
var gitlabHost string
|
||||
var gitlabToken string
|
||||
var includeArchived string
|
||||
var repoDestinationPre string
|
||||
|
||||
// keep count 🧛
|
||||
var clonedCount int
|
||||
|
|
@ -36,7 +37,7 @@ func main() {
|
|||
|
||||
// manage found repositories
|
||||
progressBar(repositories)
|
||||
checkoutRepositories(repositories)
|
||||
checkoutRepositories(repositories, concurrency)
|
||||
printSummary()
|
||||
printPullError(pullErrorMsg)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue