feat: rewrite gitlab.go

cmd/gogitlabber/gitea.go

@@ -34,7 +34,7 @@ type GiteaAPIOptions struct {
 	Page            int
 }
 
-// gitea api clien
+// gitea api client
 func NewGiteaClient(baseURL, token string) *GiteaClient {
 	return &GiteaClient{
 		httpClient: &http.Client{
@@ -141,7 +141,7 @@ func (c *GiteaClient) fetchRepositoryPage(ctx context.Context, options GiteaAPIO
 	return giteaRepos, hasMore, nil
 }
 
-// build final api url
+// build api url
 func (c *GiteaClient) buildAPIURL(options GiteaAPIOptions) (string, error) {
 	baseURL := fmt.Sprintf("https://%s/api/v1/user/repos", c.baseURL)
 

cmd/gogitlabber/gitlab.go

@@ -1,84 +1,406 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
+	"strconv"
+	"time"
 
 	"github.com/scornet256/go-logger"
 )
 
-func fetchRepositoriesGitlab() ([]Repository, error) {
+// GitLabClient encapsulates the GitLab API client functionality
-
+type GitLabClient struct {
-	// default options
+	httpClient *http.Client
-	membership := "membership=true"
+	baseURL    string
-	perpage := "per_page=100"
+	token      string
-	order := "order_by=name"
-
-	// configure archived options
-	var archived string
-	switch config.IncludeArchived {
-	case "excluded":
-		archived = "&archived=false"
-	case "only":
-		archived = "&archived=true"
-	default:
-		archived = ""
 }
 
-	url := fmt.Sprintf("https://%s/api/v4/projects?%s&%s&%s%s",
+// GitLabProject represents a project from GitLab API
-		config.GitHost, membership, order, perpage, archived)
+type GitLabProject struct {
-
+	ID                int    `json:"id"`
-	logger.Print("HTTP: Creating API request", nil)
+	Name              string `json:"name"`
-	req, err := http.NewRequest("GET", url, nil)
+	Path              string `json:"path"`
-	if err != nil {
+	PathWithNamespace string `json:"path_with_namespace"`
-		return nil, fmt.Errorf("ERROR: creating request: %v", err)
+	Archived          bool   `json:"archived"`
+	LastActivityAt    string `json:"last_activity_at"`
+	WebURL            string `json:"web_url"`
 }
 
-	logger.Print("HTTP: Adding PRIVATE-TOKEN header to API request", nil)
+// GitLabAPIOptions holds the API request parameters
-	req.Header.Set("PRIVATE-TOKEN", config.GitToken)
+type GitLabAPIOptions struct {
+	Membership      bool
+	IncludeArchived string
+	OrderBy         string
+	Sort            string
+	PerPage         int
+	Page            int
+	MinAccessLevel  int // 10=Guest, 20=Reporter, 30=Developer, 40=Maintainer, 50=Owner
+}
 
-	logger.Print("HTTP: Making request", nil)
+// gitlab pagination info
-	client := &http.Client{}
+type GitLabPaginationInfo struct {
-	resp, err := client.Do(req)
+	TotalPages   int
+	TotalItems   int
+	CurrentPage  int
+	NextPage     int
+	PreviousPage int
+}
+
+// gitlab client
+func NewGitLabClient(baseURL, token string) *GitLabClient {
+	return &GitLabClient{
+		httpClient: &http.Client{
+			Timeout: 30 * time.Second,
+		},
+		baseURL: baseURL,
+		token:   token,
+	}
+}
+
+// fetch gitlab repos
+func FetchRepositoriesGitLab() ([]Repository, error) {
+	client := NewGitLabClient(config.GitHost, config.GitToken)
+
+	options := GitLabAPIOptions{
+		Membership:      true,
+		IncludeArchived: config.IncludeArchived,
+		OrderBy:         "name",
+		Sort:            "asc",
+		PerPage:         100,
+		Page:            1,
+		MinAccessLevel:  20,
+	}
+
+	repositories, err := client.fetchAllProjects(context.Background(), options)
 	if err != nil {
-		return nil, fmt.Errorf("ERROR: making request: %v", err)
+		return nil, fmt.Errorf("fetching repositories: %w", err)
+	}
+
+	if len(repositories) == 0 {
+		return repositories, fmt.Errorf("no repositories found")
+	}
+
+	// update progress bar
+	if err := updateProgressBar(len(repositories)); err != nil {
+		logger.Print("WARNING: failed to update progress bar: "+err.Error(), nil)
+	}
+
+	logger.Print(fmt.Sprintf("Successfully fetched %d repositories", len(repositories)), nil)
+	return repositories, nil
+}
+
+// fetch all repos with pagination
+func (c *GitLabClient) fetchAllProjects(ctx context.Context, options GitLabAPIOptions) ([]Repository, error) {
+	var allRepositories []Repository
+
+	for {
+		gitlabProjects, pagination, err := c.fetchProjectPage(ctx, options)
+		if err != nil {
+			return nil, fmt.Errorf("fetching page %d: %w", options.Page, err)
+		}
+
+		// convert gitlab repositories to repo type
+		repositories := convertGitLabProjects(gitlabProjects, options.IncludeArchived)
+		allRepositories = append(allRepositories, repositories...)
+
+		logger.Print(fmt.Sprintf("Fetched page %d/%d (%d projects)",
+			pagination.CurrentPage, pagination.TotalPages, len(gitlabProjects)), nil)
+
+		// check if we have more pages
+		if pagination.NextPage == 0 || pagination.CurrentPage >= pagination.TotalPages {
+			break
+		}
+
+		options.Page = pagination.NextPage
+	}
+
+	return allRepositories, nil
+}
+
+// fetch single page of repo
+func (c *GitLabClient) fetchProjectPage(ctx context.Context, options GitLabAPIOptions) ([]GitLabProject, GitLabPaginationInfo, error) {
+	apiURL, err := c.buildAPIURL(options)
+	if err != nil {
+		return nil, GitLabPaginationInfo{}, fmt.Errorf("building API URL: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
+	if err != nil {
+		return nil, GitLabPaginationInfo{}, fmt.Errorf("creating request: %w", err)
+	}
+
+	req.Header.Set("PRIVATE-TOKEN", c.token)
+	req.Header.Set("Accept", "application/json")
+
+	logger.Print("Making API request to: "+apiURL, nil)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, GitLabPaginationInfo{}, fmt.Errorf("making request: %w", err)
 	}
 	defer func() {
-		if err := resp.Body.Close(); err != nil {
+		if closeErr := resp.Body.Close(); closeErr != nil {
-			logger.Fatal("HTTP: Error closing response body", err)
+			logger.Print("WARNING: failed to close response body: "+closeErr.Error(), nil)
 		}
 	}()
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("ERROR: API request failed with status: %d", resp.StatusCode)
+		return nil, GitLabPaginationInfo{}, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, resp.Status)
 	}
 
-	logger.Print("HTTP: Decoding JSON response", nil)
+	var gitlabProjects []GitLabProject
-	var repositories []Repository
+	if err := json.NewDecoder(resp.Body).Decode(&gitlabProjects); err != nil {
-	if err := json.NewDecoder(resp.Body).Decode(&repositories); err != nil {
+		return nil, GitLabPaginationInfo{}, fmt.Errorf("decoding JSON response: %w", err)
-		return nil, fmt.Errorf("ERROR: decoding response: %v", err)
 	}
 
-	if len(repositories) < 1 {
+	// check for more pages
-		return repositories, fmt.Errorf("ERROR: no repositories found")
+	pagination := parsePaginationHeaders(resp.Header)
+
+	return gitlabProjects, pagination, nil
 }
 
-	repoCount := len(repositories)
+// build final api url
+func (c *GitLabClient) buildAPIURL(options GitLabAPIOptions) (string, error) {
+	baseURL := fmt.Sprintf("https://%s/api/v4/projects", c.baseURL)
 
-	logger.Print("BAR: Resetting the progressbar", nil)
+	u, err := url.Parse(baseURL)
-	if !config.Debug {
-		err = bar.Set(0)
 	if err != nil {
-			logger.Fatal("Could not reset the progressbar", err)
+		return "", fmt.Errorf("parsing base URL: %w", err)
-		}
 	}
 
-	logger.Print("BAR: Increasing the max value of the progressbar", nil)
+	query := u.Query()
-	if !config.Debug {
+
-		bar.ChangeMax(repoCount)
+	if options.Membership {
+		query.Set("membership", "true")
 	}
 
-	logger.Print("HTTP: Returning repositories found", nil)
+	query.Set("order_by", options.OrderBy)
-	return repositories, nil
+	query.Set("sort", options.Sort)
+	query.Set("per_page", strconv.Itoa(options.PerPage))
+	query.Set("page", strconv.Itoa(options.Page))
+
+	if options.MinAccessLevel > 0 {
+		query.Set("min_access_level", strconv.Itoa(options.MinAccessLevel))
+	}
+
+	// handle archived
+	switch options.IncludeArchived {
+	case "excluded":
+		query.Set("archived", "false")
+	case "only":
+		query.Set("archived", "true")
+		// For "included" or any other value, don't set the archived parameter
+	}
+
+	u.RawQuery = query.Encode()
+	return u.String(), nil
+}
+
+// parse pagination headers
+func parsePaginationHeaders(headers http.Header) GitLabPaginationInfo {
+
+	pagination := GitLabPaginationInfo{}
+
+	if totalPages := headers.Get("X-Total-Pages"); totalPages != "" {
+		pagination.TotalPages, _ = strconv.Atoi(totalPages)
+	}
+
+	if totalItems := headers.Get("X-Total"); totalItems != "" {
+		pagination.TotalItems, _ = strconv.Atoi(totalItems)
+	}
+
+	if currentPage := headers.Get("X-Page"); currentPage != "" {
+		pagination.CurrentPage, _ = strconv.Atoi(currentPage)
+	}
+
+	if nextPage := headers.Get("X-Next-Page"); nextPage != "" {
+		pagination.NextPage, _ = strconv.Atoi(nextPage)
+	}
+
+	if prevPage := headers.Get("X-Prev-Page"); prevPage != "" {
+		pagination.PreviousPage, _ = strconv.Atoi(prevPage)
+	}
+
+	return pagination
+}
+
+// convert gitlab repos to repo type
+func convertGitLabProjects(gitlabProjects []GitLabProject, includeArchived string) []Repository {
+	var repositories []Repository
+
+	for _, project := range gitlabProjects {
+		// Additional filtering based on archived status if needed
+		if includeArchived == "excluded" && project.Archived {
+			continue
+		}
+		if includeArchived == "only" && !project.Archived {
+			continue
+		}
+
+		repositories = append(repositories, Repository{
+			Name:              project.Name,
+			PathWithNamespace: project.PathWithNamespace,
+		})
+	}
+
+	return repositories
+}
+
+// connection validation
+func (c *GitLabClient) ValidateConnection(ctx context.Context) error {
+	apiURL := fmt.Sprintf("https://%s/api/v4/user", c.baseURL)
+
+	req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
+	if err != nil {
+		return fmt.Errorf("creating validation request: %w", err)
+	}
+
+	req.Header.Set("PRIVATE-TOKEN", c.token)
+	req.Header.Set("Accept", "application/json")
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("making validation request: %w", err)
+	}
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			logger.Print("failed to close response body: %v", err)
+		}
+	}()
+
+	if resp.StatusCode == http.StatusUnauthorized {
+		return fmt.Errorf("invalid or expired token")
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("API validation failed with status %d", resp.StatusCode)
+	}
+
+	return nil
+}
+
+// fetch projects group
+func (c *GitLabClient) GetProjectsByGroup(ctx context.Context, groupID string, options GitLabAPIOptions) ([]Repository, error) {
+	var allRepositories []Repository
+
+	for {
+		gitlabProjects, pagination, err := c.fetchGroupProjectPage(ctx, groupID, options)
+		if err != nil {
+			return nil, fmt.Errorf("fetching group page %d: %w", options.Page, err)
+		}
+
+		// Convert GitLab projects to our Repository type
+		repositories := convertGitLabProjects(gitlabProjects, options.IncludeArchived)
+		allRepositories = append(allRepositories, repositories...)
+
+		logger.Print(fmt.Sprintf("Fetched group page %d/%d (%d projects)",
+			pagination.CurrentPage, pagination.TotalPages, len(gitlabProjects)), nil)
+
+		// Check if we have more pages
+		if pagination.NextPage == 0 || pagination.CurrentPage >= pagination.TotalPages {
+			break
+		}
+
+		options.Page = pagination.NextPage
+	}
+
+	return allRepositories, nil
+}
+
+// fetch project page
+func (c *GitLabClient) fetchGroupProjectPage(ctx context.Context, groupID string, options GitLabAPIOptions) ([]GitLabProject, GitLabPaginationInfo, error) {
+	apiURL, err := c.buildGroupAPIURL(groupID, options)
+	if err != nil {
+		return nil, GitLabPaginationInfo{}, fmt.Errorf("building group API URL: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
+	if err != nil {
+		return nil, GitLabPaginationInfo{}, fmt.Errorf("creating request: %w", err)
+	}
+
+	req.Header.Set("PRIVATE-TOKEN", c.token)
+	req.Header.Set("Accept", "application/json")
+
+	logger.Print("Making group API request to: "+apiURL, nil)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, GitLabPaginationInfo{}, fmt.Errorf("making request: %w", err)
+	}
+	defer func() {
+		if closeErr := resp.Body.Close(); closeErr != nil {
+			logger.Print("WARNING: failed to close response body: "+closeErr.Error(), nil)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, GitLabPaginationInfo{}, fmt.Errorf("group API request failed with status %d: %s", resp.StatusCode, resp.Status)
+	}
+
+	var gitlabProjects []GitLabProject
+	if err := json.NewDecoder(resp.Body).Decode(&gitlabProjects); err != nil {
+		return nil, GitLabPaginationInfo{}, fmt.Errorf("decoding JSON response: %w", err)
+	}
+
+	pagination := parsePaginationHeaders(resp.Header)
+
+	return gitlabProjects, pagination, nil
+}
+
+// build api url
+func (c *GitLabClient) buildGroupAPIURL(groupID string, options GitLabAPIOptions) (string, error) {
+	baseURL := fmt.Sprintf("https://%s/api/v4/groups/%s/projects", c.baseURL, groupID)
+
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		return "", fmt.Errorf("parsing base URL: %w", err)
+	}
+
+	query := u.Query()
+	query.Set("per_page", strconv.Itoa(options.PerPage))
+	query.Set("page", strconv.Itoa(options.Page))
+
+	if options.MinAccessLevel > 0 {
+		query.Set("min_access_level", strconv.Itoa(options.MinAccessLevel))
+	}
+
+	// Handle archived parameter
+	switch options.IncludeArchived {
+	case "excluded":
+		query.Set("archived", "false")
+	case "only":
+		query.Set("archived", "true")
+	}
+
+	u.RawQuery = query.Encode()
+	return u.String(), nil
+}
+
+// get project stats
+func (c *GitLabClient) GetProjectStatistics(ctx context.Context) (map[string]int, error) {
+	options := GitLabAPIOptions{
+		Membership: true,
+		PerPage:    100,
+		Page:       1,
+	}
+
+	projects, err := c.fetchAllProjects(ctx, options)
+	if err != nil {
+		return nil, fmt.Errorf("fetching projects for statistics: %w", err)
+	}
+
+	stats := map[string]int{
+		"total":    len(projects),
+		"archived": 0,
+		"active":   0,
+	}
+
+	stats["active"] = len(projects)
+
+	return stats, nil
 }

cmd/gogitlabber/main.go

@@ -56,7 +56,7 @@ func main() {
 			logger.Fatal("Fetching repositories failed", err)
 		}
 	case "gitlab":
-		repositories, err = fetchRepositoriesGitlab()
+		repositories, err = FetchRepositoriesGitLab()
 		if err != nil {
 			logger.Fatal("Fetching repositories failed", err)
 		}