From f9aed5bf320400c8b42d7fda9ceadb921090928f Mon Sep 17 00:00:00 2001 From: Hau Nguyen Date: Tue, 15 Aug 2023 19:18:48 +0700 Subject: [PATCH] feat: use conc pool to speed up --- go.mod | 3 ++ go.sum | 11 +++++ internal/cli/action_overlook.go | 81 ++++++++++++++++++++------------- 3 files changed, 64 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 28b694a..421f6d2 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/google/go-github/v53 v53.2.0 github.com/make-go-great/color-go v0.5.0 github.com/make-go-great/netrc-go v0.0.2 + github.com/sourcegraph/conc v0.3.0 github.com/urfave/cli/v2 v2.25.7 golang.org/x/oauth2 v0.8.0 ) @@ -21,6 +22,8 @@ require ( github.com/mattn/go-isatty v0.0.17 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.7.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect diff --git a/go.sum b/go.sum index 1b476de..6625a82 100644 --- a/go.sum +++ b/go.sum @@ -6,7 +6,9 @@ github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEM github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -30,13 +32,22 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= diff --git a/internal/cli/action_overlook.go b/internal/cli/action_overlook.go index fc27881..ae2b8c4 100644 --- a/internal/cli/action_overlook.go +++ b/internal/cli/action_overlook.go @@ -1,16 +1,21 @@ package cli import ( + "context" "fmt" "os" "regexp" "sort" + "sync" "text/tabwriter" "time" + "github.com/sourcegraph/conc/pool" "github.com/urfave/cli/v2" ) +const maxPoolGoroutine = 8 + var reGitHub = regexp.MustCompile(`github\.com/([^/]*)/([^/]*)`) type GitHubRepoData struct { @@ -40,47 +45,61 @@ func (a *action) Overlook(c *cli.Context) error { // To avoid duplicate mGHRepoData := make(map[string]struct{}) + p := pool.New().WithMaxGoroutines(maxPoolGoroutine) + var mMutex sync.RWMutex + var listMutex sync.Mutex for module := range mapImportedModules { - if !reGitHub.MatchString(module) { - continue - } + module := module + p.Go(func() { + if !reGitHub.MatchString(module) { + return + } - parts := reGitHub.FindStringSubmatch(module) - if len(parts) != 3 { - continue - } + parts := reGitHub.FindStringSubmatch(module) + if len(parts) != 3 { + return + } - ghRepoName := parts[0] - if _, ok := mGHRepoData[ghRepoName]; ok { - continue - } - mGHRepoData[ghRepoName] = struct{}{} + ghRepoName := parts[0] + mMutex.RLock() + if _, ok := mGHRepoData[ghRepoName]; ok { + mMutex.RUnlock() + return + } + mMutex.RUnlock() - owner := parts[1] - repo := parts[2] + mMutex.Lock() + mGHRepoData[ghRepoName] = struct{}{} + mMutex.Unlock() - ghRepo, _, err := a.ghClient.Repositories.Get(c.Context, owner, repo) - if err != nil { - a.log("Failed to get GitHub %s/%s: %s\n", owner, repo, err) - } + owner := parts[1] + repo := parts[2] - var ghStar int - if ghRepo.StargazersCount != nil { - ghStar = *ghRepo.StargazersCount - } + ghRepo, _, err := a.ghClient.Repositories.Get(context.Background(), owner, repo) + if err != nil { + a.log("Failed to get GitHub %s/%s: %s\n", owner, repo, err) + } - var ghUpdatedAt time.Time - if ghRepo.UpdatedAt != nil { - ghUpdatedAt = ghRepo.UpdatedAt.Time - } + var ghStar int + if ghRepo.StargazersCount != nil { + ghStar = *ghRepo.StargazersCount + } - listGHRepoData = append(listGHRepoData, GitHubRepoData{ - UpdatedAt: ghUpdatedAt, - Name: ghRepoName, - Star: ghStar, + var ghUpdatedAt time.Time + if ghRepo.UpdatedAt != nil { + ghUpdatedAt = ghRepo.UpdatedAt.Time + } + + listMutex.Lock() + listGHRepoData = append(listGHRepoData, GitHubRepoData{ + UpdatedAt: ghUpdatedAt, + Name: ghRepoName, + Star: ghStar, + }) + listMutex.Unlock() }) - } + p.Wait() // Sort for consistency sort.Slice(listGHRepoData, func(i, j int) bool {