feat: use conc pool to speed up
parent
e1b77493c6
commit
f9aed5bf32
3
go.mod
3
go.mod
|
@ -6,6 +6,7 @@ require (
|
||||||
github.com/google/go-github/v53 v53.2.0
|
github.com/google/go-github/v53 v53.2.0
|
||||||
github.com/make-go-great/color-go v0.5.0
|
github.com/make-go-great/color-go v0.5.0
|
||||||
github.com/make-go-great/netrc-go v0.0.2
|
github.com/make-go-great/netrc-go v0.0.2
|
||||||
|
github.com/sourcegraph/conc v0.3.0
|
||||||
github.com/urfave/cli/v2 v2.25.7
|
github.com/urfave/cli/v2 v2.25.7
|
||||||
golang.org/x/oauth2 v0.8.0
|
golang.org/x/oauth2 v0.8.0
|
||||||
)
|
)
|
||||||
|
@ -21,6 +22,8 @@ require (
|
||||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||||
|
go.uber.org/atomic v1.7.0 // indirect
|
||||||
|
go.uber.org/multierr v1.9.0 // indirect
|
||||||
golang.org/x/crypto v0.7.0 // indirect
|
golang.org/x/crypto v0.7.0 // indirect
|
||||||
golang.org/x/net v0.10.0 // indirect
|
golang.org/x/net v0.10.0 // indirect
|
||||||
golang.org/x/sys v0.8.0 // indirect
|
golang.org/x/sys v0.8.0 // indirect
|
||||||
|
|
11
go.sum
11
go.sum
|
@ -6,7 +6,9 @@ github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEM
|
||||||
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
|
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
|
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
|
||||||
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
|
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
|
||||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
@ -30,13 +32,22 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
|
||||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
|
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||||
|
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||||
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
|
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
|
||||||
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
|
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
|
||||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
|
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
|
||||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
|
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
|
||||||
|
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||||
|
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||||
|
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||||
|
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
|
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
|
||||||
|
|
|
@ -1,16 +1,21 @@
|
||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
|
"sync"
|
||||||
"text/tabwriter"
|
"text/tabwriter"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/sourcegraph/conc/pool"
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const maxPoolGoroutine = 8
|
||||||
|
|
||||||
var reGitHub = regexp.MustCompile(`github\.com/([^/]*)/([^/]*)`)
|
var reGitHub = regexp.MustCompile(`github\.com/([^/]*)/([^/]*)`)
|
||||||
|
|
||||||
type GitHubRepoData struct {
|
type GitHubRepoData struct {
|
||||||
|
@ -40,47 +45,61 @@ func (a *action) Overlook(c *cli.Context) error {
|
||||||
// To avoid duplicate
|
// To avoid duplicate
|
||||||
mGHRepoData := make(map[string]struct{})
|
mGHRepoData := make(map[string]struct{})
|
||||||
|
|
||||||
|
p := pool.New().WithMaxGoroutines(maxPoolGoroutine)
|
||||||
|
var mMutex sync.RWMutex
|
||||||
|
var listMutex sync.Mutex
|
||||||
for module := range mapImportedModules {
|
for module := range mapImportedModules {
|
||||||
if !reGitHub.MatchString(module) {
|
module := module
|
||||||
continue
|
p.Go(func() {
|
||||||
}
|
if !reGitHub.MatchString(module) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
parts := reGitHub.FindStringSubmatch(module)
|
parts := reGitHub.FindStringSubmatch(module)
|
||||||
if len(parts) != 3 {
|
if len(parts) != 3 {
|
||||||
continue
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ghRepoName := parts[0]
|
ghRepoName := parts[0]
|
||||||
if _, ok := mGHRepoData[ghRepoName]; ok {
|
mMutex.RLock()
|
||||||
continue
|
if _, ok := mGHRepoData[ghRepoName]; ok {
|
||||||
}
|
mMutex.RUnlock()
|
||||||
mGHRepoData[ghRepoName] = struct{}{}
|
return
|
||||||
|
}
|
||||||
|
mMutex.RUnlock()
|
||||||
|
|
||||||
owner := parts[1]
|
mMutex.Lock()
|
||||||
repo := parts[2]
|
mGHRepoData[ghRepoName] = struct{}{}
|
||||||
|
mMutex.Unlock()
|
||||||
|
|
||||||
ghRepo, _, err := a.ghClient.Repositories.Get(c.Context, owner, repo)
|
owner := parts[1]
|
||||||
if err != nil {
|
repo := parts[2]
|
||||||
a.log("Failed to get GitHub %s/%s: %s\n", owner, repo, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var ghStar int
|
ghRepo, _, err := a.ghClient.Repositories.Get(context.Background(), owner, repo)
|
||||||
if ghRepo.StargazersCount != nil {
|
if err != nil {
|
||||||
ghStar = *ghRepo.StargazersCount
|
a.log("Failed to get GitHub %s/%s: %s\n", owner, repo, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var ghUpdatedAt time.Time
|
var ghStar int
|
||||||
if ghRepo.UpdatedAt != nil {
|
if ghRepo.StargazersCount != nil {
|
||||||
ghUpdatedAt = ghRepo.UpdatedAt.Time
|
ghStar = *ghRepo.StargazersCount
|
||||||
}
|
}
|
||||||
|
|
||||||
listGHRepoData = append(listGHRepoData, GitHubRepoData{
|
var ghUpdatedAt time.Time
|
||||||
UpdatedAt: ghUpdatedAt,
|
if ghRepo.UpdatedAt != nil {
|
||||||
Name: ghRepoName,
|
ghUpdatedAt = ghRepo.UpdatedAt.Time
|
||||||
Star: ghStar,
|
}
|
||||||
|
|
||||||
|
listMutex.Lock()
|
||||||
|
listGHRepoData = append(listGHRepoData, GitHubRepoData{
|
||||||
|
UpdatedAt: ghUpdatedAt,
|
||||||
|
Name: ghRepoName,
|
||||||
|
Star: ghStar,
|
||||||
|
})
|
||||||
|
listMutex.Unlock()
|
||||||
})
|
})
|
||||||
|
|
||||||
}
|
}
|
||||||
|
p.Wait()
|
||||||
|
|
||||||
// Sort for consistency
|
// Sort for consistency
|
||||||
sort.Slice(listGHRepoData, func(i, j int) bool {
|
sort.Slice(listGHRepoData, func(i, j int) bool {
|
||||||
|
|
Loading…
Reference in New Issue