feat: use conc pool to speed up
parent
e1b77493c6
commit
f9aed5bf32
3
go.mod
3
go.mod
|
|
@ -6,6 +6,7 @@ require (
|
|||
github.com/google/go-github/v53 v53.2.0
|
||||
github.com/make-go-great/color-go v0.5.0
|
||||
github.com/make-go-great/netrc-go v0.0.2
|
||||
github.com/sourcegraph/conc v0.3.0
|
||||
github.com/urfave/cli/v2 v2.25.7
|
||||
golang.org/x/oauth2 v0.8.0
|
||||
)
|
||||
|
|
@ -21,6 +22,8 @@ require (
|
|||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
golang.org/x/crypto v0.7.0 // indirect
|
||||
golang.org/x/net v0.10.0 // indirect
|
||||
golang.org/x/sys v0.8.0 // indirect
|
||||
|
|
|
|||
11
go.sum
11
go.sum
|
|
@ -6,7 +6,9 @@ github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEM
|
|||
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
|
||||
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
|
|
@ -30,13 +32,22 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
|
|||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
|
||||
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
|
||||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
|
||||
|
|
|
|||
|
|
@ -1,16 +1,21 @@
|
|||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"sync"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/sourcegraph/conc/pool"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
const maxPoolGoroutine = 8
|
||||
|
||||
var reGitHub = regexp.MustCompile(`github\.com/([^/]*)/([^/]*)`)
|
||||
|
||||
type GitHubRepoData struct {
|
||||
|
|
@ -40,47 +45,61 @@ func (a *action) Overlook(c *cli.Context) error {
|
|||
// To avoid duplicate
|
||||
mGHRepoData := make(map[string]struct{})
|
||||
|
||||
p := pool.New().WithMaxGoroutines(maxPoolGoroutine)
|
||||
var mMutex sync.RWMutex
|
||||
var listMutex sync.Mutex
|
||||
for module := range mapImportedModules {
|
||||
if !reGitHub.MatchString(module) {
|
||||
continue
|
||||
}
|
||||
module := module
|
||||
p.Go(func() {
|
||||
if !reGitHub.MatchString(module) {
|
||||
return
|
||||
}
|
||||
|
||||
parts := reGitHub.FindStringSubmatch(module)
|
||||
if len(parts) != 3 {
|
||||
continue
|
||||
}
|
||||
parts := reGitHub.FindStringSubmatch(module)
|
||||
if len(parts) != 3 {
|
||||
return
|
||||
}
|
||||
|
||||
ghRepoName := parts[0]
|
||||
if _, ok := mGHRepoData[ghRepoName]; ok {
|
||||
continue
|
||||
}
|
||||
mGHRepoData[ghRepoName] = struct{}{}
|
||||
ghRepoName := parts[0]
|
||||
mMutex.RLock()
|
||||
if _, ok := mGHRepoData[ghRepoName]; ok {
|
||||
mMutex.RUnlock()
|
||||
return
|
||||
}
|
||||
mMutex.RUnlock()
|
||||
|
||||
owner := parts[1]
|
||||
repo := parts[2]
|
||||
mMutex.Lock()
|
||||
mGHRepoData[ghRepoName] = struct{}{}
|
||||
mMutex.Unlock()
|
||||
|
||||
ghRepo, _, err := a.ghClient.Repositories.Get(c.Context, owner, repo)
|
||||
if err != nil {
|
||||
a.log("Failed to get GitHub %s/%s: %s\n", owner, repo, err)
|
||||
}
|
||||
owner := parts[1]
|
||||
repo := parts[2]
|
||||
|
||||
var ghStar int
|
||||
if ghRepo.StargazersCount != nil {
|
||||
ghStar = *ghRepo.StargazersCount
|
||||
}
|
||||
ghRepo, _, err := a.ghClient.Repositories.Get(context.Background(), owner, repo)
|
||||
if err != nil {
|
||||
a.log("Failed to get GitHub %s/%s: %s\n", owner, repo, err)
|
||||
}
|
||||
|
||||
var ghUpdatedAt time.Time
|
||||
if ghRepo.UpdatedAt != nil {
|
||||
ghUpdatedAt = ghRepo.UpdatedAt.Time
|
||||
}
|
||||
var ghStar int
|
||||
if ghRepo.StargazersCount != nil {
|
||||
ghStar = *ghRepo.StargazersCount
|
||||
}
|
||||
|
||||
listGHRepoData = append(listGHRepoData, GitHubRepoData{
|
||||
UpdatedAt: ghUpdatedAt,
|
||||
Name: ghRepoName,
|
||||
Star: ghStar,
|
||||
var ghUpdatedAt time.Time
|
||||
if ghRepo.UpdatedAt != nil {
|
||||
ghUpdatedAt = ghRepo.UpdatedAt.Time
|
||||
}
|
||||
|
||||
listMutex.Lock()
|
||||
listGHRepoData = append(listGHRepoData, GitHubRepoData{
|
||||
UpdatedAt: ghUpdatedAt,
|
||||
Name: ghRepoName,
|
||||
Star: ghStar,
|
||||
})
|
||||
listMutex.Unlock()
|
||||
})
|
||||
|
||||
}
|
||||
p.Wait()
|
||||
|
||||
// Sort for consistency
|
||||
sort.Slice(listGHRepoData, func(i, j int) bool {
|
||||
|
|
|
|||
Loading…
Reference in New Issue