gosrc: use Bitbucket v2 api
Bitbucket Cloud REST API version 1 is deprecated effective
30 June 2018. All 1.0 APIs will be removed from the REST API permanently
on 29 April 2019.
Since these are APIs used in gosrc/bitbucket.go, this CL aims to migrate
to the version 2 APIs. Most things translated well: all references to
/1.0/ were replaced with /2.0/ and json.Marshal structs were updated.
Note, many v2 APIs replace maps with arrays.
Pagination is present in many v2 APIs requiring iteration of some APIs:
refs, watchers, and src. pagelen has been set to max, 100, to minimize
number of API calls, but this could lead to context timeouts.
Timestamp formats have moved to time.RFC3339, thus all bespoke layout
vars have been removed.
/2.0/repositories/{username}/{repo_slug}/refs can directly fetch both
branches and tags, thus that logic has been collapsed.
/2.0/repositories/{username}/{repo_slug}.parent takes the place of both
/1.0/repositories/{username}/{repo_slug}.is_fork and
/1.0/repositories/{username}/{repo_slug}.fork_of. It is left as an
interface{} since the contents are not used.
gosrc.bitbucketRefs.Values[].Target.Hash, previously
gosrc.bitbucketNode.Node, stores the full SHA1 hash, not the first 12
chars. It does not appear to break anything to use the full hash, so
that is done.
/2.0/repositories/{username}/{repo_slug}/src interleaves files and
folders, so new processing logic was required.
/1.0/repositories/{username}/{repo_slug}/raw has been deprecated in
favour of directly using /2.0/repositories/{username}/{repo_slug}/src.
Followers, now called watchers, and are not accessable via the root
/2.0/repositories/{username}/{repo_slug} API. They have their own
paginated api, /2.0/repositories/{username}/{repo_slug}/watchers, for
fetching detailed user information. Enumerating watchers will require at
least one API call, and testing indicates that this API does not agree
with the value of /1.0/repositories/{username}/{repo_slug}.followers_count
or the Web UI, which agree with each other. While this feels like a bug
on Bitbucket's end, it was decided that gosrc.Directory.Stars should be
left default. The godoc for gosrc.Directory has been updated to reflect
this change.
Fixes golang/gddo#599
Change-Id: I8c471efb32a49c9aae179f7174e018357f93b9df
Reviewed-on: https://go-review.googlesource.com/c/gddo/+/170279
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
diff --git a/gosrc/bitbucket.go b/gosrc/bitbucket.go
index 8435148..1c7fe60 100644
--- a/gosrc/bitbucket.go
+++ b/gosrc/bitbucket.go
@@ -26,19 +26,33 @@
var bitbucketEtagRe = regexp.MustCompile(`^(hg|git)-`)
type bitbucketRepo struct {
- Scm string
- CreatedOn string `json:"created_on"`
- LastUpdated string `json:"last_updated"`
- ForkOf struct {
- Scm string
- } `json:"fork_of"`
- Followers int `json:"followers_count"`
- IsFork bool `json:"is_fork"`
+ Scm string `json:"scm"`
+ CreatedOn string `json:"created_on"`
+ UpdatedOn string `json:"updated_on"`
+ Parent interface{} `json:"parent"`
}
-type bitbucketNode struct {
- Node string `json:"node"`
- Timestamp string `json:"utctimestamp"`
+type bitbucketRefs struct {
+ Values []struct {
+ Name string `json:"name"`
+ Target struct {
+ Date string `json:"date"`
+ Hash string `json:"hash"`
+ } `json:"target"`
+ } `json:"values"`
+ bitbucketPage
+}
+
+type bitbucketSrc struct {
+ Values []struct {
+ Path string `json:"path"`
+ Type string `json:"type"`
+ } `json:"values"`
+ bitbucketPage
+}
+
+type bitbucketPage struct {
+ Next string `json:"next",omitempty`
}
func getBitbucketDir(ctx context.Context, client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
@@ -59,21 +73,25 @@
tags := make(map[string]string)
timestamps := make(map[string]time.Time)
- for _, nodeType := range []string{"branches", "tags"} {
- var nodes map[string]bitbucketNode
- if _, err := c.getJSON(ctx, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/{0}", match, nodeType), &nodes); err != nil {
+ url := expand("https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/refs?pagelen=100", match)
+ for {
+ var refs bitbucketRefs
+ if _, err := c.getJSON(ctx, url, &refs); err != nil {
return nil, err
}
- for t, n := range nodes {
- tags[t] = n.Node
- const timeFormat = "2006-01-02 15:04:05Z07:00"
- committed, err := time.Parse(timeFormat, n.Timestamp)
+ for _, v := range refs.Values {
+ tags[v.Name] = v.Target.Hash
+ committed, err := time.Parse(time.RFC3339, v.Target.Date)
if err != nil {
- log.Println("error parsing timestamp:", n.Timestamp)
+ log.Println("error parsing timestamp:", v.Target.Date)
continue
}
- timestamps[t] = committed
+ timestamps[v.Name] = committed
}
+ if refs.Next == "" {
+ break
+ }
+ url = refs.Next
}
var err error
@@ -95,26 +113,33 @@
}
}
- var contents struct {
- Directories []string
- Files []struct {
- Path string
- }
- }
-
- if _, err := c.getJSON(ctx, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/src/{tag}{dir}/", match), &contents); err != nil {
- return nil, err
- }
-
+ var dirs []string
var files []*File
var dataURLs []string
- for _, f := range contents.Files {
- _, name := path.Split(f.Path)
- if isDocFile(name) {
- files = append(files, &File{Name: name, BrowseURL: expand("https://bitbucket.org/{owner}/{repo}/src/{tag}/{0}", match, f.Path)})
- dataURLs = append(dataURLs, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/raw/{tag}/{0}", match, f.Path))
+ url = expand("https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/src/{tag}{dir}/?pagelen=100", match)
+ for {
+ var contents bitbucketSrc
+ if _, err := c.getJSON(ctx, url, &contents); err != nil {
+ return nil, err
}
+
+ for _, v := range contents.Values {
+ switch v.Type {
+ case "commit_file":
+ _, name := path.Split(v.Path)
+ if isDocFile(name) {
+ files = append(files, &File{Name: name, BrowseURL: expand("https://bitbucket.org/{owner}/{repo}/src/{tag}/{0}", match, v.Path)})
+ dataURLs = append(dataURLs, expand("https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/src/{tag}/{0}", match, v.Path))
+ }
+ case "commit_directory":
+ dirs = append(dirs, v.Path)
+ }
+ }
+ if contents.Next == "" {
+ break
+ }
+ url = contents.Next
}
if err := c.getFiles(ctx, dataURLs, files); err != nil {
@@ -134,17 +159,16 @@
ProjectName: match["repo"],
ProjectRoot: expand("bitbucket.org/{owner}/{repo}", match),
ProjectURL: expand("https://bitbucket.org/{owner}/{repo}/", match),
- Subdirectories: contents.Directories,
+ Subdirectories: dirs,
VCS: match["vcs"],
Status: status,
- Fork: repo.IsFork,
- Stars: repo.Followers,
+ Fork: repo.Parent != nil,
}, nil
}
func getBitbucketRepo(ctx context.Context, c *httpClient, match map[string]string) (*bitbucketRepo, error) {
var repo bitbucketRepo
- if _, err := c.getJSON(ctx, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}", match), &repo); err != nil {
+ if _, err := c.getJSON(ctx, expand("https://api.bitbucket.org/2.0/repositories/{owner}/{repo}", match), &repo); err != nil {
return nil, err
}
@@ -152,19 +176,18 @@
}
func isBitbucketDeadEndFork(repo *bitbucketRepo) bool {
- l := "2006-01-02T15:04:05.999999999"
- created, err := time.Parse(l, repo.CreatedOn)
+ created, err := time.Parse(time.RFC3339, repo.CreatedOn)
if err != nil {
return false
}
- updated, err := time.Parse(l, repo.LastUpdated)
+ updated, err := time.Parse(time.RFC3339, repo.UpdatedOn)
if err != nil {
return false
}
isDeadEndFork := false
- if repo.ForkOf.Scm != "" && created.Unix() >= updated.Unix() {
+ if repo.Parent != nil && created.Unix() >= updated.Unix() {
isDeadEndFork = true
}
diff --git a/gosrc/gosrc.go b/gosrc/gosrc.go
index d1a0be8..4c85fe9 100644
--- a/gosrc/gosrc.go
+++ b/gosrc/gosrc.go
@@ -98,8 +98,7 @@
// Whether the repository of this directory is a fork of another one.
Fork bool
- // How many stars (for a GitHub project) or followers (for a BitBucket
- // project) the repository of this directory has.
+ // How many stars (for a GitHub project) the repository of this directory has.
Stars int
}
