tools/docs2wiki: tool that rewrites local .md file urls with wiki links
Unlike markdown style links for local md file references, wiki links
need to be the url of the wiki pages. In GitHub wiki, links to local
md files of file.md form send users to in the local disk
and wiki links are different. In GitHub Wiki, if links to local md files
send users to raw.github.com pages that serve the raw text content.
https://stackoverflow.com/questions/34804531/in-github-wiki-absolute-links-to-markdown-leads-to-raw-githubusercontent
To avoid this, rewrite local md file uris included in all the md files
in dir.
And, call this from github action wiki.yml workflow before pushing
to the wiki repo.
Updates golang/vscode-go#2094
Change-Id: Iaeb756a4c14cd98544a1a58ace859b6f48a358ce
Reviewed-on: https://go-review.googlesource.com/c/vscode-go/+/401194
Run-TryBot: Hyang-Ah Hana Kim <hyangah@gmail.com>
Reviewed-by: Jamal Carvalho <jamal@golang.org>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Suzy Mueller <suzmue@golang.org>
diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml
index 1390915..a2848f2 100644
--- a/.github/workflows/wiki.yml
+++ b/.github/workflows/wiki.yml
@@ -26,8 +26,13 @@
with:
repository: ${{github.repository}}.wiki
path: wiki
+ - name: Setup Go
+ uses: actions/setup-go@v2
- name: Push to wiki
run: |
+ cd vscode-go
+ go run ./tools/docs2wiki -w ./docs
+ cd ..
cd wiki
diff -ruN --exclude=.git . ../vscode-go/docs > ../mypatch || patch -p3 -E -f < ../mypatch
git config --local user.email "action@github.com"
diff --git a/go.mod b/go.mod
index 8e50f37..c4861c0 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,10 @@
go 1.16
require (
+ github.com/google/go-cmp v0.5.7
github.com/stamblerre/work-stats v0.0.0-20211013195910-92098c96a21a
golang.org/x/build v0.0.0-20211222221018-ee978b38c739
+ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
+ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect
+ golang.org/x/text v0.3.7 // indirect
)
diff --git a/go.sum b/go.sum
index bb7d945..241b301 100644
--- a/go.sum
+++ b/go.sum
@@ -302,8 +302,9 @@
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM=
@@ -636,8 +637,6 @@
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/stamblerre/work-stats v0.0.0-20210726215650-a14fc877fae7 h1:yZLmB9oK2DoOIQ/62SSC4PVg0LtYTXvsak9cW+O2umw=
-github.com/stamblerre/work-stats v0.0.0-20210726215650-a14fc877fae7/go.mod h1:dM4zJ9OuZuchdonBFCaFef0ZAnZuuCgX4WDLlUm6+RM=
github.com/stamblerre/work-stats v0.0.0-20211013195910-92098c96a21a h1:p2dmgMcxN88UcGsvwD0GjDAEfjFZGjDdvpuT+O3YV70=
github.com/stamblerre/work-stats v0.0.0-20211013195910-92098c96a21a/go.mod h1:dM4zJ9OuZuchdonBFCaFef0ZAnZuuCgX4WDLlUm6+RM=
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
@@ -810,8 +809,9 @@
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8=
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20170207211851-4464e7848382/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -919,8 +919,9 @@
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -930,8 +931,9 @@
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
diff --git a/tools/docs2wiki/main.go b/tools/docs2wiki/main.go
new file mode 100644
index 0000000..43b35c0
--- /dev/null
+++ b/tools/docs2wiki/main.go
@@ -0,0 +1,117 @@
+//go:build !windows
+// +build !windows
+
+// Tool docs2wiki rewrites links in ./docs/* to wiki link format.
+// This program may call the 'diff' tool which may be missing on Windows.
+package main
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "io/fs"
+ "io/ioutil"
+ "net/url"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+)
+
+var writeFlag = flag.Bool("w", false, "Overwrite new file contents to disk.")
+
+func main() {
+ flag.Parse()
+
+ if len(flag.Args()) != 1 {
+ errorf("Usage: %v <dir>", os.Args[0])
+ os.Exit(1)
+ }
+ if err := rewriteLinks(flag.Arg(0), *writeFlag); err != nil {
+ errorf("failed to rewrite links: %v", err)
+ os.Exit(1)
+ }
+}
+
+func rewriteLinks(dir string, overwrite bool) error {
+ return filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ return nil
+ }
+ name := d.Name()
+ if filepath.Ext(name) != ".md" {
+ return nil
+ }
+
+ errorf("processing %v... %v", name, path)
+
+ data, err := ioutil.ReadFile(path)
+ if err != nil {
+ return fmt.Errorf("failed to read file %v: %w", name, err)
+ }
+ converted := markdownLink2WikiLink(data)
+ if overwrite {
+ return ioutil.WriteFile(path, converted, 0644)
+ }
+ tmp, err := writeToTempFile(converted)
+ if err != nil {
+ return fmt.Errorf("failed to write to temp file for diff: %w", err)
+ }
+ defer os.Remove(tmp)
+ diff(path, tmp)
+ return nil
+ })
+}
+
+func diff(f1, f2 string) {
+ cmd := exec.Command("diff", f1, f2)
+ cmd.Stderr = os.Stderr
+ cmd.Stdout = os.Stdout
+ if err := cmd.Run(); err != nil {
+ errorf("failed diff %v %v: %v", f1, f2, err)
+ }
+}
+
+func writeToTempFile(content []byte) (filename string, err error) {
+ dst, err := ioutil.TempFile("", "tmp")
+ if err != nil {
+ return "", fmt.Errorf("failed to write to a temporary file for diff: %v", err)
+ }
+ defer func() {
+ if err == nil {
+ err = dst.Close()
+ }
+ }()
+
+ dst.Write(content)
+ return dst.Name(), nil
+}
+
+// find pattern like '](link.md)'
+var markdownLinkRE = regexp.MustCompile(`\]\(\S+(:?\.md|\.md#[^)]*)\)`)
+
+func markdownLink2WikiLink(src []byte) []byte {
+ return markdownLinkRE.ReplaceAllFunc(src, func(s []byte) []byte {
+
+ part := string(s[2 : len(s)-1]) // remove leading `](` and ending `)`
+ u, err := url.Parse(part)
+ if err != nil {
+ return s
+ }
+ if u.Scheme != "" {
+ return s
+ }
+ u.Path = strings.TrimSuffix(u.Path, ".md")
+ b := &bytes.Buffer{}
+ fmt.Fprintf(b, "](%s)", u.String())
+ return b.Bytes()
+ })
+}
+
+func errorf(format string, a ...interface{}) {
+ fmt.Fprintf(os.Stderr, format+"\n", a...)
+}
diff --git a/tools/docs2wiki/main_test.go b/tools/docs2wiki/main_test.go
new file mode 100644
index 0000000..d55bbdb
--- /dev/null
+++ b/tools/docs2wiki/main_test.go
@@ -0,0 +1,83 @@
+//go:build !windows
+// +build !windows
+
+// Tool docs2wiki rewrites links in ./docs/* to wiki link format.
+package main
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func TestRewriteLinks(t *testing.T) {
+ for _, tt := range []struct{ filename, in, want string }{
+ {filename: "doc.md", in: markdownStyle, want: wikiStyle},
+ {filename: "sub/doc.md", in: markdownStyle, want: wikiStyle},
+ {filename: "doc.txt", in: markdownStyle, want: markdownStyle},
+ } {
+ t.Run(tt.filename, func(t *testing.T) {
+ // prepareTestData writes tt.in to tt.filename.
+ dir := prepareTestData(t, tt.filename, tt.in)
+ defer os.RemoveAll(dir)
+
+ // Use overwrite=true so `rewriteLinks` overwrite the original file
+ // which we will read back for comparison against tt.want.
+ // With overwrite=false, rewriteLinks just prints out the diff
+ // which will be difficult to test.
+ err := rewriteLinks(dir, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // rewriteLinks overwrites the original file,
+ // so reread the content for comparison.
+ got, err := ioutil.ReadFile(filepath.Join(dir, tt.filename))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if diff := cmp.Diff(tt.want, string(got)); diff != "" {
+ t.Errorf("(-want +got): %v", diff)
+ }
+ })
+ }
+}
+
+// prepareTestData writes a file in a temp directory and returns the temp
+// directory path.
+func prepareTestData(t *testing.T, file, content string) (dir string) {
+ dir, err := ioutil.TempDir("", "docs2wiki_test")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fname := filepath.Join(dir, filepath.FromSlash(file))
+ os.MkdirAll(filepath.Dir(fname), 0755) // create intermediate dirs
+ if err := ioutil.WriteFile(fname, []byte(content), 0644); err != nil {
+ os.RemoveAll(dir)
+ t.Fatal(err)
+ }
+ return dir
+}
+
+var markdownStyle = `
+[This changes](doc.md)
+ [This changes too](./doc.md)
+ [This also changes](foo/doc.md)
+[Fragment works](foo.md#this-is-a-title)
+[A](doc.md) [B](doc.md) [C](doc.txt)
+
+[This doesn't change](https://go.dev/foo.md)
+[Untouchable.md](foo)`
+
+var wikiStyle = `
+[This changes](doc)
+ [This changes too](./doc)
+ [This also changes](foo/doc)
+[Fragment works](foo#this-is-a-title)
+[A](doc) [B](doc) [C](doc.txt)
+
+[This doesn't change](https://go.dev/foo.md)
+[Untouchable.md](foo)`
