blob: fa289534aca82bac9dd28433f20b20a5d1ecd815 [file] [log] [blame]
Marcel van Lohuizen731229f2015-03-03 14:02:25 +01001// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// Package gen contains common code for the various code generation tools in the
6// text repository. Its usage ensures consistency between tools.
7//
8// This package defines command line flags that are common to most generation
9// tools. The flags allow for specifying specific Unicode and CLDR versions
Kevin Burke647d7ef2018-08-04 08:55:54 -070010// in the public Unicode data repository (https://www.unicode.org/Public).
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010011//
12// A local Unicode data mirror can be set through the flag -local or the
13// environment variable UNICODE_DIR. The former takes precedence. The local
14// directory should follow the same structure as the public repository.
15//
16// IANA data can also optionally be mirrored by putting it in the iana directory
17// rooted at the top of the local mirror. Beware, though, that IANA data is not
18// versioned. So it is up to the developer to use the right version.
Hyang-Ah Hana Kim75c37a92015-11-04 17:24:59 -050019package gen // import "golang.org/x/text/internal/gen"
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010020
21import (
Marcel van Lohuizen54db2312015-11-24 19:16:13 +010022 "bytes"
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010023 "flag"
24 "fmt"
Marcel van Lohuizen17805e72016-09-14 22:31:49 +020025 "go/build"
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010026 "go/format"
27 "io"
Marcel van Lohuizen54db2312015-11-24 19:16:13 +010028 "io/ioutil"
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010029 "log"
30 "net/http"
31 "os"
32 "path"
33 "path/filepath"
Marcel van Lohuizenfe223c52018-12-15 17:54:12 +010034 "regexp"
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +010035 "strings"
Marcel van Lohuizen17805e72016-09-14 22:31:49 +020036 "sync"
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010037 "unicode"
38
Marcel van Lohuizenedeeb432015-12-05 13:06:26 +010039 "golang.org/x/text/unicode/cldr"
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010040)
41
42var (
43 url = flag.String("url",
Kevin Burke647d7ef2018-08-04 08:55:54 -070044 "https://www.unicode.org/Public",
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010045 "URL of Unicode database directory")
46 iana = flag.String("iana",
47 "http://www.iana.org",
48 "URL of the IANA repository")
Marcel van Lohuizencee5b802015-05-04 15:11:47 +020049 unicodeVersion = flag.String("unicode",
50 getEnv("UNICODE_VERSION", unicode.Version),
51 "unicode version to use")
52 cldrVersion = flag.String("cldr",
53 getEnv("CLDR_VERSION", cldr.Version),
54 "cldr version to use")
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010055)
56
Marcel van Lohuizencee5b802015-05-04 15:11:47 +020057func getEnv(name, def string) string {
58 if v := os.Getenv(name); v != "" {
59 return v
60 }
61 return def
62}
63
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010064// Init performs common initialization for a gen command. It parses the flags
65// and sets up the standard logging parameters.
66func Init() {
67 log.SetPrefix("")
68 log.SetFlags(log.Lshortfile)
69 flag.Parse()
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010070}
71
Marcel van Lohuizenfc7fa092017-03-23 11:04:54 +010072const header = `// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010073
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010074`
75
Marcel van Lohuizend1927f62015-03-10 13:01:21 +090076// UnicodeVersion reports the requested Unicode version.
77func UnicodeVersion() string {
78 return *unicodeVersion
79}
80
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +010081// CLDRVersion reports the requested CLDR version.
Marcel van Lohuizend1927f62015-03-10 13:01:21 +090082func CLDRVersion() string {
83 return *cldrVersion
84}
Marcel van Lohuizen731229f2015-03-03 14:02:25 +010085
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +010086var tags = []struct{ version, buildTags string }{
Marcel van Lohuizenfe223c52018-12-15 17:54:12 +010087 {"9.0.0", "!go1.10"},
Marcel van Lohuizen342b2e12018-12-15 18:52:45 +010088 {"10.0.0", "go1.10,!go1.13"},
Marcel van Lohuizen4b67af82019-10-31 10:06:55 +010089 {"11.0.0", "go1.13,!go1.14"},
Marcel van Lohuizen79eda682020-08-20 15:43:01 +020090 {"12.0.0", "go1.14,!go1.16"},
91 {"13.0.0", "go1.16"},
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +010092}
93
94// buildTags reports the build tags used for the current Unicode version.
95func buildTags() string {
96 v := UnicodeVersion()
Marcel van Lohuizenfe223c52018-12-15 17:54:12 +010097 for _, e := range tags {
98 if e.version == v {
99 return e.buildTags
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +0100100 }
101 }
Marcel van Lohuizenfe223c52018-12-15 17:54:12 +0100102 log.Fatalf("Unknown build tags for Unicode version %q.", v)
103 return ""
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +0100104}
105
Marcel van Lohuizen17805e72016-09-14 22:31:49 +0200106// IsLocal reports whether data files are available locally.
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100107func IsLocal() bool {
Marcel van Lohuizena35b82c2016-10-18 13:00:34 +0200108 dir, err := localReadmeFile()
109 if err != nil {
110 return false
111 }
112 if _, err = os.Stat(dir); err != nil {
Marcel van Lohuizen17805e72016-09-14 22:31:49 +0200113 return false
114 }
115 return true
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100116}
117
118// OpenUCDFile opens the requested UCD file. The file is specified relative to
119// the public Unicode root directory. It will call log.Fatal if there are any
120// errors.
121func OpenUCDFile(file string) io.ReadCloser {
122 return openUnicode(path.Join(*unicodeVersion, "ucd", file))
123}
124
125// OpenCLDRCoreZip opens the CLDR core zip file. It will call log.Fatal if there
126// are any errors.
127func OpenCLDRCoreZip() io.ReadCloser {
Marcel van Lohuizend1927f62015-03-10 13:01:21 +0900128 return OpenUnicodeFile("cldr", *cldrVersion, "core.zip")
129}
130
131// OpenUnicodeFile opens the requested file of the requested category from the
132// root of the Unicode data archive. The file is specified relative to the
133// public Unicode root directory. If version is "", it will use the default
134// Unicode version. It will call log.Fatal if there are any errors.
135func OpenUnicodeFile(category, version, file string) io.ReadCloser {
136 if version == "" {
137 version = UnicodeVersion()
138 }
139 return openUnicode(path.Join(category, version, file))
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100140}
141
142// OpenIANAFile opens the requested IANA file. The file is specified relative
143// to the IANA root, which is typically either http://www.iana.org or the
144// iana directory in the local mirror. It will call log.Fatal if there are any
145// errors.
146func OpenIANAFile(path string) io.ReadCloser {
Marcel van Lohuizen26df76b2015-03-17 12:40:29 +0900147 return Open(*iana, "iana", path)
148}
149
Marcel van Lohuizen17805e72016-09-14 22:31:49 +0200150var (
151 dirMutex sync.Mutex
152 localDir string
153)
154
155const permissions = 0755
156
Marcel van Lohuizena35b82c2016-10-18 13:00:34 +0200157func localReadmeFile() (string, error) {
Marcel van Lohuizen17805e72016-09-14 22:31:49 +0200158 p, err := build.Import("golang.org/x/text", "", build.FindOnly)
159 if err != nil {
Marcel van Lohuizena35b82c2016-10-18 13:00:34 +0200160 return "", fmt.Errorf("Could not locate package: %v", err)
Marcel van Lohuizen17805e72016-09-14 22:31:49 +0200161 }
Marcel van Lohuizena35b82c2016-10-18 13:00:34 +0200162 return filepath.Join(p.Dir, "DATA", "README"), nil
Marcel van Lohuizen17805e72016-09-14 22:31:49 +0200163}
164
165func getLocalDir() string {
166 dirMutex.Lock()
167 defer dirMutex.Unlock()
168
Marcel van Lohuizena35b82c2016-10-18 13:00:34 +0200169 readme, err := localReadmeFile()
170 if err != nil {
171 log.Fatal(err)
172 }
Marcel van Lohuizen17805e72016-09-14 22:31:49 +0200173 dir := filepath.Dir(readme)
174 if _, err := os.Stat(readme); err != nil {
175 if err := os.MkdirAll(dir, permissions); err != nil {
176 log.Fatalf("Could not create directory: %v", err)
177 }
178 ioutil.WriteFile(readme, []byte(readmeTxt), permissions)
179 }
180 return dir
181}
182
183const readmeTxt = `Generated by golang.org/x/text/internal/gen. DO NOT EDIT.
184
185This directory contains downloaded files used to generate the various tables
186in the golang.org/x/text subrepo.
187
188Note that the language subtag repo (iana/assignments/language-subtag-registry)
189and all other times in the iana subdirectory are not versioned and will need
190to be periodically manually updated. The easiest way to do this is to remove
191the entire iana directory. This is mostly of concern when updating the language
192package.
193`
194
Marcel van Lohuizen26df76b2015-03-17 12:40:29 +0900195// Open opens subdir/path if a local directory is specified and the file exists,
196// where subdir is a directory relative to the local root, or fetches it from
197// urlRoot/path otherwise. It will call log.Fatal if there are any errors.
198func Open(urlRoot, subdir, path string) io.ReadCloser {
Marcel van Lohuizen17805e72016-09-14 22:31:49 +0200199 file := filepath.Join(getLocalDir(), subdir, filepath.FromSlash(path))
200 return open(file, urlRoot, path)
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100201}
202
203func openUnicode(path string) io.ReadCloser {
Marcel van Lohuizen17805e72016-09-14 22:31:49 +0200204 file := filepath.Join(getLocalDir(), filepath.FromSlash(path))
205 return open(file, *url, path)
206}
207
208// TODO: automatically periodically update non-versioned files.
209
210func open(file, urlRoot, path string) io.ReadCloser {
211 if f, err := os.Open(file); err == nil {
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100212 return f
213 }
Marcel van Lohuizen17805e72016-09-14 22:31:49 +0200214 r := get(urlRoot, path)
215 defer r.Close()
216 b, err := ioutil.ReadAll(r)
217 if err != nil {
218 log.Fatalf("Could not download file: %v", err)
219 }
220 os.MkdirAll(filepath.Dir(file), permissions)
221 if err := ioutil.WriteFile(file, b, permissions); err != nil {
222 log.Fatalf("Could not create file: %v", err)
223 }
224 return ioutil.NopCloser(bytes.NewReader(b))
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100225}
226
227func get(root, path string) io.ReadCloser {
228 url := root + "/" + path
Marcel van Lohuizenf8e557f2015-09-11 14:46:39 +0200229 fmt.Printf("Fetching %s...", url)
230 defer fmt.Println(" done.")
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100231 resp, err := http.Get(url)
232 if err != nil {
233 log.Fatalf("HTTP GET: %v", err)
234 }
235 if resp.StatusCode != 200 {
236 log.Fatalf("Bad GET status for %q: %q", url, resp.Status)
237 }
238 return resp.Body
239}
240
Marcel van Lohuizen7c0e16d2015-07-09 23:14:16 +0200241// TODO: use Write*Version in all applicable packages.
242
243// WriteUnicodeVersion writes a constant for the Unicode version from which the
244// tables are generated.
245func WriteUnicodeVersion(w io.Writer) {
246 fmt.Fprintf(w, "// UnicodeVersion is the Unicode version from which the tables in this package are derived.\n")
247 fmt.Fprintf(w, "const UnicodeVersion = %q\n\n", UnicodeVersion())
248}
249
250// WriteCLDRVersion writes a constant for the CLDR version from which the
251// tables are generated.
252func WriteCLDRVersion(w io.Writer) {
253 fmt.Fprintf(w, "// CLDRVersion is the CLDR version from which the tables in this package are derived.\n")
254 fmt.Fprintf(w, "const CLDRVersion = %q\n\n", CLDRVersion())
255}
256
Marcel van Lohuizenf8e557f2015-09-11 14:46:39 +0200257// WriteGoFile prepends a standard file comment and package statement to the
Dmitri Shuralyovdf923bb2015-06-04 12:03:31 -0700258// given bytes, applies gofmt, and writes them to a file with the given name.
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100259// It will call log.Fatal if there are any errors.
Marcel van Lohuizen7923bc82015-03-06 09:11:58 +0000260func WriteGoFile(filename, pkg string, b []byte) {
261 w, err := os.Create(filename)
262 if err != nil {
263 log.Fatalf("Could not create file %s: %v", filename, err)
264 }
265 defer w.Close()
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +0100266 if _, err = WriteGo(w, pkg, "", b); err != nil {
267 log.Fatalf("Error writing file %s: %v", filename, err)
268 }
269}
270
Marcel van Lohuizenfe223c52018-12-15 17:54:12 +0100271func fileToPattern(filename string) string {
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +0100272 suffix := ".go"
273 if strings.HasSuffix(filename, "_test.go") {
274 suffix = "_test.go"
275 }
Marcel van Lohuizenfe223c52018-12-15 17:54:12 +0100276 prefix := filename[:len(filename)-len(suffix)]
277 return fmt.Sprint(prefix, "%s", suffix)
278}
279
280func updateBuildTags(pattern string) {
281 for _, t := range tags {
282 oldFile := fmt.Sprintf(pattern, t.version)
283 b, err := ioutil.ReadFile(oldFile)
284 if err != nil {
285 continue
286 }
287 build := fmt.Sprintf("// +build %s", t.buildTags)
288 b = regexp.MustCompile(`// \+build .*`).ReplaceAll(b, []byte(build))
289 err = ioutil.WriteFile(oldFile, b, 0644)
290 if err != nil {
291 log.Fatal(err)
292 }
293 }
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +0100294}
295
296// WriteVersionedGoFile prepends a standard file comment, adds build tags to
297// version the file for the current Unicode version, and package statement to
298// the given bytes, applies gofmt, and writes them to a file with the given
299// name. It will call log.Fatal if there are any errors.
300func WriteVersionedGoFile(filename, pkg string, b []byte) {
Marcel van Lohuizenfe223c52018-12-15 17:54:12 +0100301 pattern := fileToPattern(filename)
302 updateBuildTags(pattern)
303 filename = fmt.Sprintf(pattern, UnicodeVersion())
304
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +0100305 w, err := os.Create(filename)
306 if err != nil {
307 log.Fatalf("Could not create file %s: %v", filename, err)
308 }
309 defer w.Close()
Marcel van Lohuizenfe223c52018-12-15 17:54:12 +0100310 if _, err = WriteGo(w, pkg, buildTags(), b); err != nil {
Marcel van Lohuizenf8e557f2015-09-11 14:46:39 +0200311 log.Fatalf("Error writing file %s: %v", filename, err)
312 }
313}
314
315// WriteGo prepends a standard file comment and package statement to the given
316// bytes, applies gofmt, and writes them to w.
Marcel van Lohuizenc4d099d2017-12-11 13:26:16 +0100317func WriteGo(w io.Writer, pkg, tags string, b []byte) (n int, err error) {
318 src := []byte(header)
319 if tags != "" {
320 src = append(src, fmt.Sprintf("// +build %s\n\n", tags)...)
321 }
322 src = append(src, fmt.Sprintf("package %s\n\n", pkg)...)
Dmitri Shuralyovdf923bb2015-06-04 12:03:31 -0700323 src = append(src, b...)
324 formatted, err := format.Source(src)
Marcel van Lohuizen7923bc82015-03-06 09:11:58 +0000325 if err != nil {
Dmitri Shuralyovdf923bb2015-06-04 12:03:31 -0700326 // Print the generated code even in case of an error so that the
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100327 // returned error can be meaningfully interpreted.
Marcel van Lohuizenf8e557f2015-09-11 14:46:39 +0200328 n, _ = w.Write(src)
329 return n, err
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100330 }
Marcel van Lohuizenf8e557f2015-09-11 14:46:39 +0200331 return w.Write(formatted)
Marcel van Lohuizen731229f2015-03-03 14:02:25 +0100332}
Marcel van Lohuizen54db2312015-11-24 19:16:13 +0100333
334// Repackage rewrites a Go file from belonging to package main to belonging to
335// the given package.
336func Repackage(inFile, outFile, pkg string) {
337 src, err := ioutil.ReadFile(inFile)
338 if err != nil {
339 log.Fatalf("reading %s: %v", inFile, err)
340 }
341 const toDelete = "package main\n\n"
342 i := bytes.Index(src, []byte(toDelete))
343 if i < 0 {
344 log.Fatalf("Could not find %q in %s.", toDelete, inFile)
345 }
346 w := &bytes.Buffer{}
347 w.Write(src[i+len(toDelete):])
348 WriteGoFile(outFile, pkg, w.Bytes())
349}