unicode/cldr/makexml.go - text - Git at Google

 // Copyright 2013 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 //go:build ignore
 // +build ignore

 // This tool generates types for the various XML formats of CLDR.
 package main

 import (
 	"archive/zip"
 	"bytes"
 	"encoding/xml"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
 	"regexp"
 	"strings"

 	"golang.org/x/text/internal/gen"
 )

 var outputFile = flag.String("output", "xml.go", "output file name")

 func main() {
 	flag.Parse()

 	r := gen.OpenCLDRCoreZip()
 	buffer, err := ioutil.ReadAll(r)
 	if err != nil {
 		log.Fatal("Could not read zip file")
 	}
 	r.Close()
 	z, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
 	if err != nil {
 		log.Fatalf("Could not read zip archive: %v", err)
 	}

 	var buf bytes.Buffer

 	version := gen.CLDRVersion()

 	for _, dtd := range files {
 		for _, f := range z.File {
 			if strings.HasSuffix(f.Name, dtd.file+".dtd") {
 				r, err := f.Open()
 				failOnError(err)

 				b := makeBuilder(&buf, dtd)
 				b.parseDTD(r)
 				b.resolve(b.index[dtd.top[0]])
 				b.write()
 				if b.version != "" && version != b.version {
 					println(f.Name)
 					log.Fatalf("main: inconsistent versions: found %s; want %s", b.version, version)
 				}
 				break
 			}
 		}
 	}
 	fmt.Fprintln(&buf, "// Version is the version of CLDR from which the XML definitions are generated.")
 	fmt.Fprintf(&buf, "const Version = %q\n", version)

 	gen.WriteGoFile(*outputFile, "cldr", buf.Bytes())
 }

 func failOnError(err error) {
 	if err != nil {
 		log.New(os.Stderr, "", log.Lshortfile).Output(2, err.Error())
 		os.Exit(1)
 	}
 }

 // configuration data per DTD type
 type dtd struct {
 	file string   // base file name
 	root string   // Go name of the root XML element
 	top  []string // create a different type for this section

 	skipElem    []string // hard-coded or deprecated elements
 	skipAttr    []string // attributes to exclude
 	predefined  []string // hard-coded elements exist of the form <name>Elem
 	forceRepeat []string // elements to make slices despite DTD
 }

 var files = []dtd{
 	{
 		file: "ldmlBCP47",
 		root: "LDMLBCP47",
 		top:  []string{"ldmlBCP47"},
 		skipElem: []string{
 			"cldrVersion", // deprecated, not used
 		},
 	},
 	{
 		file: "ldmlSupplemental",
 		root: "SupplementalData",
 		top:  []string{"supplementalData"},
 		skipElem: []string{
 			"cldrVersion", // deprecated, not used
 		},
 		forceRepeat: []string{
 			"plurals", // data defined in plurals.xml and ordinals.xml
 		},
 	},
 	{
 		file: "ldml",
 		root: "LDML",
 		top: []string{
 			"ldml", "collation", "calendar", "timeZoneNames", "localeDisplayNames", "numbers",
 		},
 		skipElem: []string{
 			"cp",       // not used anywhere
 			"special",  // not used anywhere
 			"fallback", // deprecated, not used
 			"alias",    // in Common
 			"default",  // in Common
 		},
 		skipAttr: []string{
 			"hiraganaQuarternary", // typo in DTD, correct version included as well
 		},
 		predefined: []string{"rules"},
 	},
 }

 var comments = map[string]string{
 	"ldmlBCP47": `
 // LDMLBCP47 holds information on allowable values for various variables in LDML.
 `,
 	"supplementalData": `
 // SupplementalData holds information relevant for internationalization
 // and proper use of CLDR, but that is not contained in the locale hierarchy.
 `,
 	"ldml": `
 // LDML is the top-level type for locale-specific data.
 `,
 	"collation": `
 // Collation contains rules that specify a certain sort-order,
 // as a tailoring of the root order.
 // The parsed rules are obtained by passing a RuleProcessor to Collation's
 // Process method.
 `,
 	"calendar": `
 // Calendar specifies the fields used for formatting and parsing dates and times.
 // The month and quarter names are identified numerically, starting at 1.
 // The day (of the week) names are identified with short strings, since there is
 // no universally-accepted numeric designation.
 `,
 	"dates": `
 // Dates contains information regarding the format and parsing of dates and times.
 `,
 	"localeDisplayNames": `
 // LocaleDisplayNames specifies localized display names for scripts, languages,
 // countries, currencies, and variants.
 `,
 	"numbers": `
 // Numbers supplies information for formatting and parsing numbers and currencies.
 `,
 }

 type element struct {
 	name      string // XML element name
 	category  string // elements contained by this element
 	signature string // category + attrKey*

 	attr []*attribute // attributes supported by this element.
 	sub  []struct {   // parsed and evaluated sub elements of this element.
 		e      *element
 		repeat bool // true if the element needs to be a slice
 	}

 	resolved bool // prevent multiple resolutions of this element.
 }

 type attribute struct {
 	name string
 	key  string
 	list []string

 	tag string // Go tag
 }

 var (
 	reHead  = regexp.MustCompile(` *(\w+) +([\w\-]+)`)
 	reAttr  = regexp.MustCompile(` *(\w+) *(?:(\w+)|\(([\w\- \|]+)\)) *(?:#([A-Z]*) *(?:\"([\.\d+])\")?)? *("[\w\-:]*")?`)
 	reElem  = regexp.MustCompile(`^ *(EMPTY|ANY|\(.*\)[\*\+\?]?) *$`)
 	reToken = regexp.MustCompile(`\w\-`)
 )

 // builder is used to read in the DTD files from CLDR and generate Go code
 // to be used with the encoding/xml package.
 type builder struct {
 	w       io.Writer
 	index   map[string]*element
 	elem    []*element
 	info    dtd
 	version string
 }

 func makeBuilder(w io.Writer, d dtd) builder {
 	return builder{
 		w:     w,
 		index: make(map[string]*element),
 		elem:  []*element{},
 		info:  d,
 	}
 }

 // parseDTD parses a DTD file.
 func (b *builder) parseDTD(r io.Reader) {
 	for d := xml.NewDecoder(r); ; {
 		t, err := d.Token()
 		if t == nil {
 			break
 		}
 		failOnError(err)
 		dir, ok := t.(xml.Directive)
 		if !ok {
 			continue
 		}
 		m := reHead.FindSubmatch(dir)
 		dir = dir[len(m[0]):]
 		ename := string(m[2])
 		el, elementFound := b.index[ename]
 		switch string(m[1]) {
 		case "ELEMENT":
 			if elementFound {
 				log.Fatal("parseDTD: duplicate entry for element %q", ename)
 			}
 			m := reElem.FindSubmatch(dir)
 			if m == nil {
 				log.Fatalf("parseDTD: invalid element %q", string(dir))
 			}
 			if len(m[0]) != len(dir) {
 				log.Fatal("parseDTD: invalid element %q", string(dir), len(dir), len(m[0]), string(m[0]))
 			}
 			s := string(m[1])
 			el = &element{
 				name:     ename,
 				category: s,
 			}
 			b.index[ename] = el
 		case "ATTLIST":
 			if !elementFound {
 				log.Fatalf("parseDTD: unknown element %q", ename)
 			}
 			s := string(dir)
 			m := reAttr.FindStringSubmatch(s)
 			if m == nil {
 				log.Fatal(fmt.Errorf("parseDTD: invalid attribute %q", string(dir)))
 			}
 			if m[4] == "FIXED" {
 				b.version = m[5]
 			} else {
 				switch m[1] {
 				case "draft", "references", "alt", "validSubLocales", "standard" /* in Common */ :
 				case "type", "choice":
 				default:
 					el.attr = append(el.attr, &attribute{
 						name: m[1],
 						key:  s,
 						list: reToken.FindAllString(m[3], -1),
 					})
 					el.signature = fmt.Sprintf("%s=%s+%s", el.signature, m[1], m[2])
 				}
 			}
 		}
 	}
 }

 var reCat = regexp.MustCompile(`[ ,\|]*(?:(\(|\)|\#?[\w_-]+)([\*\+\?]?))?`)

 // resolve takes a parsed element and converts it into structured data
 // that can be used to generate the XML code.
 func (b *builder) resolve(e *element) {
 	if e.resolved {
 		return
 	}
 	b.elem = append(b.elem, e)
 	e.resolved = true
 	s := e.category
 	found := make(map[string]bool)
 	sequenceStart := []int{}
 	for len(s) > 0 {
 		m := reCat.FindStringSubmatch(s)
 		if m == nil {
 			log.Fatalf("%s: invalid category string %q", e.name, s)
 		}
 		repeat := m[2] == "*" || m[2] == "+" || in(b.info.forceRepeat, m[1])
 		switch m[1] {
 		case "":
 		case "(":
 			sequenceStart = append(sequenceStart, len(e.sub))
 		case ")":
 			if len(sequenceStart) == 0 {
 				log.Fatalf("%s: unmatched closing parenthesis", e.name)
 			}
 			for i := sequenceStart[len(sequenceStart)-1]; i < len(e.sub); i++ {
 				e.sub[i].repeat = e.sub[i].repeat || repeat
 			}
 			sequenceStart = sequenceStart[:len(sequenceStart)-1]
 		default:
 			if in(b.info.skipElem, m[1]) {
 			} else if sub, ok := b.index[m[1]]; ok {
 				if !found[sub.name] {
 					e.sub = append(e.sub, struct {
 						e      *element
 						repeat bool
 					}{sub, repeat})
 					found[sub.name] = true
 					b.resolve(sub)
 				}
 			} else if m[1] == "#PCDATA" || m[1] == "ANY" {
 			} else if m[1] != "EMPTY" {
 				log.Fatalf("resolve:%s: element %q not found", e.name, m[1])
 			}
 		}
 		s = s[len(m[0]):]
 	}
 }

 // return true if s is contained in set.
 func in(set []string, s string) bool {
 	for _, v := range set {
 		if v == s {
 			return true
 		}
 	}
 	return false
 }

 var repl = strings.NewReplacer("-", " ", "_", " ")

 // title puts the first character or each character following '_' in title case and
 // removes all occurrences of '_'.
 func title(s string) string {
 	return strings.Replace(strings.Title(repl.Replace(s)), " ", "", -1)
 }

 // writeElem generates Go code for a single element, recursively.
 func (b *builder) writeElem(tab int, e *element) {
 	p := func(f string, x ...interface{}) {
 		f = strings.Replace(f, "\n", "\n"+strings.Repeat("\t", tab), -1)
 		fmt.Fprintf(b.w, f, x...)
 	}
 	if len(e.sub) == 0 && len(e.attr) == 0 {
 		p("Common")
 		return
 	}
 	p("struct {")
 	tab++
 	p("\nCommon")
 	for _, attr := range e.attr {
 		if !in(b.info.skipAttr, attr.name) {
 			p("\n%s string `xml:\"%s,attr\"`", title(attr.name), attr.name)
 		}
 	}
 	for _, sub := range e.sub {
 		if in(b.info.predefined, sub.e.name) {
 			p("\n%sElem", sub.e.name)
 			continue
 		}
 		if in(b.info.skipElem, sub.e.name) {
 			continue
 		}
 		p("\n%s ", title(sub.e.name))
 		if sub.repeat {
 			p("[]")
 		}
 		p("*")
 		if in(b.info.top, sub.e.name) {
 			p(title(sub.e.name))
 		} else {
 			b.writeElem(tab, sub.e)
 		}
 		p(" `xml:\"%s\"`", sub.e.name)
 	}
 	tab--
 	p("\n}")
 }

 // write generates the Go XML code.
 func (b *builder) write() {
 	for i, name := range b.info.top {
 		e := b.index[name]
 		if e != nil {
 			fmt.Fprintf(b.w, comments[name])
 			name := title(e.name)
 			if i == 0 {
 				name = b.info.root
 			}
 			fmt.Fprintf(b.w, "type %s ", name)
 			b.writeElem(0, e)
 			fmt.Fprint(b.w, "\n")
 		}
 	}
 }
	// Copyright 2013 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	//go:build ignore
	// +build ignore

	// This tool generates types for the various XML formats of CLDR.
	package main

	import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	"golang.org/x/text/internal/gen"
	)

	var outputFile = flag.String("output", "xml.go", "output file name")

	func main() {
	flag.Parse()

	r := gen.OpenCLDRCoreZip()
	buffer, err := ioutil.ReadAll(r)
	if err != nil {
	log.Fatal("Could not read zip file")
	}
	r.Close()
	z, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
	if err != nil {
	log.Fatalf("Could not read zip archive: %v", err)
	}

	var buf bytes.Buffer

	version := gen.CLDRVersion()

	for _, dtd := range files {
	for _, f := range z.File {
	if strings.HasSuffix(f.Name, dtd.file+".dtd") {
	r, err := f.Open()
	failOnError(err)

	b := makeBuilder(&buf, dtd)
	b.parseDTD(r)
	b.resolve(b.index[dtd.top[0]])
	b.write()
	if b.version != "" && version != b.version {
	println(f.Name)
	log.Fatalf("main: inconsistent versions: found %s; want %s", b.version, version)
	}
	break
	}
	}
	}
	fmt.Fprintln(&buf, "// Version is the version of CLDR from which the XML definitions are generated.")
	fmt.Fprintf(&buf, "const Version = %q\n", version)

	gen.WriteGoFile(*outputFile, "cldr", buf.Bytes())
	}

	func failOnError(err error) {
	if err != nil {
	log.New(os.Stderr, "", log.Lshortfile).Output(2, err.Error())
	os.Exit(1)
	}
	}

	// configuration data per DTD type
	type dtd struct {
	file string // base file name
	root string // Go name of the root XML element
	top []string // create a different type for this section

	skipElem []string // hard-coded or deprecated elements
	skipAttr []string // attributes to exclude
	predefined []string // hard-coded elements exist of the form <name>Elem
	forceRepeat []string // elements to make slices despite DTD
	}

	var files = []dtd{
	{
	file: "ldmlBCP47",
	root: "LDMLBCP47",
	top: []string{"ldmlBCP47"},
	skipElem: []string{
	"cldrVersion", // deprecated, not used
	},
	},
	{
	file: "ldmlSupplemental",
	root: "SupplementalData",
	top: []string{"supplementalData"},
	skipElem: []string{
	"cldrVersion", // deprecated, not used
	},
	forceRepeat: []string{
	"plurals", // data defined in plurals.xml and ordinals.xml
	},
	},
	{
	file: "ldml",
	root: "LDML",
	top: []string{
	"ldml", "collation", "calendar", "timeZoneNames", "localeDisplayNames", "numbers",
	},
	skipElem: []string{
	"cp", // not used anywhere
	"special", // not used anywhere
	"fallback", // deprecated, not used
	"alias", // in Common
	"default", // in Common
	},
	skipAttr: []string{
	"hiraganaQuarternary", // typo in DTD, correct version included as well
	},
	predefined: []string{"rules"},
	},
	}

	var comments = map[string]string{
	"ldmlBCP47": `
	// LDMLBCP47 holds information on allowable values for various variables in LDML.
	`,
	"supplementalData": `
	// SupplementalData holds information relevant for internationalization
	// and proper use of CLDR, but that is not contained in the locale hierarchy.
	`,
	"ldml": `
	// LDML is the top-level type for locale-specific data.
	`,
	"collation": `
	// Collation contains rules that specify a certain sort-order,
	// as a tailoring of the root order.
	// The parsed rules are obtained by passing a RuleProcessor to Collation's
	// Process method.
	`,
	"calendar": `
	// Calendar specifies the fields used for formatting and parsing dates and times.
	// The month and quarter names are identified numerically, starting at 1.
	// The day (of the week) names are identified with short strings, since there is
	// no universally-accepted numeric designation.
	`,
	"dates": `
	// Dates contains information regarding the format and parsing of dates and times.
	`,
	"localeDisplayNames": `
	// LocaleDisplayNames specifies localized display names for scripts, languages,
	// countries, currencies, and variants.
	`,
	"numbers": `
	// Numbers supplies information for formatting and parsing numbers and currencies.
	`,
	}

	type element struct {
	name string // XML element name
	category string // elements contained by this element
	signature string // category + attrKey*

	attr []*attribute // attributes supported by this element.
	sub []struct { // parsed and evaluated sub elements of this element.
	e *element
	repeat bool // true if the element needs to be a slice
	}

	resolved bool // prevent multiple resolutions of this element.
	}

	type attribute struct {
	name string
	key string
	list []string

	tag string // Go tag
	}

	var (
	reHead = regexp.MustCompile(` *(\w+) +([\w\-]+)`)
	reAttr = regexp.MustCompile(` (\w+) (?:(\w+)\|\(([\w\- \\|]+)\)) (?:#([A-Z]) (?:\"([\.\d+])\")?)? ("[\w\-:]*")?`)
	reElem = regexp.MustCompile(`^ (EMPTY\|ANY\|\(.\)[\\+\?]?) $`)
	reToken = regexp.MustCompile(`\w\-`)
	)

	// builder is used to read in the DTD files from CLDR and generate Go code
	// to be used with the encoding/xml package.
	type builder struct {
	w io.Writer
	index map[string]*element
	elem []*element
	info dtd
	version string
	}

	func makeBuilder(w io.Writer, d dtd) builder {
	return builder{
	w: w,
	index: make(map[string]*element),
	elem: []*element{},
	info: d,
	}
	}

	// parseDTD parses a DTD file.
	func (b *builder) parseDTD(r io.Reader) {
	for d := xml.NewDecoder(r); ; {
	t, err := d.Token()
	if t == nil {
	break
	}
	failOnError(err)
	dir, ok := t.(xml.Directive)
	if !ok {
	continue
	}
	m := reHead.FindSubmatch(dir)
	dir = dir[len(m[0]):]
	ename := string(m[2])
	el, elementFound := b.index[ename]
	switch string(m[1]) {
	case "ELEMENT":
	if elementFound {
	log.Fatal("parseDTD: duplicate entry for element %q", ename)
	}
	m := reElem.FindSubmatch(dir)
	if m == nil {
	log.Fatalf("parseDTD: invalid element %q", string(dir))
	}
	if len(m[0]) != len(dir) {
	log.Fatal("parseDTD: invalid element %q", string(dir), len(dir), len(m[0]), string(m[0]))
	}
	s := string(m[1])
	el = &element{
	name: ename,
	category: s,
	}
	b.index[ename] = el
	case "ATTLIST":
	if !elementFound {
	log.Fatalf("parseDTD: unknown element %q", ename)
	}
	s := string(dir)
	m := reAttr.FindStringSubmatch(s)
	if m == nil {
	log.Fatal(fmt.Errorf("parseDTD: invalid attribute %q", string(dir)))
	}
	if m[4] == "FIXED" {
	b.version = m[5]
	} else {
	switch m[1] {
	case "draft", "references", "alt", "validSubLocales", "standard" /* in Common */ :
	case "type", "choice":
	default:
	el.attr = append(el.attr, &attribute{
	name: m[1],
	key: s,
	list: reToken.FindAllString(m[3], -1),
	})
	el.signature = fmt.Sprintf("%s=%s+%s", el.signature, m[1], m[2])
	}
	}
	}
	}
	}

	var reCat = regexp.MustCompile(`[ ,\\|](?:(\(\|\)\|\#?[\w_-]+)([\\+\?]?))?`)

	// resolve takes a parsed element and converts it into structured data
	// that can be used to generate the XML code.
	func (b builder) resolve(e element) {
	if e.resolved {
	return
	}
	b.elem = append(b.elem, e)
	e.resolved = true
	s := e.category
	found := make(map[string]bool)
	sequenceStart := []int{}
	for len(s) > 0 {
	m := reCat.FindStringSubmatch(s)
	if m == nil {
	log.Fatalf("%s: invalid category string %q", e.name, s)
	}
	repeat := m[2] == "*" \|\| m[2] == "+" \|\| in(b.info.forceRepeat, m[1])
	switch m[1] {
	case "":
	case "(":
	sequenceStart = append(sequenceStart, len(e.sub))
	case ")":
	if len(sequenceStart) == 0 {
	log.Fatalf("%s: unmatched closing parenthesis", e.name)
	}
	for i := sequenceStart[len(sequenceStart)-1]; i < len(e.sub); i++ {
	e.sub[i].repeat = e.sub[i].repeat \|\| repeat
	}
	sequenceStart = sequenceStart[:len(sequenceStart)-1]
	default:
	if in(b.info.skipElem, m[1]) {
	} else if sub, ok := b.index[m[1]]; ok {
	if !found[sub.name] {
	e.sub = append(e.sub, struct {
	e *element
	repeat bool
	}{sub, repeat})
	found[sub.name] = true
	b.resolve(sub)
	}
	} else if m[1] == "#PCDATA" \|\| m[1] == "ANY" {
	} else if m[1] != "EMPTY" {
	log.Fatalf("resolve:%s: element %q not found", e.name, m[1])
	}
	}
	s = s[len(m[0]):]
	}
	}

	// return true if s is contained in set.
	func in(set []string, s string) bool {
	for _, v := range set {
	if v == s {
	return true
	}
	}
	return false
	}

	var repl = strings.NewReplacer("-", " ", "_", " ")

	// title puts the first character or each character following '_' in title case and
	// removes all occurrences of '_'.
	func title(s string) string {
	return strings.Replace(strings.Title(repl.Replace(s)), " ", "", -1)
	}

	// writeElem generates Go code for a single element, recursively.
	func (b builder) writeElem(tab int, e element) {
	p := func(f string, x ...interface{}) {
	f = strings.Replace(f, "\n", "\n"+strings.Repeat("\t", tab), -1)
	fmt.Fprintf(b.w, f, x...)
	}
	if len(e.sub) == 0 && len(e.attr) == 0 {
	p("Common")
	return
	}
	p("struct {")
	tab++
	p("\nCommon")
	for _, attr := range e.attr {
	if !in(b.info.skipAttr, attr.name) {
	p("\n%s string `xml:\"%s,attr\"`", title(attr.name), attr.name)
	}
	}
	for _, sub := range e.sub {
	if in(b.info.predefined, sub.e.name) {
	p("\n%sElem", sub.e.name)
	continue
	}
	if in(b.info.skipElem, sub.e.name) {
	continue
	}
	p("\n%s ", title(sub.e.name))
	if sub.repeat {
	p("[]")
	}
	p("*")
	if in(b.info.top, sub.e.name) {
	p(title(sub.e.name))
	} else {
	b.writeElem(tab, sub.e)
	}
	p(" `xml:\"%s\"`", sub.e.name)
	}
	tab--
	p("\n}")
	}

	// write generates the Go XML code.
	func (b *builder) write() {
	for i, name := range b.info.top {
	e := b.index[name]
	if e != nil {
	fmt.Fprintf(b.w, comments[name])
	name := title(e.name)
	if i == 0 {
	name = b.info.root
	}
	fmt.Fprintf(b.w, "type %s ", name)
	b.writeElem(0, e)
	fmt.Fprint(b.w, "\n")
	}
	}
	}