blob: 01eb6b6ec54bbaf3fe7b3ec760ab1fe19adfaa2c [file] [log] [blame]
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
/*
* make is(upper|lower|title|space|alpha)rune and
* to(upper|lower|title)rune from a UnicodeData.txt file.
* these can be found at unicode.org
*
* with -c, runs a check of the existing runetype functions vs.
* those extracted from UnicodeData.
*
* with -p, generates tables for pairs of chars, as well as for ranges
* and singletons.
*
* UnicodeData defines 4 fields of interest:
* 1) a category
* 2) an upper case mapping
* 3) a lower case mapping
* 4) a title case mapping
*
* toupper, tolower, and totitle are defined directly from the mapping.
*
* isalpharune(c) is true iff c is a "letter" category
* isupperrune(c) is true iff c is the target of toupperrune,
* or is in the uppercase letter category
* similarly for islowerrune and istitlerune.
* isspacerune is true for space category chars, "C" locale white space chars,
* and two additions:
* 0085 "next line" control char
* feff] "zero-width non-break space"
* isdigitrune is true iff c is a numeric-digit category.
*/
#include <u.h>
#include <libc.h>
#include <stdio.h>
#include "utf.h"
#include "utfdef.h"
enum {
/*
* fields in the unicode data file
*/
FIELD_CODE,
FIELD_NAME,
FIELD_CATEGORY,
FIELD_COMBINING,
FIELD_BIDIR,
FIELD_DECOMP,
FIELD_DECIMAL_DIG,
FIELD_DIG,
FIELD_NUMERIC_VAL,
FIELD_MIRRORED,
FIELD_UNICODE_1_NAME,
FIELD_COMMENT,
FIELD_UPPER,
FIELD_LOWER,
FIELD_TITLE,
NFIELDS,
MAX_LINE = 1024,
TO_OFFSET = 1 << 20,
NRUNES = 1 << 21,
};
#define TO_DELTA(xmapped,x) (TO_OFFSET + (xmapped) - (x))
static char myisspace[NRUNES];
static char myisalpha[NRUNES];
static char myisdigit[NRUNES];
static char myisupper[NRUNES];
static char myislower[NRUNES];
static char myistitle[NRUNES];
static int mytoupper[NRUNES];
static int mytolower[NRUNES];
static int mytotitle[NRUNES];
static void check(void);
static void mktables(char *src, int usepairs);
static void fatal(const char *fmt, ...);
static int mygetfields(char **fields, int nfields, char *str, const char *delim);
static int getunicodeline(FILE *in, char **fields, char *buf);
static int getcode(char *s);
static void
usage(void)
{
fprintf(stderr, "usage: mktables [-cp] <UnicodeData.txt>\n");
exit(1);
}
void
main(int argc, char *argv[])
{
FILE *in;
char buf[MAX_LINE], buf2[MAX_LINE];
char *fields[NFIELDS + 1], *fields2[NFIELDS + 1];
char *p;
int i, code, last, docheck, usepairs;
docheck = 0;
usepairs = 0;
ARGBEGIN{
case 'c':
docheck = 1;
break;
case 'p':
usepairs = 1;
break;
default:
usage();
}ARGEND
if(argc != 1){
usage();
}
in = fopen(argv[0], "r");
if(in == NULL){
fatal("can't open %s", argv[0]);
}
for(i = 0; i < NRUNES; i++){
mytoupper[i] = i;
mytolower[i] = i;
mytotitle[i] = i;
}
/*
* make sure isspace has all of the "C" locale whitespace chars
*/
myisspace['\t'] = 1;
myisspace['\n'] = 1;
myisspace['\r'] = 1;
myisspace['\f'] = 1;
myisspace['\v'] = 1;
/*
* a couple of other exceptions
*/
myisspace[0x85] = 1; /* control char, "next line" */
myisspace[0xfeff] = 1; /* zero-width non-break space */
last = -1;
while(getunicodeline(in, fields, buf)){
code = getcode(fields[FIELD_CODE]);
if (code >= NRUNES)
fatal("code-point value too big: %x", code);
if(code <= last)
fatal("bad code sequence: %x then %x", last, code);
last = code;
/*
* check for ranges
*/
p = fields[FIELD_CATEGORY];
if(strstr(fields[FIELD_NAME], ", First>") != NULL){
if(!getunicodeline(in, fields2, buf2))
fatal("range start at eof");
if (strstr(fields2[FIELD_NAME], ", Last>") == NULL)
fatal("range start not followed by range end");
last = getcode(fields2[FIELD_CODE]);
if(last <= code)
fatal("range out of sequence: %x then %x", code, last);
if(strcmp(p, fields2[FIELD_CATEGORY]) != 0)
fatal("range with mismatched category");
}
/*
* set properties and conversions
*/
for (; code <= last; code++){
if(p[0] == 'L')
myisalpha[code] = 1;
if(p[0] == 'Z')
myisspace[code] = 1;
if(strcmp(p, "Lu") == 0)
myisupper[code] = 1;
if(strcmp(p, "Ll") == 0)
myislower[code] = 1;
if(strcmp(p, "Lt") == 0)
myistitle[code] = 1;
if(strcmp(p, "Nd") == 0)
myisdigit[code] = 1;
/*
* when finding conversions, also need to mark
* upper/lower case, since some chars, like
* "III" (0x2162), aren't defined as letters but have a
* lower case mapping ("iii" (0x2172)).
*/
if(fields[FIELD_UPPER][0] != '\0'){
mytoupper[code] = getcode(fields[FIELD_UPPER]);
}
if(fields[FIELD_LOWER][0] != '\0'){
mytolower[code] = getcode(fields[FIELD_LOWER]);
}
if(fields[FIELD_TITLE][0] != '\0'){
mytotitle[code] = getcode(fields[FIELD_TITLE]);
}
}
}
fclose(in);
/*
* check for codes with no totitle mapping but a toupper mapping.
* these appear in UnicodeData-2.0.14.txt, but are almost certainly
* erroneous.
*/
for(i = 0; i < NRUNES; i++){
if(mytotitle[i] == i
&& mytoupper[i] != i
&& !myistitle[i])
fprintf(stderr, "warning: code=%.4x not istitle, totitle is same, toupper=%.4x\n", i, mytoupper[i]);
}
/*
* make sure isupper[c] is true if for some x toupper[x] == c
* ditto for islower and istitle
*/
for(i = 0; i < NRUNES; i++) {
if(mytoupper[i] != i)
myisupper[mytoupper[i]] = 1;
if(mytolower[i] != i)
myislower[mytolower[i]] = 1;
if(mytotitle[i] != i)
myistitle[mytotitle[i]] = 1;
}
if(docheck){
check();
}else{
mktables(argv[0], usepairs);
}
exit(0);
}
/*
* generate a properties array for ranges, clearing those cases covered.
* if force, generate one-entry ranges for singletons.
*/
static int
mkisrange(const char* label, char* prop, int force)
{
int start, stop, some;
/*
* first, the ranges
*/
some = 0;
for(start = 0; start < NRUNES; ) {
if(!prop[start]){
start++;
continue;
}
for(stop = start + 1; stop < NRUNES; stop++){
if(!prop[stop]){
break;
}
prop[stop] = 0;
}
if(force || stop != start + 1){
if(!some){
printf("static Rune __is%sr[] = {\n", label);
some = 1;
}
prop[start] = 0;
printf("\t0x%.4x, 0x%.4x,\n", start, stop - 1);
}
start = stop;
}
if(some)
printf("};\n\n");
return some;
}
/*
* generate a mapping array for pairs with a skip between,
* clearing those entries covered.
*/
static int
mkispair(const char *label, char *prop)
{
int start, stop, some;
some = 0;
for(start = 0; start + 2 < NRUNES; ) {
if(!prop[start]){
start++;
continue;
}
for(stop = start + 2; stop < NRUNES; stop += 2){
if(!prop[stop]){
break;
}
prop[stop] = 0;
}
if(stop != start + 2){
if(!some){
printf("static Rune __is%sp[] = {\n", label);
some = 1;
}
prop[start] = 0;
printf("\t0x%.4x, 0x%.4x,\n", start, stop - 2);
}
start = stop;
}
if(some)
printf("};\n\n");
return some;
}
/*
* generate a properties array for singletons, clearing those cases covered.
*/
static int
mkissingle(const char *label, char *prop)
{
int start, some;
some = 0;
for(start = 0; start < NRUNES; start++) {
if(!prop[start]){
continue;
}
if(!some){
printf("static Rune __is%ss[] = {\n", label);
some = 1;
}
prop[start] = 0;
printf("\t0x%.4x,\n", start);
}
if(some)
printf("};\n\n");
return some;
}
/*
* generate tables and a function for is<label>rune
*/
static void
mkis(const char* label, char* prop, int usepairs)
{
int isr, isp, iss;
isr = mkisrange(label, prop, 0);
isp = 0;
if(usepairs)
isp = mkispair(label, prop);
iss = mkissingle(label, prop);
printf(
"int\n"
"is%srune(Rune c)\n"
"{\n"
" Rune *p;\n"
"\n",
label);
if(isr)
printf(
" p = rbsearch(c, __is%sr, nelem(__is%sr)/2, 2);\n"
" if(p && c >= p[0] && c <= p[1])\n"
" return 1;\n",
label, label);
if(isp)
printf(
" p = rbsearch(c, __is%sp, nelem(__is%sp)/2, 2);\n"
" if(p && c >= p[0] && c <= p[1] && !((c - p[0]) & 1))\n"
" return 1;\n",
label, label);
if(iss)
printf(
" p = rbsearch(c, __is%ss, nelem(__is%ss), 1);\n"
" if(p && c == p[0])\n"
" return 1;\n",
label, label);
printf(
" return 0;\n"
"}\n"
"\n"
);
}
/*
* generate a mapping array for ranges, clearing those entries covered.
* if force, generate one-entry ranges for singletons.
*/
static int
mktorange(const char* label, int* map, int force)
{
int start, stop, delta, some;
some = 0;
for(start = 0; start < NRUNES; ) {
if(map[start] == start){
start++;
continue;
}
delta = TO_DELTA(map[start], start);
if(delta != (Rune)delta)
fatal("bad map delta %d", delta);
for(stop = start + 1; stop < NRUNES; stop++){
if(TO_DELTA(map[stop], stop) != delta){
break;
}
map[stop] = stop;
}
if(stop != start + 1){
if(!some){
printf("static Rune __to%sr[] = {\n", label);
some = 1;
}
map[start] = start;
printf("\t0x%.4x, 0x%.4x, %d,\n", start, stop - 1, delta);
}
start = stop;
}
if(some)
printf("};\n\n");
return some;
}
/*
* generate a mapping array for pairs with a skip between,
* clearing those entries covered.
*/
static int
mktopair(const char* label, int* map)
{
int start, stop, delta, some;
some = 0;
for(start = 0; start + 2 < NRUNES; ) {
if(map[start] == start){
start++;
continue;
}
delta = TO_DELTA(map[start], start);
if(delta != (Rune)delta)
fatal("bad map delta %d", delta);
for(stop = start + 2; stop < NRUNES; stop += 2){
if(TO_DELTA(map[stop], stop) != delta){
break;
}
map[stop] = stop;
}
if(stop != start + 2){
if(!some){
printf("static Rune __to%sp[] = {\n", label);
some = 1;
}
map[start] = start;
printf("\t0x%.4x, 0x%.4x, %d,\n", start, stop - 2, delta);
}
start = stop;
}
if(some)
printf("};\n\n");
return some;
}
/*
* generate a mapping array for singletons, clearing those entries covered.
*/
static int
mktosingle(const char* label, int* map)
{
int start, delta, some;
some = 0;
for(start = 0; start < NRUNES; start++) {
if(map[start] == start){
continue;
}
delta = TO_DELTA(map[start], start);
if(delta != (Rune)delta)
fatal("bad map delta %d", delta);
if(!some){
printf("static Rune __to%ss[] = {\n", label);
some = 1;
}
map[start] = start;
printf("\t0x%.4x, %d,\n", start, delta);
}
if(some)
printf("};\n\n");
return some;
}
/*
* generate tables and a function for to<label>rune
*/
static void
mkto(const char* label, int* map, int usepairs)
{
int tor, top, tos;
tor = mktorange(label, map, 0);
top = 0;
if(usepairs)
top = mktopair(label, map);
tos = mktosingle(label, map);
printf(
"Rune\n"
"to%srune(Rune c)\n"
"{\n"
" Rune *p;\n"
"\n",
label);
if(tor)
printf(
" p = rbsearch(c, __to%sr, nelem(__to%sr)/3, 3);\n"
" if(p && c >= p[0] && c <= p[1])\n"
" return c + p[2] - %d;\n",
label, label, TO_OFFSET);
if(top)
printf(
" p = rbsearch(c, __to%sp, nelem(__to%sp)/3, 3);\n"
" if(p && c >= p[0] && c <= p[1] && !((c - p[0]) & 1))\n"
" return c + p[2] - %d;\n",
label, label, TO_OFFSET);
if(tos)
printf(
" p = rbsearch(c, __to%ss, nelem(__to%ss)/2, 2);\n"
" if(p && c == p[0])\n"
" return c + p[1] - %d;\n",
label, label, TO_OFFSET);
printf(
" return c;\n"
"}\n"
"\n"
);
}
// Make only range tables and a function for is<label>rune.
static void
mkisronly(const char* label, char* prop)
{
mkisrange(label, prop, 1);
printf(
"int\n"
"is%srune(Rune c)\n"
"{\n"
" Rune *p;\n"
"\n"
" p = rbsearch(c, __is%sr, nelem(__is%sr)/2, 2);\n"
" if(p && c >= p[0] && c <= p[1])\n"
" return 1;\n"
" return 0;\n"
"}\n"
"\n",
label, label, label);
}
/*
* generate the body of runetype.
* assumes there is a function Rune* rbsearch(Rune c, Rune *t, int n, int ne);
*/
static void
mktables(char *src, int usepairs)
{
printf("/* generated automatically by mkrunetype.c from %s */\n\n", src);
/*
* we special case the space and digit tables, since they are assumed
* to be small with several ranges.
*/
mkisronly("space", myisspace);
mkisronly("digit", myisdigit);
mkis("alpha", myisalpha, 0);
mkis("upper", myisupper, usepairs);
mkis("lower", myislower, usepairs);
mkis("title", myistitle, usepairs);
mkto("upper", mytoupper, usepairs);
mkto("lower", mytolower, usepairs);
mkto("title", mytotitle, usepairs);
}
/*
* find differences between the newly generated tables and current runetypes.
*/
static void
check(void)
{
int i;
for(i = 0; i < NRUNES; i++){
if(isdigitrune(i) != myisdigit[i])
fprintf(stderr, "isdigit diff at %x: runetype=%x, unicode=%x\n",
i, isdigitrune(i), myisdigit[i]);
if(isspacerune(i) != myisspace[i])
fprintf(stderr, "isspace diff at %x: runetype=%x, unicode=%x\n",
i, isspacerune(i), myisspace[i]);
if(isupperrune(i) != myisupper[i])
fprintf(stderr, "isupper diff at %x: runetype=%x, unicode=%x\n",
i, isupperrune(i), myisupper[i]);
if(islowerrune(i) != myislower[i])
fprintf(stderr, "islower diff at %x: runetype=%x, unicode=%x\n",
i, islowerrune(i), myislower[i]);
if(isalpharune(i) != myisalpha[i])
fprintf(stderr, "isalpha diff at %x: runetype=%x, unicode=%x\n",
i, isalpharune(i), myisalpha[i]);
if(toupperrune(i) != mytoupper[i])
fprintf(stderr, "toupper diff at %x: runetype=%x, unicode=%x\n",
i, toupperrune(i), mytoupper[i]);
if(tolowerrune(i) != mytolower[i])
fprintf(stderr, "tolower diff at %x: runetype=%x, unicode=%x\n",
i, tolowerrune(i), mytolower[i]);
if(istitlerune(i) != myistitle[i])
fprintf(stderr, "istitle diff at %x: runetype=%x, unicode=%x\n",
i, istitlerune(i), myistitle[i]);
if(totitlerune(i) != mytotitle[i])
fprintf(stderr, "totitle diff at %x: runetype=%x, unicode=%x\n",
i, totitlerune(i), mytotitle[i]);
}
}
static int
mygetfields(char **fields, int nfields, char *str, const char *delim)
{
int nf;
fields[0] = str;
nf = 1;
if(nf >= nfields)
return nf;
for(; *str; str++){
if(strchr(delim, *str) != NULL){
*str = '\0';
fields[nf++] = str + 1;
if(nf >= nfields)
break;
}
}
return nf;
}
static int
getunicodeline(FILE *in, char **fields, char *buf)
{
char *p;
if(fgets(buf, MAX_LINE, in) == NULL)
return 0;
p = strchr(buf, '\n');
if (p == NULL)
fatal("line too long");
*p = '\0';
if (mygetfields(fields, NFIELDS + 1, buf, ";") != NFIELDS)
fatal("bad number of fields");
return 1;
}
static int
getcode(char *s)
{
int i, code;
code = 0;
i = 0;
/* Parse a hex number */
while(s[i]) {
code <<= 4;
if(s[i] >= '0' && s[i] <= '9')
code += s[i] - '0';
else if(s[i] >= 'A' && s[i] <= 'F')
code += s[i] - 'A' + 10;
else
fatal("bad code char '%c'", s[i]);
i++;
}
return code;
}
static void
fatal(const char *fmt, ...)
{
va_list arg;
fprintf(stderr, "%s: fatal error: ", argv0);
va_start(arg, fmt);
vfprintf(stderr, fmt, arg);
va_end(arg);
fprintf(stderr, "\n");
exit(1);
}