src/lib9/utf/mkrunetype.c

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
 * make is(upper|lower|title|space|alpha)rune and
 * to(upper|lower|title)rune from a UnicodeData.txt file.
 * these can be found at unicode.org
 *
 * with -c, runs a check of the existing runetype functions vs.
 * those extracted from UnicodeData.
 *
 * with -p, generates tables for pairs of chars, as well as for ranges
 * and singletons.
 *
 * UnicodeData defines 4 fields of interest:
 * 1) a category
 * 2) an upper case mapping
 * 3) a lower case mapping
 * 4) a title case mapping
 *
 * toupper, tolower, and totitle are defined directly from the mapping.
 *
 * isalpharune(c) is true iff c is a "letter" category
 * isupperrune(c) is true iff c is the target of toupperrune,
 *	or is in the uppercase letter category
 * similarly for islowerrune and istitlerune.
 * isspacerune is true for space category chars, "C" locale white space chars,
 *	and two additions:
 *	0085	"next line" control char
 *	feff]	"zero-width non-break space"
 * isdigitrune is true iff c is a numeric-digit category.
 */

#include <u.h>
#include <libc.h>
#include <stdio.h>
#include "utf.h"
#include "utfdef.h"

enum {
	/*
	 * fields in the unicode data file
	 */
	FIELD_CODE,
	FIELD_NAME,
	FIELD_CATEGORY,
	FIELD_COMBINING,
	FIELD_BIDIR,
	FIELD_DECOMP,
	FIELD_DECIMAL_DIG,
	FIELD_DIG,
	FIELD_NUMERIC_VAL,
	FIELD_MIRRORED,
	FIELD_UNICODE_1_NAME,
	FIELD_COMMENT,
	FIELD_UPPER,
	FIELD_LOWER,
	FIELD_TITLE,
	NFIELDS,

	MAX_LINE	= 1024,

	TO_OFFSET	= 1 << 20,

	NRUNES		= 1 << 21,
};

#define TO_DELTA(xmapped,x)	(TO_OFFSET + (xmapped) - (x))

static char	myisspace[NRUNES];
static char	myisalpha[NRUNES];
static char	myisdigit[NRUNES];
static char	myisupper[NRUNES];
static char	myislower[NRUNES];
static char	myistitle[NRUNES];

static int	mytoupper[NRUNES];
static int	mytolower[NRUNES];
static int	mytotitle[NRUNES];

static void	check(void);
static void	mktables(char *src, int usepairs);
static void	fatal(const char *fmt, ...);
static int	mygetfields(char **fields, int nfields, char *str, const char *delim);
static int	getunicodeline(FILE *in, char **fields, char *buf);
static int	getcode(char *s);

static void
usage(void)
{
	fprintf(stderr, "usage: mktables [-cp] <UnicodeData.txt>\n");
	exit(1);
}

void
main(int argc, char *argv[])
{
	FILE *in;
	char buf[MAX_LINE], buf2[MAX_LINE];
	char *fields[NFIELDS + 1], *fields2[NFIELDS + 1];
	char *p;
	int i, code, last, docheck, usepairs;

	docheck = 0;
	usepairs = 0;
	ARGBEGIN{
	case 'c':
		docheck = 1;
		break;
	case 'p':
		usepairs = 1;
		break;
	default:
		usage();
	}ARGEND

	if(argc != 1){
		usage();
	}

	in = fopen(argv[0], "r");
	if(in == NULL){
		fatal("can't open %s", argv[0]);
	}

	for(i = 0; i < NRUNES; i++){
		mytoupper[i] = i;
		mytolower[i] = i;
		mytotitle[i] = i;
	}

	/*
	 * make sure isspace has all of the "C" locale whitespace chars
	 */
	myisspace['\t'] = 1;
	myisspace['\n'] = 1;
	myisspace['\r'] = 1;
	myisspace['\f'] = 1;
	myisspace['\v'] = 1;

	/*
	 * a couple of other exceptions
	 */
	myisspace[0x85] = 1;	/* control char, "next line" */
	myisspace[0xfeff] = 1;	/* zero-width non-break space */

	last = -1;
	while(getunicodeline(in, fields, buf)){
		code = getcode(fields[FIELD_CODE]);
		if (code >= NRUNES)
			fatal("code-point value too big: %x", code);
		if(code <= last)
			fatal("bad code sequence: %x then %x", last, code);
		last = code;

		/*
		 * check for ranges
		 */
		p = fields[FIELD_CATEGORY];
		if(strstr(fields[FIELD_NAME], ", First>") != NULL){
			if(!getunicodeline(in, fields2, buf2))
				fatal("range start at eof");
			if (strstr(fields2[FIELD_NAME], ", Last>") == NULL)
				fatal("range start not followed by range end");
			last = getcode(fields2[FIELD_CODE]);
			if(last <= code)
				fatal("range out of sequence: %x then %x", code, last);
			if(strcmp(p, fields2[FIELD_CATEGORY]) != 0)
				fatal("range with mismatched category");
		}

		/*
		 * set properties and conversions
		 */
		for (; code <= last; code++){
			if(p[0] == 'L')
				myisalpha[code] = 1;
			if(p[0] == 'Z')
				myisspace[code] = 1;

			if(strcmp(p, "Lu") == 0)
				myisupper[code] = 1;
			if(strcmp(p, "Ll") == 0)
				myislower[code] = 1;

			if(strcmp(p, "Lt") == 0)
				myistitle[code] = 1;

			if(strcmp(p, "Nd") == 0)
				myisdigit[code] = 1;

			/*
			 * when finding conversions, also need to mark
			 * upper/lower case, since some chars, like
			 * "III" (0x2162), aren't defined as letters but have a
			 * lower case mapping ("iii" (0x2172)).
			 */
			if(fields[FIELD_UPPER][0] != '\0'){
				mytoupper[code] = getcode(fields[FIELD_UPPER]);
			}
			if(fields[FIELD_LOWER][0] != '\0'){
				mytolower[code] = getcode(fields[FIELD_LOWER]);
			}
			if(fields[FIELD_TITLE][0] != '\0'){
				mytotitle[code] = getcode(fields[FIELD_TITLE]);
			}
		}
	}

	fclose(in);

	/*
	 * check for codes with no totitle mapping but a toupper mapping.
	 * these appear in UnicodeData-2.0.14.txt, but are almost certainly
	 * erroneous.
	 */
	for(i = 0; i < NRUNES; i++){
		if(mytotitle[i] == i
		&& mytoupper[i] != i
		&& !myistitle[i])
			fprintf(stderr, "warning: code=%.4x not istitle, totitle is same, toupper=%.4x\n", i, mytoupper[i]);
	}

	/*
	 * make sure isupper[c] is true if for some x toupper[x]  == c
	 * ditto for islower and istitle
	 */
	for(i = 0; i < NRUNES; i++) {
		if(mytoupper[i] != i)
			myisupper[mytoupper[i]] = 1;
		if(mytolower[i] != i)
			myislower[mytolower[i]] = 1;
		if(mytotitle[i] != i)
			myistitle[mytotitle[i]] = 1;
	}

	if(docheck){
		check();
	}else{
		mktables(argv[0], usepairs);
	}
	exit(0);
}

/*
 * generate a properties array for ranges, clearing those cases covered.
 * if force, generate one-entry ranges for singletons.
 */
static int
mkisrange(const char* label, char* prop, int force)
{
	int start, stop, some;

	/*
	 * first, the ranges
	 */
	some = 0;
	for(start = 0; start < NRUNES; ) {
		if(!prop[start]){
			start++;
			continue;
		}

		for(stop = start + 1; stop < NRUNES; stop++){
			if(!prop[stop]){
				break;
			}
			prop[stop] = 0;
		}
		if(force || stop != start + 1){
			if(!some){
				printf("static Rune __is%sr[] = {\n", label);
				some = 1;
			}
			prop[start] = 0;
			printf("\t0x%.4x, 0x%.4x,\n", start, stop - 1);
		}

		start = stop;
	}
	if(some)
		printf("};\n\n");
	return some;
}

/*
 * generate a mapping array for pairs with a skip between,
 * clearing those entries covered.
 */
static int
mkispair(const char *label, char *prop)
{
	int start, stop, some;

	some = 0;
	for(start = 0; start + 2 < NRUNES; ) {
		if(!prop[start]){
			start++;
			continue;
		}

		for(stop = start + 2; stop < NRUNES; stop += 2){
			if(!prop[stop]){
				break;
			}
			prop[stop] = 0;
		}
		if(stop != start + 2){
			if(!some){
				printf("static Rune __is%sp[] = {\n", label);
				some = 1;
			}
			prop[start] = 0;
			printf("\t0x%.4x, 0x%.4x,\n", start, stop - 2);
		}

		start = stop;
	}
	if(some)
		printf("};\n\n");
	return some;
}

/*
 * generate a properties array for singletons, clearing those cases covered.
 */
static int
mkissingle(const char *label, char *prop)
{
	int start, some;

	some = 0;
	for(start = 0; start < NRUNES; start++) {
		if(!prop[start]){
			continue;
		}

		if(!some){
			printf("static Rune __is%ss[] = {\n", label);
			some = 1;
		}
		prop[start] = 0;
		printf("\t0x%.4x,\n", start);
	}
	if(some)
		printf("};\n\n");
	return some;
}

/*
 * generate tables and a function for is<label>rune
 */
static void
mkis(const char* label, char* prop, int usepairs)
{
	int isr, isp, iss;

	isr = mkisrange(label, prop, 0);
	isp = 0;
	if(usepairs)
		isp = mkispair(label, prop);
	iss = mkissingle(label, prop);

	printf(
		"int\n"
		"is%srune(Rune c)\n"
		"{\n"
		"	Rune *p;\n"
		"\n",
		label);

	if(isr)
		printf(
			"	p = rbsearch(c, __is%sr, nelem(__is%sr)/2, 2);\n"
			"	if(p && c >= p[0] && c <= p[1])\n"
			"		return 1;\n",
			label, label);

	if(isp)
		printf(
			"	p = rbsearch(c, __is%sp, nelem(__is%sp)/2, 2);\n"
			"	if(p && c >= p[0] && c <= p[1] && !((c - p[0]) & 1))\n"
			"		return 1;\n",
			label, label);

	if(iss)
		printf(
			"	p = rbsearch(c, __is%ss, nelem(__is%ss), 1);\n"
			"	if(p && c == p[0])\n"
			"		return 1;\n",
			label, label);


	printf(
		"	return 0;\n"
		"}\n"
		"\n"
	);
}

/*
 * generate a mapping array for ranges, clearing those entries covered.
 * if force, generate one-entry ranges for singletons.
 */
static int
mktorange(const char* label, int* map, int force)
{
	int start, stop, delta, some;

	some = 0;
	for(start = 0; start < NRUNES; ) {
		if(map[start] == start){
			start++;
			continue;
		}

		delta = TO_DELTA(map[start], start);
		if(delta != (Rune)delta)
			fatal("bad map delta %d", delta);
		for(stop = start + 1; stop < NRUNES; stop++){
			if(TO_DELTA(map[stop], stop) != delta){
				break;
			}
			map[stop] = stop;
		}
		if(stop != start + 1){
			if(!some){
				printf("static Rune __to%sr[] = {\n", label);
				some = 1;
			}
			map[start] = start;
			printf("\t0x%.4x, 0x%.4x, %d,\n", start, stop - 1, delta);
		}

		start = stop;
	}
	if(some)
		printf("};\n\n");
	return some;
}

/*
 * generate a mapping array for pairs with a skip between,
 * clearing those entries covered.
 */
static int
mktopair(const char* label, int* map)
{
	int start, stop, delta, some;

	some = 0;
	for(start = 0; start + 2 < NRUNES; ) {
		if(map[start] == start){
			start++;
			continue;
		}

		delta = TO_DELTA(map[start], start);
		if(delta != (Rune)delta)
			fatal("bad map delta %d", delta);
		for(stop = start + 2; stop < NRUNES; stop += 2){
			if(TO_DELTA(map[stop], stop) != delta){
				break;
			}
			map[stop] = stop;
		}
		if(stop != start + 2){
			if(!some){
				printf("static Rune __to%sp[] = {\n", label);
				some = 1;
			}
			map[start] = start;
			printf("\t0x%.4x, 0x%.4x, %d,\n", start, stop - 2, delta);
		}

		start = stop;
	}
	if(some)
		printf("};\n\n");
	return some;
}

/*
 * generate a mapping array for singletons, clearing those entries covered.
 */
static int
mktosingle(const char* label, int* map)
{
	int start, delta, some;

	some = 0;
	for(start = 0; start < NRUNES; start++) {
		if(map[start] == start){
			continue;
		}

		delta = TO_DELTA(map[start], start);
		if(delta != (Rune)delta)
			fatal("bad map delta %d", delta);
		if(!some){
			printf("static Rune __to%ss[] = {\n", label);
			some = 1;
		}
		map[start] = start;
		printf("\t0x%.4x, %d,\n", start, delta);
	}
	if(some)
		printf("};\n\n");
	return some;
}

/*
 * generate tables and a function for to<label>rune
 */
static void
mkto(const char* label, int* map, int usepairs)
{
	int tor, top, tos;

	tor = mktorange(label, map, 0);
	top = 0;
	if(usepairs)
		top = mktopair(label, map);
	tos = mktosingle(label, map);

	printf(
		"Rune\n"
		"to%srune(Rune c)\n"
		"{\n"
		"	Rune *p;\n"
		"\n",
		label);

	if(tor)
		printf(
			"	p = rbsearch(c, __to%sr, nelem(__to%sr)/3, 3);\n"
			"	if(p && c >= p[0] && c <= p[1])\n"
			"		return c + p[2] - %d;\n",
			label, label, TO_OFFSET);

	if(top)
		printf(
			"	p = rbsearch(c, __to%sp, nelem(__to%sp)/3, 3);\n"
			"	if(p && c >= p[0] && c <= p[1] && !((c - p[0]) & 1))\n"
			"		return c + p[2] - %d;\n",
			label, label, TO_OFFSET);

	if(tos)
		printf(
			"	p = rbsearch(c, __to%ss, nelem(__to%ss)/2, 2);\n"
			"	if(p && c == p[0])\n"
			"		return c + p[1] - %d;\n",
			label, label, TO_OFFSET);


	printf(
		"	return c;\n"
		"}\n"
		"\n"
	);
}

// Make only range tables and a function for is<label>rune.
static void
mkisronly(const char* label, char* prop)
{
	mkisrange(label, prop, 1);
	printf(
		"int\n"
		"is%srune(Rune c)\n"
		"{\n"
		"	Rune *p;\n"
		"\n"
		"	p = rbsearch(c, __is%sr, nelem(__is%sr)/2, 2);\n"
		"	if(p && c >= p[0] && c <= p[1])\n"
		"		return 1;\n"
		"	return 0;\n"
		"}\n"
		"\n",
	        label, label, label);
}

/*
 * generate the body of runetype.
 * assumes there is a function Rune* rbsearch(Rune c, Rune *t, int n, int ne);
 */
static void
mktables(char *src, int usepairs)
{
	printf("/* generated automatically by mkrunetype.c from %s */\n\n", src);

	/*
	 * we special case the space and digit tables, since they are assumed
	 * to be small with several ranges.
	 */
	mkisronly("space", myisspace);
	mkisronly("digit", myisdigit);

	mkis("alpha", myisalpha, 0);
	mkis("upper", myisupper, usepairs);
	mkis("lower", myislower, usepairs);
	mkis("title", myistitle, usepairs);

	mkto("upper", mytoupper, usepairs);
	mkto("lower", mytolower, usepairs);
	mkto("title", mytotitle, usepairs);
}

/*
 * find differences between the newly generated tables and current runetypes.
 */
static void
check(void)
{
	int i;

	for(i = 0; i < NRUNES; i++){
		if(isdigitrune(i) != myisdigit[i])
			fprintf(stderr, "isdigit diff at %x: runetype=%x, unicode=%x\n",
				i, isdigitrune(i), myisdigit[i]);

		if(isspacerune(i) != myisspace[i])
			fprintf(stderr, "isspace diff at %x: runetype=%x, unicode=%x\n",
				i, isspacerune(i), myisspace[i]);

		if(isupperrune(i) != myisupper[i])
			fprintf(stderr, "isupper diff at %x: runetype=%x, unicode=%x\n",
				i, isupperrune(i), myisupper[i]);

		if(islowerrune(i) != myislower[i])
			fprintf(stderr, "islower diff at %x: runetype=%x, unicode=%x\n",
				i, islowerrune(i), myislower[i]);

		if(isalpharune(i) != myisalpha[i])
			fprintf(stderr, "isalpha diff at %x: runetype=%x, unicode=%x\n",
				i, isalpharune(i), myisalpha[i]);

		if(toupperrune(i) != mytoupper[i])
			fprintf(stderr, "toupper diff at %x: runetype=%x, unicode=%x\n",
				i, toupperrune(i), mytoupper[i]);

		if(tolowerrune(i) != mytolower[i])
			fprintf(stderr, "tolower diff at %x: runetype=%x, unicode=%x\n",
				i, tolowerrune(i), mytolower[i]);

		if(istitlerune(i) != myistitle[i])
			fprintf(stderr, "istitle diff at %x: runetype=%x, unicode=%x\n",
				i, istitlerune(i), myistitle[i]);

		if(totitlerune(i) != mytotitle[i])
			fprintf(stderr, "totitle diff at %x: runetype=%x, unicode=%x\n",
				i, totitlerune(i), mytotitle[i]);


	}
}

static int
mygetfields(char **fields, int nfields, char *str, const char *delim)
{
	int nf;

	fields[0] = str;
	nf = 1;
	if(nf >= nfields)
		return nf;

	for(; *str; str++){
		if(strchr(delim, *str) != NULL){
			*str = '\0';
			fields[nf++] = str + 1;
			if(nf >= nfields)
				break;
		}
	}
	return nf;
}

static int
getunicodeline(FILE *in, char **fields, char *buf)
{
	char *p;

	if(fgets(buf, MAX_LINE, in) == NULL)
		return 0;

	p = strchr(buf, '\n');
	if (p == NULL)
		fatal("line too long");
	*p = '\0';

	if (mygetfields(fields, NFIELDS + 1, buf, ";") != NFIELDS)
		fatal("bad number of fields");

	return 1;
}

static int
getcode(char *s)
{
	int i, code;

	code = 0;
	i = 0;
	/* Parse a hex number */
	while(s[i]) {
		code <<= 4;
		if(s[i] >= '0' && s[i] <= '9')
			code += s[i] - '0';
		else if(s[i] >= 'A' && s[i] <= 'F')
			code += s[i] - 'A' + 10;
		else
			fatal("bad code char '%c'", s[i]);
		i++;
	}
	return code;
}

static void
fatal(const char *fmt, ...)
{
	va_list arg;

	fprintf(stderr, "%s: fatal error: ", argv0);
	va_start(arg, fmt);
	vfprintf(stderr, fmt, arg);
	va_end(arg);
	fprintf(stderr, "\n");

	exit(1);
}