- implement scanner token stream via channel
- change test_scanner to scan using both methods
- add -pscan flag to Go front-end to choose between conventional
synchronous or parallel asynchronous scanning
R=r
OCL=13937
CL=13937
diff --git a/usr/gri/gosrc/compilation.go b/usr/gri/gosrc/compilation.go
index dd10471..0f284d3 100644
--- a/usr/gri/gosrc/compilation.go
+++ b/usr/gri/gosrc/compilation.go
@@ -26,9 +26,15 @@
scanner := new(Scanner.Scanner);
scanner.Open(file_name, src);
+
+ var tstream *chan *Scanner.Token;
+ if comp.flags.pscan {
+ tstream = new(chan *Scanner.Token, 100);
+ go scanner.Server(tstream);
+ }
parser := new(Parser.Parser);
- parser.Open(comp, scanner);
+ parser.Open(comp, scanner, tstream);
parser.ParseProgram();
if parser.S.nerrors > 0 {
diff --git a/usr/gri/gosrc/globals.go b/usr/gri/gosrc/globals.go
index ed42e4a..121052c 100644
--- a/usr/gri/gosrc/globals.go
+++ b/usr/gri/gosrc/globals.go
@@ -63,7 +63,8 @@
print_export bool;
semantic_checks bool;
verbose int;
- sixg bool;
+ sixg bool; // 6g compatibility
+ pscan bool; // parallel scanning using a token channel
}
diff --git a/usr/gri/gosrc/go.go b/usr/gri/gosrc/go.go
index 31f1b87..1097c4e 100644
--- a/usr/gri/gosrc/go.go
+++ b/usr/gri/gosrc/go.go
@@ -23,6 +23,7 @@
print " -v verbose mode\n";
print " -vv very verbose mode\n";
print " -6g 6g compatibility mode\n";
+ print " -pscan scan and parse in parallel (use token channel)\n";
}
@@ -43,6 +44,7 @@
case "-v": flags.verbose = 1;
case "-vv": flags.verbose = 2;
case "-6g": flags.sixg = true;
+ case "-pscan": flags.pscan = true;
default: files.AddStr(arg);
}
}
diff --git a/usr/gri/gosrc/parser.go b/usr/gri/gosrc/parser.go
index 3433b63..2e2346e 100644
--- a/usr/gri/gosrc/parser.go
+++ b/usr/gri/gosrc/parser.go
@@ -19,6 +19,7 @@
semantic_checks bool;
verbose, indent int;
S *Scanner.Scanner;
+ C *chan *Scanner.Token;
// Token
tok int; // one token look-ahead
@@ -62,7 +63,12 @@
func (P *Parser) Next() {
- P.tok, P.pos, P.val = P.S.Scan();
+ if P.C == nil {
+ P.tok, P.pos, P.val = P.S.Scan();
+ } else {
+ t := <- P.C;
+ P.tok, P.pos, P.val = t.tok, t.pos, t.val;
+ }
if P.verbose > 1 {
P.PrintIndent();
print "[", P.pos, "] ", Scanner.TokenName(P.tok), "\n";
@@ -70,12 +76,13 @@
}
-func (P *Parser) Open(comp *Globals.Compilation, S *Scanner.Scanner) {
+func (P *Parser) Open(comp *Globals.Compilation, S *Scanner.Scanner, C *chan *Scanner.Token) {
P.comp = comp;
P.semantic_checks = comp.flags.semantic_checks;
P.verbose = comp.flags.verbose;
P.indent = 0;
P.S = S;
+ P.C = C;
P.Next();
P.level = 0;
P.top_scope = Universe.scope;
diff --git a/usr/gri/gosrc/scanner.go b/usr/gri/gosrc/scanner.go
index add320e..a50ad2a 100644
--- a/usr/gri/gosrc/scanner.go
+++ b/usr/gri/gosrc/scanner.go
@@ -804,3 +804,22 @@
return tok, pos, val;
}
+
+
+export type Token struct {
+ pos int;
+ tok int;
+ val string;
+}
+
+
+func (S *Scanner) Server(c *chan *Token) {
+ for {
+ t := new(Token);
+ t.tok, t.pos, t.val = S.Scan();
+ c -< t;
+ if t.tok == EOF {
+ break;
+ }
+ }
+}
diff --git a/usr/gri/gosrc/test_scanner.go b/usr/gri/gosrc/test_scanner.go
index 2ce097f..5c23acf 100644
--- a/usr/gri/gosrc/test_scanner.go
+++ b/usr/gri/gosrc/test_scanner.go
@@ -7,7 +7,7 @@
import Scanner "scanner"
-func Scan(filename, src string) {
+func Scan1(filename, src string) {
S := new(Scanner.Scanner);
S.Open(filename, src);
for {
@@ -24,16 +24,41 @@
}
+func Scan2(filename, src string) {
+ S := new(Scanner.Scanner);
+ S.Open(filename, src);
+ c := new(chan *Scanner.Token, 32);
+ go S.Server(c);
+ for {
+ var t *Scanner.Token;
+ t = <- c;
+ tok, pos, val := t.tok, t.pos, t.val;
+ print pos, ": ", Scanner.TokenName(tok);
+ if tok == Scanner.IDENT || tok == Scanner.INT || tok == Scanner.FLOAT || tok == Scanner.STRING {
+ print " ", val;
+ }
+ print "\n";
+ if tok == Scanner.EOF {
+ return;
+ }
+ }
+}
+
+
func main() {
for i := 1; i < sys.argc(); i++ {
var src string;
var ok bool;
src, ok = sys.readfile(sys.argv(i));
if ok {
- print "scanning " + sys.argv(i) + "\n";
- Scan(sys.argv(i), src);
+ print "scanning (standard) " + sys.argv(i) + "\n";
+ Scan1(sys.argv(i), src);
+ print "\n";
+ print "scanning (channels) " + sys.argv(i) + "\n";
+ Scan2(sys.argv(i), src);
} else {
print "error: cannot read " + sys.argv(i) + "\n";
}
+ print "\n";
}
}
