internal/lsp.protocol: identify the version of the LSP that code is generated for

Changes go.ts to check that the commit hash of the vscode is the one that it
is expecting. README.md now contains more explanation.

Change-Id: Ia5a947c6d5d026c2b7d9ab18877c320e8a7f45d2
Reviewed-on: https://go-review.googlesource.com/c/tools/+/195438
Reviewed-by: Ian Cottrell <iancottrell@google.com>
diff --git a/internal/lsp/protocol/typescript/README.md b/internal/lsp/protocol/typescript/README.md
index e62ca38..2c93997 100644
--- a/internal/lsp/protocol/typescript/README.md
+++ b/internal/lsp/protocol/typescript/README.md
@@ -1,4 +1,4 @@
-# Generate Go types for the LSP protocol
+# Generate Go types and signatures for the LSP protocol
 
 ## Setup
 
@@ -8,7 +8,6 @@
 2. Install the typescript compiler, with `npm install typescript`
 3. Make sure `tsc` and `node` are in your execution path.
 4. Get the typescript code for the jsonrpc protocol with `git clone git@github.com:microsoft/vscode-languageserver-node.git`
-5. go.ts and requests.ts, and the files they generate, are from commit 8801c20b667945f455d7e023c71d2f741caeda25
 
 ## Usage
 
@@ -23,13 +22,19 @@
 It defaults to `$(HOME)`.
 
 `-o out.go` says where the generated go code goes.
-It defaults to `/tmp/tsprotocol.go`.
+It defaults to `tsprotocol.go`.
 
 To generate the client and server boilerplate (tsclient.go and tsserver.go)
 ```tsc requests.ts && node requests.js [-d dir] && gofmt -w tsclient.go tsserver.go```
 
 -d dir is the same as above. The output files are written into the current directory.
 
-## Note
+## Notes
 
-`go.ts` and `requests.ts` use the Typescript compiler's API, which is [introduced](https://github.com/Microsoft/TypeScript/wiki/Architectural-Overview) in their wiki.
+1. `go.ts` and `requests.ts` use the Typescript compiler's API, which is [introduced](https://github.com/Microsoft/TypeScript/wiki/Architectural-Overview) in their wiki.
+2. Because the Typescript and Go type systems are incompatible, `go.ts ` and `request.ts` are filled with heuristics and special cases. Therefore they are tied to a specific commit of `vscode-languageserver-node`. The hash code of the commit is included in the header of `tsprotocol.go` and stored in the variable `gitHash` in `go.ts`. It is checked (see `git()` in `go.ts`) on every execution of `go.ts`.
+3. Generating the `ts*.go` files is only semi-automated. Please file an issue if the released version is too far behind.
+4. For the impatient, first change `gitHash` by hand (`git()` shows how to find the hash).
+    1. Then try to run `go.ts` and  `requests.ts`. This will likely fail because the heuristics don't cover some new case. For instance, some simple type like `string` might have changed to a union type `string | [number,number]`. (Look at the `UnionTypeNode` code near line 588 of `go.ts`.) Another example is that some formal parameter generated by `requests.ts` will have anonymous structure type, which is essentially unusable. (See the code related to `ourTypes`.)
+    1. Next step is to try to move the generated code to `internal/lsp/protocol` and try to build `gopls` and its tests. This will likely fail because types have changed. Generally the fixes are fairly easy. (The code for `ourTypes` was a case where changes had to be made to `requests.ts`.)
+    1. Since there are not adequate integration tests, the next step is to run `gopls`. A common failure will be a nil dereference, because some previously simple default is now in an optional structure.
\ No newline at end of file
diff --git a/internal/lsp/protocol/typescript/go.ts b/internal/lsp/protocol/typescript/go.ts
index 0f86cb9..3f4290c 100644
--- a/internal/lsp/protocol/typescript/go.ts
+++ b/internal/lsp/protocol/typescript/go.ts
@@ -53,7 +53,8 @@
   `${srcDir}/protocol/src/protocol.ts`, `${srcDir}/types/src/main.ts`,
   `${srcDir}/jsonrpc/src/main.ts`
 ];
-let outFname = '/tmp/tsprotocol.go';
+let gitHash = 'fda16d6b63ba0fbdbd21d437ea810685528a0018';
+let outFname = 'tsprotocol.go';
 let fda: number, fdb: number, fde: number;  // file descriptors
 
 function createOutputFiles() {
@@ -101,9 +102,9 @@
   function genTypes(node: ts.Node) {
     // Ignore top-level items that produce no output
     if (ts.isExpressionStatement(node) || ts.isFunctionDeclaration(node) ||
-        ts.isImportDeclaration(node) || ts.isVariableStatement(node) ||
-        ts.isExportDeclaration(node) || ts.isEmptyStatement(node) ||
-        node.kind == ts.SyntaxKind.EndOfFileToken) {
+      ts.isImportDeclaration(node) || ts.isVariableStatement(node) ||
+      ts.isExportDeclaration(node) || ts.isEmptyStatement(node) ||
+      node.kind == ts.SyntaxKind.EndOfFileToken) {
       return;
     }
     if (ts.isInterfaceDeclaration(node)) {
@@ -139,8 +140,8 @@
         return
       }
       if (n.kind == ts.SyntaxKind.Constructor || ts.isMethodDeclaration(n) ||
-          ts.isGetAccessor(n) || ts.isSetAccessor(n) ||
-          ts.isTypeParameterDeclaration(n)) {
+        ts.isGetAccessor(n) || ts.isSetAccessor(n) ||
+        ts.isTypeParameterDeclaration(n)) {
         bad = true;
         return
       }
@@ -290,7 +291,7 @@
       }
     })
     let goName = toGoName(id.text)
-    let {goType, gostuff, optional, fields} = computeType(thing)
+    let { goType, gostuff, optional, fields } = computeType(thing)
     // Generics
     if (gen && gen.text == goType) goType = 'interface{}';
     opt = opt || optional;
@@ -451,7 +452,7 @@
         } else
           throw new Error(`expected TypeRef ${strKind(n)} ${loc(n)}`)
       })
-      let ans = {me: node, name: toGoName(getText(id)), embeds: embeds};
+      let ans = { me: node, name: toGoName(getText(id)), embeds: embeds };
       Structs.push(ans)
       return
     }
@@ -478,7 +479,7 @@
       return
     }
     throw new Error(
-        `in doTypeAlias ${loc(alias)} ${kinds(node)}: ${strKind(alias)}\n`)
+      `in doTypeAlias ${loc(alias)} ${kinds(node)}: ${strKind(alias)}\n`)
   }
 
   // string, or number, or DocumentFilter
@@ -494,11 +495,11 @@
       }
       if (ts.isLiteralTypeNode(n)) {
         n.literal.kind == ts.SyntaxKind.NumericLiteral ? aNumber = true :
-                                                         aString = true;
+          aString = true;
         return;
       }
       if (n.kind == ts.SyntaxKind.NumberKeyword ||
-          n.kind == ts.SyntaxKind.StringKeyword) {
+        n.kind == ts.SyntaxKind.StringKeyword) {
         n.kind == ts.SyntaxKind.NumberKeyword ? aNumber = true : aString = true;
         return
       }
@@ -522,25 +523,24 @@
   }
 
   // complex and filled with heuristics
-  function computeType(node: ts.Node):
-      {goType: string, gostuff?: string, optional?: boolean, fields?: Field[]} {
+  function computeType(node: ts.Node): { goType: string, gostuff?: string, optional?: boolean, fields?: Field[] } {
     switch (node.kind) {
       case ts.SyntaxKind.AnyKeyword:
       case ts.SyntaxKind.ObjectKeyword:
-        return {goType: 'interface{}'};
+        return { goType: 'interface{}' };
       case ts.SyntaxKind.BooleanKeyword:
-        return {goType: 'bool'};
+        return { goType: 'bool' };
       case ts.SyntaxKind.NumberKeyword:
-        return {goType: 'float64'};
+        return { goType: 'float64' };
       case ts.SyntaxKind.StringKeyword:
-        return {goType: 'string'};
+        return { goType: 'string' };
       case ts.SyntaxKind.NullKeyword:
       case ts.SyntaxKind.UndefinedKeyword:
-        return {goType: 'nil'};
+        return { goType: 'nil' };
     }
     if (ts.isArrayTypeNode(node)) {
-      let {goType, gostuff, optional} = computeType(node.elementType)
-      return ({goType: '[]' + goType, gostuff: gostuff, optional: optional})
+      let { goType, gostuff, optional } = computeType(node.elementType)
+      return ({ goType: '[]' + goType, gostuff: gostuff, optional: optional })
     } else if (ts.isTypeReferenceNode(node)) {
       // typeArguments?: NodeArray<TypeNode>;typeName: EntityName;
       // typeArguments won't show up in the generated Go
@@ -549,10 +549,10 @@
       if (ts.isQualifiedName(tn)) {
         throw new Error(`qualified name at ${loc(node)}`);
       } else if (ts.isIdentifier(tn)) {
-        return {goType: toGoName(tn.text)};
+        return { goType: toGoName(tn.text) };
       } else {
         throw new Error(
-            `expected identifier got ${strKind(node.typeName)} at ${loc(tn)}`)
+          `expected identifier got ${strKind(node.typeName)} at ${loc(tn)}`)
       }
     } else if (ts.isLiteralTypeNode(node)) {
       // string|float64 (are there other possibilities?)
@@ -562,7 +562,7 @@
       if (txt.charAt(0) == '\'') {
         typ = 'string'
       }
-      return {goType: typ, gostuff: getText(node)};
+      return { goType: typ, gostuff: getText(node) };
     } else if (ts.isTypeLiteralNode(node)) {
       // {[uri:string]: TextEdit[];} -> map[string][]TextEdit
       let x: Field[] = [];
@@ -581,33 +581,33 @@
       if (indexCnt > 0) {
         if (indexCnt != 1 || x.length != 1)
           throw new Error(`undexpected Index ${loc(x[0].me)}`)
-          // instead of {map...} just the map
-          return ({goType: x[0].goType, gostuff: x[0].gostuff})
+        // instead of {map...} just the map
+        return ({ goType: x[0].goType, gostuff: x[0].gostuff })
       }
-      return ({goType: 'embedded!', fields: x})
+      return ({ goType: 'embedded!', fields: x })
     } else if (ts.isUnionTypeNode(node)) {
       // The major heuristics
-      let x = new Array<{goType: string, gostuff?: string, optiona?: boolean}>()
-      node.forEachChild((n: ts.Node) => {x.push(computeType(n))})
+      let x = new Array<{ goType: string, gostuff?: string, optiona?: boolean }>()
+      node.forEachChild((n: ts.Node) => { x.push(computeType(n)) })
       if (x.length == 2 && x[1].goType == 'nil') {
         // Foo | null, or Foo | undefined
         return x[0]  // make it optional somehow? TODO
       }
       if (x[0].goType == 'bool') {  // take it, mostly
         if (x[1].goType == 'RenameOptions' ||
-            x[1].goType == 'CodeActionOptions') {
-          return ({goType: 'interface{}', gostuff: getText(node)})
+          x[1].goType == 'CodeActionOptions') {
+          return ({ goType: 'interface{}', gostuff: getText(node) })
         }
-        return ({goType: 'bool', gostuff: getText(node)})
+        return ({ goType: 'bool', gostuff: getText(node) })
       }
       // these are special cases from looking at the source
       let gostuff = getText(node);
       if (x[0].goType == `"off"` || x[0].goType == 'string') {
-        return ({goType: 'string', gostuff: gostuff})
+        return ({ goType: 'string', gostuff: gostuff })
       }
       if (x[0].goType == 'TextDocumentSyncOptions') {
         // TextDocumentSyncOptions | TextDocumentSyncKind
-        return ({goType: 'interface{}', gostuff: gostuff})
+        return ({ goType: 'interface{}', gostuff: gostuff })
       }
       if (x[0].goType == 'float64' && x[1].goType == 'string') {
         return {
@@ -658,8 +658,8 @@
       if (ts.isParameter(n)) {
         parm = n
       } else if (
-          ts.isArrayTypeNode(n) || n.kind == ts.SyntaxKind.AnyKeyword ||
-          ts.isUnionTypeNode(n)) {
+        ts.isArrayTypeNode(n) || n.kind == ts.SyntaxKind.AnyKeyword ||
+        ts.isUnionTypeNode(n)) {
         at = n
       } else
         throw new Error(`fromIndexSig ${strKind(n)} ${loc(n)}`)
@@ -676,8 +676,8 @@
     goType = `map[string]${goType}`
     return {
       me: node, goName: toGoName(id.text), id: null, goType: goType,
-          optional: false, json: `\`json:"${id.text}"\``,
-          gostuff: `${getText(node)}`
+      optional: false, json: `\`json:"${id.text}"\``,
+      gostuff: `${getText(node)}`
     }
   }
 
@@ -686,7 +686,7 @@
     if (s.charAt(0) == '_') {
       ans = 'Inner' + s.substring(1)
     }
-    else {ans = s.substring(0, 1).toUpperCase() + s.substring(1)};
+    else { ans = s.substring(0, 1).toUpperCase() + s.substring(1) };
     ans = ans.replace(/Uri$/, 'URI')
     ans = ans.replace(/Id$/, 'ID')
     return ans
@@ -702,7 +702,7 @@
   // return a string of the kinds of the immediate descendants
   function kinds(n: ts.Node): string {
     let res = 'Seen ' + strKind(n);
-    function f(n: ts.Node): void{res += ' ' + strKind(n)};
+    function f(n: ts.Node): void { res += ' ' + strKind(n) };
     ts.forEachChild(n, f)
     return res
   }
@@ -752,7 +752,7 @@
         let m = n.members
         pra(`${indent} ${loc(n)} ${strKind(n)} ${m.length}\n`)
       }
-      else {pra(`${indent} ${loc(n)} ${strKind(n)}\n`)};
+      else { pra(`${indent} ${loc(n)} ${strKind(n)}\n`) };
       indent += '  '
       ts.forEachChild(n, f)
       indent = indent.slice(0, indent.length - 2)
@@ -769,14 +769,15 @@
   return x
 }
 
-function loc(node: ts.Node): string{const sf = node.getSourceFile()
-const start = node.getStart()
-const x = sf.getLineAndCharacterOfPosition(start)
-const full = node.getFullStart()
-const y = sf.getLineAndCharacterOfPosition(full)
-let fn = sf.fileName
-const n = fn.search(/-node./)
-fn = fn.substring(n + 6)
+function loc(node: ts.Node): string {
+  const sf = node.getSourceFile();
+  const start = node.getStart()
+  const x = sf.getLineAndCharacterOfPosition(start)
+  const full = node.getFullStart()
+  const y = sf.getLineAndCharacterOfPosition(full)
+  let fn = sf.fileName
+  const n = fn.search(/-node./)
+  fn = fn.substring(n + 6)
   return `${fn} ${x.line + 1}:${x.character + 1} (${y.line + 1}:${
     y.character + 1})`
 }
@@ -796,248 +797,270 @@
 }
 
 let byName = new Map<string, Struct>();
-  function emitStructs() {
-    dontEmit.set('Thenable', true);
-    dontEmit.set('EmitterOptions', true);
-    dontEmit.set('MessageReader', true);
-    dontEmit.set('MessageWriter', true);
-    dontEmit.set('CancellationToken', true);
-    dontEmit.set('PipeTransport', true);
-    dontEmit.set('SocketTransport', true);
-    dontEmit.set('Item', true);
-    dontEmit.set('Event', true);
-    dontEmit.set('Logger', true);
-    dontEmit.set('Disposable', true);
-    dontEmit.set('PartialMessageInfo', true);
-    dontEmit.set('MessageConnection', true);
-    dontEmit.set('ResponsePromise', true);
-    dontEmit.set('ResponseMessage', true);
-    dontEmit.set('ErrorMessage', true);
-    dontEmit.set('NotificationMessage', true);
-    dontEmit.set('RequestHandlerElement', true);
-    dontEmit.set('RequestMessage', true);
-    dontEmit.set('NotificationHandlerElement', true);
-    dontEmit.set('Message', true);  // duplicate of jsonrpc2:wire.go
-    dontEmit.set('LSPLogMessage', true);
-    dontEmit.set('InnerEM', true);
-    dontEmit.set('ResponseErrorLiteral', true);
-    dontEmit.set('TraceOptions', true);
-    dontEmit.set('MessageType', true);  // want the enum
-    // backwards compatibility, done in requests.ts:
-    dontEmit.set('CancelParams', true);
+function emitStructs() {
+  dontEmit.set('Thenable', true);
+  dontEmit.set('EmitterOptions', true);
+  dontEmit.set('MessageReader', true);
+  dontEmit.set('MessageWriter', true);
+  dontEmit.set('CancellationToken', true);
+  dontEmit.set('PipeTransport', true);
+  dontEmit.set('SocketTransport', true);
+  dontEmit.set('Item', true);
+  dontEmit.set('Event', true);
+  dontEmit.set('Logger', true);
+  dontEmit.set('Disposable', true);
+  dontEmit.set('PartialMessageInfo', true);
+  dontEmit.set('MessageConnection', true);
+  dontEmit.set('ResponsePromise', true);
+  dontEmit.set('ResponseMessage', true);
+  dontEmit.set('ErrorMessage', true);
+  dontEmit.set('NotificationMessage', true);
+  dontEmit.set('RequestHandlerElement', true);
+  dontEmit.set('RequestMessage', true);
+  dontEmit.set('NotificationHandlerElement', true);
+  dontEmit.set('Message', true);  // duplicate of jsonrpc2:wire.go
+  dontEmit.set('LSPLogMessage', true);
+  dontEmit.set('InnerEM', true);
+  dontEmit.set('ResponseErrorLiteral', true);
+  dontEmit.set('TraceOptions', true);
+  dontEmit.set('MessageType', true);  // want the enum
+  // backwards compatibility, done in requests.ts:
+  dontEmit.set('CancelParams', true);
 
-    for (const str of Structs) {
-      byName.set(str.name, str)
-    }
-    let seenName = new Map<string, boolean>()
-    for (const str of Structs) {
-      if (str.name == 'InitializeError') {
-        // only want its consts, not the struct
-        continue
-      }
-      if (seenName.get(str.name) || dontEmit.get(str.name)) {
-        continue
-      }
-      let noopt = false;
-      seenName.set(str.name, true)
-      prgo(genComments(str.name, getComments(str.me)))
-      prgo(`type ${str.name} struct {\n`)
-      // if it has fields, generate them
-      if (str.fields != undefined) {
-        for (const f of str.fields) {
-          prgo(strField(f, noopt))
-        }
-      }
-      if (str.extends) {
-        // ResourceOperation just repeats the Kind field
-        for (const s of str.extends) {
-          if (s != 'ResourceOperation')
-            prgo(`\t${s}\n`)  // what this type extends.
-        }
-      } else if (str.embeds) {
-        prb(`embeds: ${str.name}\n`);
-        noopt = (str.name == 'ClientCapabilities');
-        // embedded struct. the hard case is from intersection types,
-        // where fields with the same name have to be combined into
-        // a single struct
-        let fields = new Map<string, Field[]>();
-        for (const e of str.embeds) {
-          const nm = byName.get(e);
-          if (nm.embeds) throw new Error(`${nm.name} is an embedded embed`);
-          // each of these fields might be a something that needs amalgamating
-          for (const f of nm.fields) {
-            let x = fields.get(f.goName);
-            if (x === undefined) x = [];
-            x.push(f);
-            fields.set(f.goName, x);
-          }
-        }
-        fields.forEach((val, key) => {
-          if (val.length > 1) {
-            // merge the fields with the same name
-            prgo(strField(val[0], noopt, val));
-          } else {
-            prgo(strField(val[0], noopt));
-          }
-        });
-      }
-      prgo(`}\n`);
-    }
+  for (const str of Structs) {
+    byName.set(str.name, str)
   }
-
-  function genComments(name: string, maybe: string): string {
-    if (maybe == '') return `\n\t// ${name} is\n`;
-    if (maybe.indexOf('/**') == 0) {
-      return maybe.replace('/**', `\n/*${name} defined:`)
+  let seenName = new Map<string, boolean>()
+  for (const str of Structs) {
+    if (str.name == 'InitializeError') {
+      // only want its consts, not the struct
+      continue
     }
-    throw new Error(`weird comment ${maybe.indexOf('/**')}`)
-  }
-
-  // Turn a Field into an output string
-  // flds is for merging
-  function strField(f: Field, noopt?: boolean, flds?: Field[]): string {
-    let ans: string[] = [];
-    let opt = (!noopt && f.optional) ? '*' : ''
-    switch (f.goType.charAt(0)) {
-      case 's':  // string
-      case 'b':  // bool
-      case 'f':  // float64
-      case 'i':  // interface{}
-      case '[':  // []foo
-        opt = ''
+    if (seenName.get(str.name) || dontEmit.get(str.name)) {
+      continue
     }
-    let stuff = (f.gostuff == undefined) ? '' : ` // ${f.gostuff}`
-    ans.push(genComments(f.goName, getComments(f.me)))
-    if (flds === undefined && f.substruct == undefined) {
-      ans.push(`\t${f.goName} ${opt}${f.goType} ${f.json}${stuff}\n`)
+    let noopt = false;
+    seenName.set(str.name, true)
+    prgo(genComments(str.name, getComments(str.me)))
+    prgo(`type ${str.name} struct {\n`)
+    // if it has fields, generate them
+    if (str.fields != undefined) {
+      for (const f of str.fields) {
+        prgo(strField(f, noopt))
+      }
     }
-    else if (flds !== undefined) {
-      // The logic that got us here is imprecise, so it is possible that
-      // the fields are really all the same, and don't need to be
-      // combined into a struct.
-      let simple = true;
-      for (const ff of flds) {
-        if (ff.substruct !== undefined || byName.get(ff.goType) !== undefined) {
-          simple = false
-          break
+    if (str.extends) {
+      // ResourceOperation just repeats the Kind field
+      for (const s of str.extends) {
+        if (s != 'ResourceOperation')
+          prgo(`\t${s}\n`)  // what this type extends.
+      }
+    } else if (str.embeds) {
+      prb(`embeds: ${str.name}\n`);
+      noopt = (str.name == 'ClientCapabilities');
+      // embedded struct. the hard case is from intersection types,
+      // where fields with the same name have to be combined into
+      // a single struct
+      let fields = new Map<string, Field[]>();
+      for (const e of str.embeds) {
+        const nm = byName.get(e);
+        if (nm.embeds) throw new Error(`${nm.name} is an embedded embed`);
+        // each of these fields might be a something that needs amalgamating
+        for (const f of nm.fields) {
+          let x = fields.get(f.goName);
+          if (x === undefined) x = [];
+          x.push(f);
+          fields.set(f.goName, x);
         }
       }
-      if (simple) {
-        // should check that the ffs are really all the same
-        return strField(flds[0], noopt)
-      }
-      ans.push(`\t${f.goName} ${opt}struct{\n`);
-      for (const ff of flds) {
-        if (ff.substruct !== undefined) {
-          for (const x of ff.substruct) {
-            ans.push(strField(x, noopt))
-          }
-        } else if (byName.get(ff.goType) !== undefined) {
-          const st = byName.get(ff.goType);
-          for (let i = 0; i < st.fields.length; i++) {
-            ans.push(strField(st.fields[i], noopt))
-          }
+      fields.forEach((val, key) => {
+        if (val.length > 1) {
+          // merge the fields with the same name
+          prgo(strField(val[0], noopt, val));
         } else {
-          ans.push(strField(ff, noopt));
+          prgo(strField(val[0], noopt));
         }
-      }
-      ans.push(`\t} ${f.json}${stuff}\n`);
+      });
     }
-    else {
-      ans.push(`\t${f.goName} ${opt}struct {\n`)
-      for (const x of f.substruct) {
-        ans.push(strField(x, noopt))
-      }
-      ans.push(`\t} ${f.json}${stuff}\n`)
-    }
-    return (''.concat(...ans))
+    prgo(`}\n`);
   }
+}
 
-  function emitConsts() {
-    // need the consts too! Generate modifying prefixes and suffixes to ensure
-    // consts are unique. (Go consts are package-level, but Typescript's are
-    // not.) Use suffixes to minimize changes to gopls.
-    let pref = new Map<string, string>([
-      ['DiagnosticSeverity', 'Severity'], ['WatchKind', 'Watch']
-    ])  // typeName->prefix
-    let suff = new Map<string, string>([
-      ['CompletionItemKind', 'Completion'], ['InsertTextFormat', 'TextFormat']
-    ])
-    for (const c of Consts) {
-      if (seenConstTypes.get(c.typeName)) {
-        continue
-      }
-      seenConstTypes.set(c.typeName, true);
-      if (pref.get(c.typeName) == undefined) {
-        pref.set(c.typeName, '')  // initialize to empty value
-      }
-      if (suff.get(c.typeName) == undefined) {
-        suff.set(c.typeName, '')
-      }
-      prgo(`// ${c.typeName} defines constants\n`)
-      prgo(`type ${c.typeName} ${c.goType}\n`)
-    }
-    prgo('const (\n')
-    let seenConsts = new Map<string, boolean>()  // to avoid duplicates
-    for (const c of Consts) {
-      const x = `${pref.get(c.typeName)}${c.name}${suff.get(c.typeName)}`
-      if (seenConsts.get(x)) {
-        continue
-      }
-      seenConsts.set(x, true)
-      if (c.value === undefined) continue;      // didn't figure it out
-      if (x.startsWith('undefined')) continue;  // what's going on here?
-      prgo(genComments(x, getComments(c.me)))
-      prgo(`\t${x} ${c.typeName} = ${c.value}\n`)
-    }
-    prgo(')\n')
+function genComments(name: string, maybe: string): string {
+  if (maybe == '') return `\n\t// ${name} is\n`;
+  if (maybe.indexOf('/**') == 0) {
+    return maybe.replace('/**', `\n/*${name} defined:`)
   }
+  throw new Error(`weird comment ${maybe.indexOf('/**')}`)
+}
 
-  function emitHeader(files: string[]) {
-    let lastMod = 0
-    let lastDate: Date
-    for (const f of files) {
-      const st = fs.statSync(f)
-      if (st.mtimeMs > lastMod) {
-        lastMod = st.mtimeMs
-        lastDate = st.mtime
-      }
-    }
-    let a = fs.readFileSync(`${dir}${srcDir}/.git/refs/heads/master`);
-    prgo(`// Package protocol contains data types and code for LSP jsonrpcs\n`)
-    prgo(`// generated automatically from vscode-languageserver-node\n`)
-    prgo(`// commit: ${a.toString()}`)
-    prgo(`// last fetched ${lastDate}\n`)
-    prgo('package protocol\n\n')
-    prgo(`// Code generated (see typescript/README.md) DO NOT EDIT.\n`);
-  };
-
-  // ad hoc argument parsing: [-d dir] [-o outputfile], and order matters
-  function main() {
-    let args = process.argv.slice(2)  // effective command line
-    if (args.length > 0) {
-      let j = 0;
-      if (args[j] == '-d') {
-        dir = args[j + 1]
-        j += 2
-      }
-      if (args[j] == '-o') {
-        outFname = args[j + 1]
-        j += 2
-      }
-      if (j != args.length) throw new Error(`incomprehensible args ${args}`)
-    }
-    let files: string[] = [];
-    for (let i = 0; i < fnames.length; i++) {
-      files.push(`${dir}${fnames[i]}`)
-    }
-    createOutputFiles()
-    generate(
-        files, {target: ts.ScriptTarget.ES5, module: ts.ModuleKind.CommonJS});
-    emitHeader(files)
-    emitStructs()
-    emitConsts()
-    emitTypes()
+// Turn a Field into an output string
+// flds is for merging
+function strField(f: Field, noopt?: boolean, flds?: Field[]): string {
+  let ans: string[] = [];
+  let opt = (!noopt && f.optional) ? '*' : ''
+  switch (f.goType.charAt(0)) {
+    case 's':  // string
+    case 'b':  // bool
+    case 'f':  // float64
+    case 'i':  // interface{}
+    case '[':  // []foo
+      opt = ''
   }
+  let stuff = (f.gostuff == undefined) ? '' : ` // ${f.gostuff}`
+  ans.push(genComments(f.goName, getComments(f.me)))
+  if (flds === undefined && f.substruct == undefined) {
+    ans.push(`\t${f.goName} ${opt}${f.goType} ${f.json}${stuff}\n`)
+  }
+  else if (flds !== undefined) {
+    // The logic that got us here is imprecise, so it is possible that
+    // the fields are really all the same, and don't need to be
+    // combined into a struct.
+    let simple = true;
+    for (const ff of flds) {
+      if (ff.substruct !== undefined || byName.get(ff.goType) !== undefined) {
+        simple = false
+        break
+      }
+    }
+    if (simple) {
+      // should check that the ffs are really all the same
+      return strField(flds[0], noopt)
+    }
+    ans.push(`\t${f.goName} ${opt}struct{\n`);
+    for (const ff of flds) {
+      if (ff.substruct !== undefined) {
+        for (const x of ff.substruct) {
+          ans.push(strField(x, noopt))
+        }
+      } else if (byName.get(ff.goType) !== undefined) {
+        const st = byName.get(ff.goType);
+        for (let i = 0; i < st.fields.length; i++) {
+          ans.push(strField(st.fields[i], noopt))
+        }
+      } else {
+        ans.push(strField(ff, noopt));
+      }
+    }
+    ans.push(`\t} ${f.json}${stuff}\n`);
+  }
+  else {
+    ans.push(`\t${f.goName} ${opt}struct {\n`)
+    for (const x of f.substruct) {
+      ans.push(strField(x, noopt))
+    }
+    ans.push(`\t} ${f.json}${stuff}\n`)
+  }
+  return (''.concat(...ans))
+}
 
-  main()
+function emitConsts() {
+  // need the consts too! Generate modifying prefixes and suffixes to ensure
+  // consts are unique. (Go consts are package-level, but Typescript's are
+  // not.) Use suffixes to minimize changes to gopls.
+  let pref = new Map<string, string>([
+    ['DiagnosticSeverity', 'Severity'], ['WatchKind', 'Watch']
+  ])  // typeName->prefix
+  let suff = new Map<string, string>([
+    ['CompletionItemKind', 'Completion'], ['InsertTextFormat', 'TextFormat']
+  ])
+  for (const c of Consts) {
+    if (seenConstTypes.get(c.typeName)) {
+      continue
+    }
+    seenConstTypes.set(c.typeName, true);
+    if (pref.get(c.typeName) == undefined) {
+      pref.set(c.typeName, '')  // initialize to empty value
+    }
+    if (suff.get(c.typeName) == undefined) {
+      suff.set(c.typeName, '')
+    }
+    prgo(`// ${c.typeName} defines constants\n`)
+    prgo(`type ${c.typeName} ${c.goType}\n`)
+  }
+  prgo('const (\n')
+  let seenConsts = new Map<string, boolean>()  // to avoid duplicates
+  for (const c of Consts) {
+    const x = `${pref.get(c.typeName)}${c.name}${suff.get(c.typeName)}`
+    if (seenConsts.get(x)) {
+      continue
+    }
+    seenConsts.set(x, true)
+    if (c.value === undefined) continue;      // didn't figure it out
+    if (x.startsWith('undefined')) continue;  // what's going on here?
+    prgo(genComments(x, getComments(c.me)))
+    prgo(`\t${x} ${c.typeName} = ${c.value}\n`)
+  }
+  prgo(')\n')
+}
+
+function emitHeader(files: string[]) {
+  let lastMod = 0
+  let lastDate: Date
+  for (const f of files) {
+    const st = fs.statSync(f)
+    if (st.mtimeMs > lastMod) {
+      lastMod = st.mtimeMs
+      lastDate = st.mtime
+    }
+  }
+  let a = fs.readFileSync(`${dir}${srcDir}/.git/refs/heads/master`);
+  prgo(`// Package protocol contains data types and code for LSP jsonrpcs\n`)
+  prgo(`// generated automatically from vscode-languageserver-node\n`)
+  prgo(`// commit: ${gitHash}\n`)
+  prgo(`// last fetched ${lastDate}\n`)
+  prgo('package protocol\n\n')
+  prgo(`// Code generated (see typescript/README.md) DO NOT EDIT.\n`);
+};
+
+function git(): string {
+  let a = fs.readFileSync(`${dir}${srcDir}/.git/HEAD`).toString();
+  // ref: refs/heads/foo, or a hash like cc12d1a1c7df935012cdef5d085cdba04a7c8ebe
+  if (a.charAt(a.length - 1) == '\n') {
+    a = a.substring(0, a.length - 1);
+  }
+  if (a.length == 40) {
+    return a // a hash
+  }
+  if (a.substring(0, 5) == 'ref: ') {
+    const fname = `${dir}${srcDir}/.git/` + a.substring(5);
+    let b = fs.readFileSync(fname).toString()
+    if (b.length == 41) {
+      return b.substring(0, 40);
+    }
+  }
+  throw new Error("failed to find the git commit hash")
+}
+
+// ad hoc argument parsing: [-d dir] [-o outputfile], and order matters
+function main() {
+  if (gitHash != git()) {
+    throw new Error(`git hash mismatch, wanted\n${gitHash} but source is at\n${git()}`)
+  }
+  let args = process.argv.slice(2)  // effective command line
+  if (args.length > 0) {
+    let j = 0;
+    if (args[j] == '-d') {
+      dir = args[j + 1]
+      j += 2
+    }
+    if (args[j] == '-o') {
+      outFname = args[j + 1]
+      j += 2
+    }
+    if (j != args.length) throw new Error(`incomprehensible args ${args}`)
+  }
+  let files: string[] = [];
+  for (let i = 0; i < fnames.length; i++) {
+    files.push(`${dir}${fnames[i]}`)
+  }
+  createOutputFiles()
+  generate(
+    files, { target: ts.ScriptTarget.ES5, module: ts.ModuleKind.CommonJS });
+  emitHeader(files)
+  emitStructs()
+  emitConsts()
+  emitTypes()
+}
+
+main()