Bring back SourceFile.EndOfFileToken

CHANGES.md

@@ -6,15 +6,14 @@ CHANGES.md lists intentional changes between the Strada (Typescript) and Corsa (
 
 ## Parser
 
-1. Source files do not contain an EndOfFile token as their last child.
-2. Malformed `...T?` at the end of a tuple now fails with a parse error instead of a grammar error.
-3. Malformed string ImportSpecifiers (`import x as "OOPS" from "y"`) now contain the string's text instead of an empty identifier.
-4. Empty binding elements no longer have a separate kind for OmittedExpression. Instead they have Kind=BindingElement with a nil Initialiser, Name and DotDotDotToken.
-5. ShorthandPropertyAssignment no longer includes an EqualsToken as a child when it has an ObjectAssignmentInitializer.
-6. JSDoc nodes now include leading whitespace in their location.
-7. The parser always parses a JSDocText node for comments in JSDoc. `string` is no longer part of the type of `comment`.
-8. In cases where Strada did produce a JSDocText node, Corsa no longer (incorrectly) includes all leading and trailing whitespace/asterisks, as well as initial `/**`.
-9. JSDocMemberName is now parsed as QualifiedName. These two nodes previously only differed by type, and now QualifiedName has a much less restrictive type for its left child.
+1. Malformed `...T?` at the end of a tuple now fails with a parse error instead of a grammar error.
+2. Malformed string ImportSpecifiers (`import x as "OOPS" from "y"`) now contain the string's text instead of an empty identifier.
+3. Empty binding elements no longer have a separate kind for OmittedExpression. Instead they have Kind=BindingElement with a nil Initialiser, Name and DotDotDotToken.
+4. ShorthandPropertyAssignment no longer includes an EqualsToken as a child when it has an ObjectAssignmentInitializer.
+5. JSDoc nodes now include leading whitespace in their location.
+6. The parser always parses a JSDocText node for comments in JSDoc. `string` is no longer part of the type of `comment`.
+7. In cases where Strada did produce a JSDocText node, Corsa no longer (incorrectly) includes all leading and trailing whitespace/asterisks, as well as initial `/**`.
+8. JSDocMemberName is now parsed as QualifiedName. These two nodes previously only differed by type, and now QualifiedName has a much less restrictive type for its left child.
 
 JSDoc types are parsed in normal type annotation position but show a grammar error. Corsa no longer parses the JSDoc types below, giving a parse error instead of a grammar error.
 

_packages/api/src/node.ts

@@ -20,6 +20,7 @@ declare module "@typescript/ast" {
 const popcount8 = [0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8];
 
 const childProperties: Readonly<Partial<Record<SyntaxKind, readonly string[]>>> = {
+    [SyntaxKind.SourceFile]: ["statements", "endOfFileToken"],
     [SyntaxKind.QualifiedName]: ["left", "right"],
     [SyntaxKind.TypeParameter]: ["modifiers", "name", "constraint", "defaultType"],
     [SyntaxKind.IfStatement]: ["expression", "thenStatement", "elseStatement"],
@@ -221,7 +222,7 @@ export class RemoteNodeBase {
 
     protected get childMask(): number {
         if (this.dataType !== NODE_DATA_TYPE_CHILDREN) {
-            return 0;
+            return -1;
         }
         return this.data & NODE_CHILD_MASK;
     }
@@ -674,6 +675,9 @@ export class RemoteNode extends RemoteNodeBase implements Node {
     get elseStatement(): RemoteNode | undefined {
         return this.getNamedChild("elseStatement") as RemoteNode;
     }
+    get endOfFileToken(): RemoteNode | undefined {
+        return this.getNamedChild("endOfFileToken") as RemoteNode;
+    }
     get equalsGreaterThanToken(): RemoteNode | undefined {
         return this.getNamedChild("equalsGreaterThanToken") as RemoteNode;
     }

_packages/api/test/api.test.ts

@@ -109,7 +109,7 @@ describe("SourceFile", () => {
             nodeCount++;
             node.forEachChild(visit);
         });
-        assert.equal(nodeCount, 7);
+        assert.equal(nodeCount, 8);
     });
 });
 

_packages/ast/src/nodes.ts

@@ -14,6 +14,7 @@ export interface Node extends ReadonlyTextRange {
 export interface SourceFile extends Node {
     readonly kind: SyntaxKind.SourceFile;
     readonly statements: NodeArray<Statement>;
+    readonly endOfFileToken: EndOfFile;
     readonly text: string;
     readonly fileName: string;
 }

internal/ast/ast.go

@@ -10017,10 +10017,11 @@ type SourceFile struct {
 	compositeNodeBase
 
 	// Fields set by NewSourceFile
-	fileName     string // For debugging convenience
-	parseOptions SourceFileParseOptions
-	text         string
-	Statements   *NodeList // NodeList[*Statement]
+	fileName       string // For debugging convenience
+	parseOptions   SourceFileParseOptions
+	text           string
+	Statements     *NodeList  // NodeList[*Statement]
+	EndOfFileToken *TokenNode // TokenNode[*EndOfFileToken]
 
 	// Fields set by parser
 	diagnostics                 []*Diagnostic
@@ -10070,7 +10071,7 @@ type SourceFile struct {
 	declarationMap   map[string][]*Node
 }
 
-func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, statements *NodeList) *Node {
+func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, statements *NodeList, endOfFileToken *TokenNode) *Node {
 	if (tspath.GetEncodedRootLength(opts.FileName) == 0 && !strings.HasPrefix(opts.FileName, "^/")) || opts.FileName != tspath.NormalizePath(opts.FileName) {
 		panic(fmt.Sprintf("fileName should be normalized and absolute: %q", opts.FileName))
 	}
@@ -10079,6 +10080,7 @@ func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, st
 	data.parseOptions = opts
 	data.text = text
 	data.Statements = statements
+	data.EndOfFileToken = endOfFileToken
 	return f.newNode(KindSourceFile, data)
 }
 
@@ -10139,11 +10141,11 @@ func (node *SourceFile) SetBindDiagnostics(diags []*Diagnostic) {
 }
 
 func (node *SourceFile) ForEachChild(v Visitor) bool {
-	return visitNodeList(v, node.Statements)
+	return visitNodeList(v, node.Statements) || visit(v, node.EndOfFileToken)
 }
 
 func (node *SourceFile) VisitEachChild(v *NodeVisitor) *Node {
-	return v.Factory.UpdateSourceFile(node, v.visitTopLevelStatements(node.Statements))
+	return v.Factory.UpdateSourceFile(node, v.visitTopLevelStatements(node.Statements), v.visitToken(node.EndOfFileToken))
 }
 
 func (node *SourceFile) IsJS() bool {
@@ -10171,7 +10173,7 @@ func (node *SourceFile) copyFrom(other *SourceFile) {
 }
 
 func (node *SourceFile) Clone(f NodeFactoryCoercible) *Node {
-	updated := f.AsNodeFactory().NewSourceFile(node.parseOptions, node.text, node.Statements)
+	updated := f.AsNodeFactory().NewSourceFile(node.parseOptions, node.text, node.Statements, node.EndOfFileToken)
 	newFile := updated.AsSourceFile()
 	newFile.copyFrom(node)
 	return cloneNode(updated, node.AsNode(), f.AsNodeFactory().hooks)
@@ -10181,9 +10183,9 @@ func (node *SourceFile) computeSubtreeFacts() SubtreeFacts {
 	return propagateNodeListSubtreeFacts(node.Statements, propagateSubtreeFacts)
 }
 
-func (f *NodeFactory) UpdateSourceFile(node *SourceFile, statements *StatementList) *Node {
-	if statements != node.Statements {
-		updated := f.NewSourceFile(node.parseOptions, node.text, statements).AsSourceFile()
+func (f *NodeFactory) UpdateSourceFile(node *SourceFile, statements *StatementList, endOfFileToken *TokenNode) *Node {
+	if statements != node.Statements || endOfFileToken != node.EndOfFileToken {
+		updated := f.NewSourceFile(node.parseOptions, node.text, statements, endOfFileToken).AsSourceFile()
 		updated.copyFrom(node)
 		return updateNode(updated.AsNode(), node.AsNode(), f.hooks)
 	}

internal/astnav/tokens.go

@@ -49,7 +49,7 @@ func getTokenAtPosition(
 	left := 0
 
 	testNode := func(node *ast.Node) int {
-		if node.End() == position && includePrecedingTokenAtEndPosition != nil {
+		if node.Kind != ast.KindEndOfFile && node.End() == position && includePrecedingTokenAtEndPosition != nil {
 			prevSubtree = node
 		}
 
@@ -247,7 +247,7 @@ func FindPrecedingToken(sourceFile *ast.SourceFile, position int) *ast.Node {
 func FindPrecedingTokenEx(sourceFile *ast.SourceFile, position int, startNode *ast.Node, excludeJSDoc bool) *ast.Node {
 	var find func(node *ast.Node) *ast.Node
 	find = func(n *ast.Node) *ast.Node {
-		if ast.IsNonWhitespaceToken(n) {
+		if ast.IsNonWhitespaceToken(n) && n.Kind != ast.KindEndOfFile {
 			return n
 		}
 

internal/astnav/tokens_test.go

@@ -20,8 +20,6 @@ import (
 )
 
 var testFiles = []string{
-	// !!! EOFToken JSDoc parsing is missing
-	// filepath.Join(repo.TestDataPath, "fixtures/astnav/eofJSDoc.ts"),
 	filepath.Join(repo.TypeScriptSubmodulePath, "src/services/mapCode.ts"),
 }
 

internal/binder/binder.go

@@ -1637,8 +1637,9 @@ func (b *Binder) bindChildren(node *ast.Node) {
 	// case *JSDocImportTag:
 	// 	b.bindJSDocImportTag(node)
 	case ast.KindSourceFile:
-		b.bindEachStatementFunctionsFirst(node.AsSourceFile().Statements)
-		// b.bind(node.endOfFileToken)
+		sourceFile := node.AsSourceFile()
+		b.bindEachStatementFunctionsFirst(sourceFile.Statements)
+		b.bind(sourceFile.EndOfFileToken)
 	case ast.KindBlock:
 		b.bindEachStatementFunctionsFirst(node.AsBlock().Statements)
 	case ast.KindModuleBlock:

internal/fourslash/_scripts/failingTests.txt

@@ -1,4 +1,3 @@
-TestAsOperatorCompletion
 TestAutoImportsWithRootDirsAndRootedPath01
 TestClosedCommentsInConstructor
 TestCodeCompletionEscaping

internal/fourslash/tests/gen/asOperatorCompletion_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestAsOperatorCompletion(t *testing.T) {
 	t.Parallel()
-	t.Skip()
+
 	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
 	const content = `type T = number;
 var x;

internal/ls/completions.go

@@ -2203,81 +2203,79 @@ func shouldIncludeSymbol(
 ) bool {
 	allFlags := symbol.Flags
 	location := data.location
-	if !ast.IsSourceFile(location) {
-		// export = /**/ here we want to get all meanings, so any symbol is ok
-		if ast.IsExportAssignment(location.Parent) {
-			return true
-		}
+	// export = /**/ here we want to get all meanings, so any symbol is ok
+	if location.Parent != nil && ast.IsExportAssignment(location.Parent) {
+		return true
+	}
 
-		// Filter out variables from their own initializers
-		// `const a = /* no 'a' here */`
-		if closestSymbolDeclaration != nil &&
-			ast.IsVariableDeclaration(closestSymbolDeclaration) &&
-			symbol.ValueDeclaration == closestSymbolDeclaration {
-			return false
-		}
+	// Filter out variables from their own initializers
+	// `const a = /* no 'a' here */`
+	if closestSymbolDeclaration != nil &&
+		ast.IsVariableDeclaration(closestSymbolDeclaration) &&
+		symbol.ValueDeclaration == closestSymbolDeclaration {
+		return false
+	}
 
-		// Filter out current and latter parameters from defaults
-		// `function f(a = /* no 'a' and 'b' here */, b) { }` or
-		// `function f<T = /* no 'T' and 'T2' here */>(a: T, b: T2) { }`
-		var symbolDeclaration *ast.Declaration
-		if symbol.ValueDeclaration != nil {
-			symbolDeclaration = symbol.ValueDeclaration
-		} else if len(symbol.Declarations) > 0 {
-			symbolDeclaration = symbol.Declarations[0]
-		}
+	// Filter out current and latter parameters from defaults
+	// `function f(a = /* no 'a' and 'b' here */, b) { }` or
+	// `function f<T = /* no 'T' and 'T2' here */>(a: T, b: T2) { }`
+	var symbolDeclaration *ast.Declaration
+	if symbol.ValueDeclaration != nil {
+		symbolDeclaration = symbol.ValueDeclaration
+	} else if len(symbol.Declarations) > 0 {
+		symbolDeclaration = symbol.Declarations[0]
+	}
 
-		if closestSymbolDeclaration != nil && symbolDeclaration != nil {
-			if ast.IsParameter(closestSymbolDeclaration) && ast.IsParameter(symbolDeclaration) {
-				parameters := closestSymbolDeclaration.Parent.ParameterList()
-				if symbolDeclaration.Pos() >= closestSymbolDeclaration.Pos() &&
-					symbolDeclaration.Pos() < parameters.End() {
-					return false
-				}
-			} else if ast.IsTypeParameterDeclaration(closestSymbolDeclaration) &&
-				ast.IsTypeParameterDeclaration(symbolDeclaration) {
-				if closestSymbolDeclaration == symbolDeclaration && data.contextToken != nil && data.contextToken.Kind == ast.KindExtendsKeyword {
-					// filter out the directly self-recursive type parameters
-					// `type A<K extends /* no 'K' here*/> = K`
+	if closestSymbolDeclaration != nil && symbolDeclaration != nil {
+		if ast.IsParameter(closestSymbolDeclaration) && ast.IsParameter(symbolDeclaration) {
+			parameters := closestSymbolDeclaration.Parent.ParameterList()
+			if symbolDeclaration.Pos() >= closestSymbolDeclaration.Pos() &&
+				symbolDeclaration.Pos() < parameters.End() {
+				return false
+			}
+		} else if ast.IsTypeParameterDeclaration(closestSymbolDeclaration) &&
+			ast.IsTypeParameterDeclaration(symbolDeclaration) {
+			if closestSymbolDeclaration == symbolDeclaration && data.contextToken != nil && data.contextToken.Kind == ast.KindExtendsKeyword {
+				// filter out the directly self-recursive type parameters
+				// `type A<K extends /* no 'K' here*/> = K`
+				return false
+			}
+			if isInTypeParameterDefault(data.contextToken) && !ast.IsInferTypeNode(closestSymbolDeclaration.Parent) {
+				typeParameters := closestSymbolDeclaration.Parent.TypeParameterList()
+				if typeParameters != nil && symbolDeclaration.Pos() >= closestSymbolDeclaration.Pos() &&
+					symbolDeclaration.Pos() < typeParameters.End() {
 					return false
 				}
-				if isInTypeParameterDefault(data.contextToken) && !ast.IsInferTypeNode(closestSymbolDeclaration.Parent) {
-					typeParameters := closestSymbolDeclaration.Parent.TypeParameterList()
-					if typeParameters != nil && symbolDeclaration.Pos() >= closestSymbolDeclaration.Pos() &&
-						symbolDeclaration.Pos() < typeParameters.End() {
-						return false
-					}
-				}
 			}
 		}
+	}
 
-		// External modules can have global export declarations that will be
-		// available as global keywords in all scopes. But if the external module
-		// already has an explicit export and user only wants to use explicit
-		// module imports then the global keywords will be filtered out so auto
-		// import suggestions will win in the completion.
-		symbolOrigin := checker.SkipAlias(symbol, typeChecker)
-		// We only want to filter out the global keywords.
-		// Auto Imports are not available for scripts so this conditional is always false.
-		if file.AsSourceFile().ExternalModuleIndicator != nil &&
-			compilerOptions.AllowUmdGlobalAccess != core.TSTrue &&
-			data.symbolToSortTextMap[ast.GetSymbolId(symbol)] == SortTextGlobalsOrKeywords &&
-			(data.symbolToSortTextMap[ast.GetSymbolId(symbolOrigin)] == SortTextAutoImportSuggestions ||
-				data.symbolToSortTextMap[ast.GetSymbolId(symbolOrigin)] == SortTextLocationPriority) {
-			return false
-		}
+	// External modules can have global export declarations that will be
+	// available as global keywords in all scopes. But if the external module
+	// already has an explicit export and user only wants to use explicit
+	// module imports then the global keywords will be filtered out so auto
+	// import suggestions will win in the completion.
+	symbolOrigin := checker.SkipAlias(symbol, typeChecker)
+	// We only want to filter out the global keywords.
+	// Auto Imports are not available for scripts so this conditional is always false.
+	if file.AsSourceFile().ExternalModuleIndicator != nil &&
+		compilerOptions.AllowUmdGlobalAccess != core.TSTrue &&
+		data.symbolToSortTextMap[ast.GetSymbolId(symbol)] == SortTextGlobalsOrKeywords &&
+		(data.symbolToSortTextMap[ast.GetSymbolId(symbolOrigin)] == SortTextAutoImportSuggestions ||
+			data.symbolToSortTextMap[ast.GetSymbolId(symbolOrigin)] == SortTextLocationPriority) {
+		return false
+	}
 
-		allFlags = allFlags | checker.GetCombinedLocalAndExportSymbolFlags(symbolOrigin)
+	allFlags = allFlags | checker.GetCombinedLocalAndExportSymbolFlags(symbolOrigin)
 
-		// import m = /**/ <-- It can only access namespace (if typing import = x. this would get member symbols and not namespace)
-		if isInRightSideOfInternalImportEqualsDeclaration(data.location) {
-			return allFlags&ast.SymbolFlagsNamespace != 0
-		}
+	// import m = /**/ <-- It can only access namespace (if typing import = x. this would get member symbols and not namespace)
+	if isInRightSideOfInternalImportEqualsDeclaration(data.location) {
+		return allFlags&ast.SymbolFlagsNamespace != 0
+	}
 
-		if data.isTypeOnlyLocation {
-			// It's a type, but you can reach it by namespace.type as well.
-			return symbolCanBeReferencedAtTypeLocation(symbol, typeChecker, collections.Set[ast.SymbolId]{})
-		}
+	if data.isTypeOnlyLocation {
+		// It's a type, but you can reach it by namespace.type as well.
+		return symbolCanBeReferencedAtTypeLocation(symbol, typeChecker, collections.Set[ast.SymbolId]{})
 	}
 
 	// expressions are value space (which includes the value namespaces)
@@ -3677,7 +3675,6 @@ func tryGetObjectTypeDeclarationCompletionContainer(
 			return location.Parent
 		}
 		return nil
-	// !!! we don't include EOF token anymore, verify what we should do in this case.
 	case ast.KindEndOfFile:
 		stmtList := location.Parent.AsSourceFile().Statements
 		if stmtList != nil && len(stmtList.Nodes) > 0 && ast.IsObjectTypeDeclaration(stmtList.Nodes[len(stmtList.Nodes)-1]) {

internal/ls/utilities.go

@@ -382,6 +382,9 @@ func isTypeReference(node *ast.Node) bool {
 }
 
 func isInRightSideOfInternalImportEqualsDeclaration(node *ast.Node) bool {
+	if node.Parent == nil {
+		return false
+	}
 	for node.Parent.Kind == ast.KindQualifiedName {
 		node = node.Parent
 	}