Skip to content

Bring back SourceFile.EndOfFileToken #1257

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Jul 2, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 8 additions & 9 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,14 @@ CHANGES.md lists intentional changes between the Strada (Typescript) and Corsa (

## Parser

1. Source files do not contain an EndOfFile token as their last child.
2. Malformed `...T?` at the end of a tuple now fails with a parse error instead of a grammar error.
3. Malformed string ImportSpecifiers (`import x as "OOPS" from "y"`) now contain the string's text instead of an empty identifier.
4. Empty binding elements no longer have a separate kind for OmittedExpression. Instead they have Kind=BindingElement with a nil Initialiser, Name and DotDotDotToken.
5. ShorthandPropertyAssignment no longer includes an EqualsToken as a child when it has an ObjectAssignmentInitializer.
6. JSDoc nodes now include leading whitespace in their location.
7. The parser always parses a JSDocText node for comments in JSDoc. `string` is no longer part of the type of `comment`.
8. In cases where Strada did produce a JSDocText node, Corsa no longer (incorrectly) includes all leading and trailing whitespace/asterisks, as well as initial `/**`.
9. JSDocMemberName is now parsed as QualifiedName. These two nodes previously only differed by type, and now QualifiedName has a much less restrictive type for its left child.
1. Malformed `...T?` at the end of a tuple now fails with a parse error instead of a grammar error.
2. Malformed string ImportSpecifiers (`import x as "OOPS" from "y"`) now contain the string's text instead of an empty identifier.
3. Empty binding elements no longer have a separate kind for OmittedExpression. Instead they have Kind=BindingElement with a nil Initialiser, Name and DotDotDotToken.
4. ShorthandPropertyAssignment no longer includes an EqualsToken as a child when it has an ObjectAssignmentInitializer.
5. JSDoc nodes now include leading whitespace in their location.
6. The parser always parses a JSDocText node for comments in JSDoc. `string` is no longer part of the type of `comment`.
7. In cases where Strada did produce a JSDocText node, Corsa no longer (incorrectly) includes all leading and trailing whitespace/asterisks, as well as initial `/**`.
8. JSDocMemberName is now parsed as QualifiedName. These two nodes previously only differed by type, and now QualifiedName has a much less restrictive type for its left child.

JSDoc types are parsed in normal type annotation position but show a grammar error. Corsa no longer parses the JSDoc types below, giving a parse error instead of a grammar error.

Expand Down
6 changes: 5 additions & 1 deletion _packages/api/src/node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ declare module "@typescript/ast" {
const popcount8 = [0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8];

const childProperties: Readonly<Partial<Record<SyntaxKind, readonly string[]>>> = {
[SyntaxKind.SourceFile]: ["statements", "endOfFileToken"],
[SyntaxKind.QualifiedName]: ["left", "right"],
[SyntaxKind.TypeParameter]: ["modifiers", "name", "constraint", "defaultType"],
[SyntaxKind.IfStatement]: ["expression", "thenStatement", "elseStatement"],
Expand Down Expand Up @@ -221,7 +222,7 @@ export class RemoteNodeBase {

protected get childMask(): number {
if (this.dataType !== NODE_DATA_TYPE_CHILDREN) {
return 0;
return -1;
}
return this.data & NODE_CHILD_MASK;
}
Expand Down Expand Up @@ -674,6 +675,9 @@ export class RemoteNode extends RemoteNodeBase implements Node {
get elseStatement(): RemoteNode | undefined {
return this.getNamedChild("elseStatement") as RemoteNode;
}
get endOfFileToken(): RemoteNode | undefined {
return this.getNamedChild("endOfFileToken") as RemoteNode;
}
get equalsGreaterThanToken(): RemoteNode | undefined {
return this.getNamedChild("equalsGreaterThanToken") as RemoteNode;
}
Expand Down
2 changes: 1 addition & 1 deletion _packages/api/test/api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ describe("SourceFile", () => {
nodeCount++;
node.forEachChild(visit);
});
assert.equal(nodeCount, 7);
assert.equal(nodeCount, 8);
});
});

Expand Down
1 change: 1 addition & 0 deletions _packages/ast/src/nodes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ export interface Node extends ReadonlyTextRange {
export interface SourceFile extends Node {
readonly kind: SyntaxKind.SourceFile;
readonly statements: NodeArray<Statement>;
readonly endOfFileToken: EndOfFile;
readonly text: string;
readonly fileName: string;
}
Expand Down
24 changes: 13 additions & 11 deletions internal/ast/ast.go
Original file line number Diff line number Diff line change
Expand Up @@ -10017,10 +10017,11 @@ type SourceFile struct {
compositeNodeBase

// Fields set by NewSourceFile
fileName string // For debugging convenience
parseOptions SourceFileParseOptions
text string
Statements *NodeList // NodeList[*Statement]
fileName string // For debugging convenience
parseOptions SourceFileParseOptions
text string
Statements *NodeList // NodeList[*Statement]
EndOfFileToken *TokenNode // TokenNode[*EndOfFileToken]

// Fields set by parser
diagnostics []*Diagnostic
Expand Down Expand Up @@ -10070,7 +10071,7 @@ type SourceFile struct {
declarationMap map[string][]*Node
}

func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, statements *NodeList) *Node {
func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, statements *NodeList, endOfFileToken *TokenNode) *Node {
if (tspath.GetEncodedRootLength(opts.FileName) == 0 && !strings.HasPrefix(opts.FileName, "^/")) || opts.FileName != tspath.NormalizePath(opts.FileName) {
panic(fmt.Sprintf("fileName should be normalized and absolute: %q", opts.FileName))
}
Expand All @@ -10079,6 +10080,7 @@ func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, st
data.parseOptions = opts
data.text = text
data.Statements = statements
data.EndOfFileToken = endOfFileToken
return f.newNode(KindSourceFile, data)
}

Expand Down Expand Up @@ -10139,11 +10141,11 @@ func (node *SourceFile) SetBindDiagnostics(diags []*Diagnostic) {
}

func (node *SourceFile) ForEachChild(v Visitor) bool {
return visitNodeList(v, node.Statements)
return visitNodeList(v, node.Statements) || visit(v, node.EndOfFileToken)
}

func (node *SourceFile) VisitEachChild(v *NodeVisitor) *Node {
return v.Factory.UpdateSourceFile(node, v.visitTopLevelStatements(node.Statements))
return v.Factory.UpdateSourceFile(node, v.visitTopLevelStatements(node.Statements), v.visitToken(node.EndOfFileToken))
}

func (node *SourceFile) IsJS() bool {
Expand Down Expand Up @@ -10171,7 +10173,7 @@ func (node *SourceFile) copyFrom(other *SourceFile) {
}

func (node *SourceFile) Clone(f NodeFactoryCoercible) *Node {
updated := f.AsNodeFactory().NewSourceFile(node.parseOptions, node.text, node.Statements)
updated := f.AsNodeFactory().NewSourceFile(node.parseOptions, node.text, node.Statements, node.EndOfFileToken)
newFile := updated.AsSourceFile()
newFile.copyFrom(node)
return cloneNode(updated, node.AsNode(), f.AsNodeFactory().hooks)
Expand All @@ -10181,9 +10183,9 @@ func (node *SourceFile) computeSubtreeFacts() SubtreeFacts {
return propagateNodeListSubtreeFacts(node.Statements, propagateSubtreeFacts)
}

func (f *NodeFactory) UpdateSourceFile(node *SourceFile, statements *StatementList) *Node {
if statements != node.Statements {
updated := f.NewSourceFile(node.parseOptions, node.text, statements).AsSourceFile()
func (f *NodeFactory) UpdateSourceFile(node *SourceFile, statements *StatementList, endOfFileToken *TokenNode) *Node {
if statements != node.Statements || endOfFileToken != node.EndOfFileToken {
updated := f.NewSourceFile(node.parseOptions, node.text, statements, endOfFileToken).AsSourceFile()
updated.copyFrom(node)
return updateNode(updated.AsNode(), node.AsNode(), f.hooks)
}
Expand Down
4 changes: 2 additions & 2 deletions internal/astnav/tokens.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func getTokenAtPosition(
left := 0

testNode := func(node *ast.Node) int {
if node.End() == position && includePrecedingTokenAtEndPosition != nil {
if node.Kind != ast.KindEndOfFile && node.End() == position && includePrecedingTokenAtEndPosition != nil {
prevSubtree = node
}

Expand Down Expand Up @@ -247,7 +247,7 @@ func FindPrecedingToken(sourceFile *ast.SourceFile, position int) *ast.Node {
func FindPrecedingTokenEx(sourceFile *ast.SourceFile, position int, startNode *ast.Node, excludeJSDoc bool) *ast.Node {
var find func(node *ast.Node) *ast.Node
find = func(n *ast.Node) *ast.Node {
if ast.IsNonWhitespaceToken(n) {
if ast.IsNonWhitespaceToken(n) && n.Kind != ast.KindEndOfFile {
return n
}

Expand Down
2 changes: 0 additions & 2 deletions internal/astnav/tokens_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ import (
)

var testFiles = []string{
// !!! EOFToken JSDoc parsing is missing
// filepath.Join(repo.TestDataPath, "fixtures/astnav/eofJSDoc.ts"),
filepath.Join(repo.TypeScriptSubmodulePath, "src/services/mapCode.ts"),
}

Expand Down
5 changes: 3 additions & 2 deletions internal/binder/binder.go
Original file line number Diff line number Diff line change
Expand Up @@ -1637,8 +1637,9 @@ func (b *Binder) bindChildren(node *ast.Node) {
// case *JSDocImportTag:
// b.bindJSDocImportTag(node)
case ast.KindSourceFile:
b.bindEachStatementFunctionsFirst(node.AsSourceFile().Statements)
// b.bind(node.endOfFileToken)
sourceFile := node.AsSourceFile()
b.bindEachStatementFunctionsFirst(sourceFile.Statements)
b.bind(sourceFile.EndOfFileToken)
case ast.KindBlock:
b.bindEachStatementFunctionsFirst(node.AsBlock().Statements)
case ast.KindModuleBlock:
Expand Down
1 change: 0 additions & 1 deletion internal/fourslash/_scripts/failingTests.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
TestAsOperatorCompletion
TestAutoImportsWithRootDirsAndRootedPath01
TestClosedCommentsInConstructor
TestCodeCompletionEscaping
Expand Down
2 changes: 1 addition & 1 deletion internal/fourslash/tests/gen/asOperatorCompletion_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

func TestAsOperatorCompletion(t *testing.T) {
t.Parallel()
t.Skip()

defer testutil.RecoverAndFail(t, "Panic on fourslash test")
const content = `type T = number;
var x;
Expand Down
127 changes: 62 additions & 65 deletions internal/ls/completions.go
Original file line number Diff line number Diff line change
Expand Up @@ -2203,81 +2203,79 @@ func shouldIncludeSymbol(
) bool {
allFlags := symbol.Flags
location := data.location
if !ast.IsSourceFile(location) {
// export = /**/ here we want to get all meanings, so any symbol is ok
if ast.IsExportAssignment(location.Parent) {
return true
}
// export = /**/ here we want to get all meanings, so any symbol is ok
if location.Parent != nil && ast.IsExportAssignment(location.Parent) {
return true
}

// Filter out variables from their own initializers
// `const a = /* no 'a' here */`
if closestSymbolDeclaration != nil &&
ast.IsVariableDeclaration(closestSymbolDeclaration) &&
symbol.ValueDeclaration == closestSymbolDeclaration {
return false
}
// Filter out variables from their own initializers
// `const a = /* no 'a' here */`
if closestSymbolDeclaration != nil &&
ast.IsVariableDeclaration(closestSymbolDeclaration) &&
symbol.ValueDeclaration == closestSymbolDeclaration {
return false
}

// Filter out current and latter parameters from defaults
// `function f(a = /* no 'a' and 'b' here */, b) { }` or
// `function f<T = /* no 'T' and 'T2' here */>(a: T, b: T2) { }`
var symbolDeclaration *ast.Declaration
if symbol.ValueDeclaration != nil {
symbolDeclaration = symbol.ValueDeclaration
} else if len(symbol.Declarations) > 0 {
symbolDeclaration = symbol.Declarations[0]
}
// Filter out current and latter parameters from defaults
// `function f(a = /* no 'a' and 'b' here */, b) { }` or
// `function f<T = /* no 'T' and 'T2' here */>(a: T, b: T2) { }`
var symbolDeclaration *ast.Declaration
if symbol.ValueDeclaration != nil {
symbolDeclaration = symbol.ValueDeclaration
} else if len(symbol.Declarations) > 0 {
symbolDeclaration = symbol.Declarations[0]
}

if closestSymbolDeclaration != nil && symbolDeclaration != nil {
if ast.IsParameter(closestSymbolDeclaration) && ast.IsParameter(symbolDeclaration) {
parameters := closestSymbolDeclaration.Parent.ParameterList()
if symbolDeclaration.Pos() >= closestSymbolDeclaration.Pos() &&
symbolDeclaration.Pos() < parameters.End() {
return false
}
} else if ast.IsTypeParameterDeclaration(closestSymbolDeclaration) &&
ast.IsTypeParameterDeclaration(symbolDeclaration) {
if closestSymbolDeclaration == symbolDeclaration && data.contextToken != nil && data.contextToken.Kind == ast.KindExtendsKeyword {
// filter out the directly self-recursive type parameters
// `type A<K extends /* no 'K' here*/> = K`
if closestSymbolDeclaration != nil && symbolDeclaration != nil {
if ast.IsParameter(closestSymbolDeclaration) && ast.IsParameter(symbolDeclaration) {
parameters := closestSymbolDeclaration.Parent.ParameterList()
if symbolDeclaration.Pos() >= closestSymbolDeclaration.Pos() &&
symbolDeclaration.Pos() < parameters.End() {
return false
}
} else if ast.IsTypeParameterDeclaration(closestSymbolDeclaration) &&
ast.IsTypeParameterDeclaration(symbolDeclaration) {
if closestSymbolDeclaration == symbolDeclaration && data.contextToken != nil && data.contextToken.Kind == ast.KindExtendsKeyword {
// filter out the directly self-recursive type parameters
// `type A<K extends /* no 'K' here*/> = K`
return false
}
if isInTypeParameterDefault(data.contextToken) && !ast.IsInferTypeNode(closestSymbolDeclaration.Parent) {
typeParameters := closestSymbolDeclaration.Parent.TypeParameterList()
if typeParameters != nil && symbolDeclaration.Pos() >= closestSymbolDeclaration.Pos() &&
symbolDeclaration.Pos() < typeParameters.End() {
return false
}
if isInTypeParameterDefault(data.contextToken) && !ast.IsInferTypeNode(closestSymbolDeclaration.Parent) {
typeParameters := closestSymbolDeclaration.Parent.TypeParameterList()
if typeParameters != nil && symbolDeclaration.Pos() >= closestSymbolDeclaration.Pos() &&
symbolDeclaration.Pos() < typeParameters.End() {
return false
}
}
}
}
}

// External modules can have global export declarations that will be
// available as global keywords in all scopes. But if the external module
// already has an explicit export and user only wants to use explicit
// module imports then the global keywords will be filtered out so auto
// import suggestions will win in the completion.
symbolOrigin := checker.SkipAlias(symbol, typeChecker)
// We only want to filter out the global keywords.
// Auto Imports are not available for scripts so this conditional is always false.
if file.AsSourceFile().ExternalModuleIndicator != nil &&
compilerOptions.AllowUmdGlobalAccess != core.TSTrue &&
data.symbolToSortTextMap[ast.GetSymbolId(symbol)] == SortTextGlobalsOrKeywords &&
(data.symbolToSortTextMap[ast.GetSymbolId(symbolOrigin)] == SortTextAutoImportSuggestions ||
data.symbolToSortTextMap[ast.GetSymbolId(symbolOrigin)] == SortTextLocationPriority) {
return false
}
// External modules can have global export declarations that will be
// available as global keywords in all scopes. But if the external module
// already has an explicit export and user only wants to use explicit
// module imports then the global keywords will be filtered out so auto
// import suggestions will win in the completion.
symbolOrigin := checker.SkipAlias(symbol, typeChecker)
// We only want to filter out the global keywords.
// Auto Imports are not available for scripts so this conditional is always false.
if file.AsSourceFile().ExternalModuleIndicator != nil &&
compilerOptions.AllowUmdGlobalAccess != core.TSTrue &&
data.symbolToSortTextMap[ast.GetSymbolId(symbol)] == SortTextGlobalsOrKeywords &&
(data.symbolToSortTextMap[ast.GetSymbolId(symbolOrigin)] == SortTextAutoImportSuggestions ||
data.symbolToSortTextMap[ast.GetSymbolId(symbolOrigin)] == SortTextLocationPriority) {
return false
}

allFlags = allFlags | checker.GetCombinedLocalAndExportSymbolFlags(symbolOrigin)
allFlags = allFlags | checker.GetCombinedLocalAndExportSymbolFlags(symbolOrigin)

// import m = /**/ <-- It can only access namespace (if typing import = x. this would get member symbols and not namespace)
if isInRightSideOfInternalImportEqualsDeclaration(data.location) {
return allFlags&ast.SymbolFlagsNamespace != 0
}
// import m = /**/ <-- It can only access namespace (if typing import = x. this would get member symbols and not namespace)
if isInRightSideOfInternalImportEqualsDeclaration(data.location) {
return allFlags&ast.SymbolFlagsNamespace != 0
}

if data.isTypeOnlyLocation {
// It's a type, but you can reach it by namespace.type as well.
return symbolCanBeReferencedAtTypeLocation(symbol, typeChecker, collections.Set[ast.SymbolId]{})
}
if data.isTypeOnlyLocation {
// It's a type, but you can reach it by namespace.type as well.
return symbolCanBeReferencedAtTypeLocation(symbol, typeChecker, collections.Set[ast.SymbolId]{})
}

// expressions are value space (which includes the value namespaces)
Expand Down Expand Up @@ -3677,7 +3675,6 @@ func tryGetObjectTypeDeclarationCompletionContainer(
return location.Parent
}
return nil
// !!! we don't include EOF token anymore, verify what we should do in this case.
case ast.KindEndOfFile:
stmtList := location.Parent.AsSourceFile().Statements
if stmtList != nil && len(stmtList.Nodes) > 0 && ast.IsObjectTypeDeclaration(stmtList.Nodes[len(stmtList.Nodes)-1]) {
Expand Down
3 changes: 3 additions & 0 deletions internal/ls/utilities.go
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,9 @@ func isTypeReference(node *ast.Node) bool {
}

func isInRightSideOfInternalImportEqualsDeclaration(node *ast.Node) bool {
if node.Parent == nil {
return false
}
for node.Parent.Kind == ast.KindQualifiedName {
node = node.Parent
}
Expand Down
Loading