From f96b8d65d895bf4567dec64b01a8229789d873e7 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Sat, 18 Nov 2023 16:05:36 -0500 Subject: [PATCH 01/10] Add Lua bindings for building stack graphs We're using the `mlua` crate, since it seems most actively maintained and performant. --- .github/workflows/ci.yml | 2 +- .github/workflows/publish-lsp-positions.yml | 2 +- .github/workflows/publish-stack-graphs.yml | 2 +- .../publish-tree-sitter-stack-graphs-java.yml | 2 +- ...sh-tree-sitter-stack-graphs-javascript.yml | 2 +- ...sh-tree-sitter-stack-graphs-typescript.yml | 2 +- .../publish-tree-sitter-stack-graphs.yml | 2 +- lsp-positions/Cargo.toml | 2 + lsp-positions/src/lib.rs | 3 + lsp-positions/src/lua.rs | 144 +++++ stack-graphs/CHANGELOG.md | 6 + stack-graphs/Cargo.toml | 5 + stack-graphs/README.md | 25 + stack-graphs/src/lib.rs | 2 + stack-graphs/src/lua.rs | 544 ++++++++++++++++++ stack-graphs/tests/it/lua.rs | 295 ++++++++++ stack-graphs/tests/it/main.rs | 2 + 17 files changed, 1035 insertions(+), 7 deletions(-) create mode 100644 lsp-positions/src/lua.rs create mode 100644 stack-graphs/src/lua.rs create mode 100644 stack-graphs/tests/it/lua.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c561bd429..a0baa0917 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Run lsp-positions tests without tree-sitter run: cargo test -p lsp-positions --no-default-features - name: Run test suite with all features enabled - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored - name: Run test suite with all optimizations run: cargo test --release # Do the new project test last, because it adds the crate in the current source diff --git a/.github/workflows/publish-lsp-positions.yml b/.github/workflows/publish-lsp-positions.yml index 210662a00..6fd1b213a 100644 --- a/.github/workflows/publish-lsp-positions.yml +++ b/.github/workflows/publish-lsp-positions.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/.github/workflows/publish-stack-graphs.yml b/.github/workflows/publish-stack-graphs.yml index 641a93488..a2689033d 100644 --- a/.github/workflows/publish-stack-graphs.yml +++ b/.github/workflows/publish-stack-graphs.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-java.yml b/.github/workflows/publish-tree-sitter-stack-graphs-java.yml index 6d7afb769..c1d904942 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-java.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-java.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml b/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml index ff6826770..a9ab7b448 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml b/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml index a3409f397..5446e2c5a 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/.github/workflows/publish-tree-sitter-stack-graphs.yml b/.github/workflows/publish-tree-sitter-stack-graphs.yml index e9395ab1f..7b7db7e0f 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/lsp-positions/Cargo.toml b/lsp-positions/Cargo.toml index 242a894ff..f3bd926b9 100644 --- a/lsp-positions/Cargo.toml +++ b/lsp-positions/Cargo.toml @@ -18,10 +18,12 @@ test = false [features] bincode = ["dep:bincode"] +lua = ["dep:mlua"] tree-sitter = ["dep:tree-sitter"] [dependencies] memchr = "2.4" +mlua = { version = "0.9", optional = true } tree-sitter = { version=">= 0.19", optional=true } unicode-segmentation = { version="1.8" } serde = { version="1", optional=true, features=["derive"] } diff --git a/lsp-positions/src/lib.rs b/lsp-positions/src/lib.rs index 628cec2a7..91dadd00a 100644 --- a/lsp-positions/src/lib.rs +++ b/lsp-positions/src/lib.rs @@ -33,6 +33,9 @@ use memchr::memchr; use unicode_segmentation::UnicodeSegmentation as _; +#[cfg(feature = "lua")] +mod lua; + fn grapheme_len(string: &str) -> usize { string.graphemes(true).count() } diff --git a/lsp-positions/src/lua.rs b/lsp-positions/src/lua.rs new file mode 100644 index 000000000..047814280 --- /dev/null +++ b/lsp-positions/src/lua.rs @@ -0,0 +1,144 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use std::ops::Range; + +use mlua::Error; +use mlua::FromLua; +use mlua::IntoLua; +use mlua::Lua; +use mlua::Value; + +use crate::Offset; +use crate::Position; +use crate::Span; + +impl<'lua> FromLua<'lua> for Offset { + fn from_lua(value: Value<'lua>, _: &'lua Lua) -> Result { + let table = match value { + Value::Table(table) => table, + Value::Nil => return Ok(Offset::default()), + _ => { + return Err(mlua::Error::FromLuaConversionError { + from: value.type_name(), + to: "Offset", + message: None, + }) + } + }; + let utf8_offset = table.get::<_, Option<_>>("utf8_offset")?.unwrap_or(0); + let utf16_offset = table.get::<_, Option<_>>("utf16_offset")?.unwrap_or(0); + let grapheme_offset = table.get::<_, Option<_>>("grapheme_offset")?.unwrap_or(0); + Ok(Offset { + utf8_offset, + utf16_offset, + grapheme_offset, + }) + } +} + +impl<'lua> IntoLua<'lua> for Offset { + fn into_lua(self, l: &'lua Lua) -> Result, Error> { + let result = l.create_table()?; + result.set("utf8_offset", self.utf8_offset)?; + result.set("utf16_offset", self.utf16_offset)?; + result.set("grapheme_offset", self.grapheme_offset)?; + Ok(Value::Table(result)) + } +} + +fn range_from_lua<'lua>(value: Value<'lua>) -> Result, Error> { + let table = match value { + Value::Table(table) => table, + Value::Nil => return Ok(0..0), + _ => { + return Err(mlua::Error::FromLuaConversionError { + from: value.type_name(), + to: "Range", + message: None, + }) + } + }; + let start = table.get("start")?; + let end = table.get("end")?; + Ok(start..end) +} + +fn range_into_lua<'lua>(range: Range, l: &'lua Lua) -> Result, Error> { + let result = l.create_table()?; + result.set("start", range.start)?; + result.set("end", range.end)?; + Ok(Value::Table(result)) +} + +impl<'lua> FromLua<'lua> for Position { + fn from_lua(value: Value<'lua>, _: &'lua Lua) -> Result { + let table = match value { + Value::Table(table) => table, + Value::Nil => return Ok(Position::default()), + _ => { + return Err(mlua::Error::FromLuaConversionError { + from: value.type_name(), + to: "Position", + message: None, + }) + } + }; + let line = table.get("line")?; + let column = table.get("column")?; + let containing_line = range_from_lua(table.get("containing_line")?)?; + let trimmed_line = range_from_lua(table.get("trimmed_line")?)?; + Ok(Position { + line, + column, + containing_line, + trimmed_line, + }) + } +} + +impl<'lua> IntoLua<'lua> for Position { + fn into_lua(self, l: &'lua Lua) -> Result, Error> { + let result = l.create_table()?; + result.set("line", self.line)?; + result.set("column", self.column)?; + result.set("containing_line", range_into_lua(self.containing_line, l)?)?; + result.set("trimmed_line", range_into_lua(self.trimmed_line, l)?)?; + Ok(Value::Table(result)) + } +} + +impl<'lua> FromLua<'lua> for Span { + fn from_lua(value: Value<'lua>, _: &'lua Lua) -> Result { + let table = match value { + Value::Table(table) => table, + Value::Nil => return Ok(Span::default()), + _ => { + return Err(mlua::Error::FromLuaConversionError { + from: value.type_name(), + to: "Span", + message: None, + }) + } + }; + let start = table.get("start")?; + let end = table.get("end")?; + Ok(Span { start, end }) + } +} + +impl<'lua> IntoLua<'lua> for Span { + fn into_lua(self, l: &'lua Lua) -> Result, Error> { + if self == Span::default() { + return Ok(Value::Nil); + } + let result = l.create_table()?; + result.set("start", self.start)?; + result.set("end", self.end)?; + Ok(Value::Table(result)) + } +} diff --git a/stack-graphs/CHANGELOG.md b/stack-graphs/CHANGELOG.md index c61221e8e..0c15532a2 100644 --- a/stack-graphs/CHANGELOG.md +++ b/stack-graphs/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.13.0 -- Unreleased + +### Added + +- Added Lua bindings for constructing stack graphs. These bindings are optional, and will only be built when the `lua` feature flag is enabled. + ## v0.12.0 -- 2023-07-27 ### Added diff --git a/stack-graphs/Cargo.toml b/stack-graphs/Cargo.toml index 2e55e1768..c52131288 100644 --- a/stack-graphs/Cargo.toml +++ b/stack-graphs/Cargo.toml @@ -15,6 +15,7 @@ edition = "2018" [features] bincode = ["dep:bincode", "lsp-positions/bincode"] copious-debugging = [] +lua = ["dep:mlua", "lsp-positions/lua"] serde = ["dep:serde", "serde_with", "lsp-positions/serde"] storage = ["bincode", "rusqlite"] visualization = ["serde", "serde_json"] @@ -33,6 +34,7 @@ fxhash = "0.2" itertools = "0.10" libc = "0.2" lsp-positions = { version = "0.3", path = "../lsp-positions" } +mlua = { version = "0.9", optional = true } rusqlite = { version = "0.28", optional = true, features = ["bundled", "functions"] } serde = { version = "1.0", optional = true, features = ["derive"] } serde_json = { version = "1.0", optional = true } @@ -41,6 +43,7 @@ smallvec = { version = "1.6", features = ["union"] } thiserror = { version = "1.0" } [dev-dependencies] +anyhow = "1.0" assert-json-diff = "2" itertools = "0.10" maplit = "1.0" @@ -49,3 +52,5 @@ serde_json = { version = "1.0" } [package.metadata.docs.rs] all-features = true +features = ["mlua/lua54", "mlua/vendored"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/stack-graphs/README.md b/stack-graphs/README.md index 7805d19ef..9fed588ba 100644 --- a/stack-graphs/README.md +++ b/stack-graphs/README.md @@ -17,6 +17,31 @@ how to use this library. Notable changes for each version are documented in the [release notes](https://github.com/github/stack-graphs/blob/main/stack-graphs/CHANGELOG.md). +## Lua bindings + +This crate includes optional Lua bindings, allowing you to construct stack +graphs using Lua code. Lua support is only enabled if you compile with the `lua` +feature. This feature is not enough on its own, because the `mlua` crate +supports multiple Lua versions, and can either link against a system-installed +copy of Lua, or build its own copy from vendored Lua source. These choices are +all controlled via additional features on the `mlua` crate. + +When building and testing this crate, make sure to provide all necessary +features on the command line: + +``` console +$ cargo test --features lua,mlua/lua54,mlua/vendored +``` + +When building a crate that depends on this crate, add a dependency on `mlua` so +that you can set its feature flags: + +``` toml +[dependencies] +stack-graphs = { version="0.13", features=["lua"] } +mlua = { version="0.9", features=["lua54", "vendored"] } +``` + ## Credits Stack graphs are heavily based on the [_scope graphs_][scope graphs] framework diff --git a/stack-graphs/src/lib.rs b/stack-graphs/src/lib.rs index 009076af9..edcae832c 100644 --- a/stack-graphs/src/lib.rs +++ b/stack-graphs/src/lib.rs @@ -66,6 +66,8 @@ pub mod cycles; #[macro_use] mod debugging; pub mod graph; +#[cfg(feature = "lua")] +pub mod lua; pub mod partial; pub mod paths; pub mod serde; diff --git a/stack-graphs/src/lua.rs b/stack-graphs/src/lua.rs new file mode 100644 index 000000000..8831ea242 --- /dev/null +++ b/stack-graphs/src/lua.rs @@ -0,0 +1,544 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +#![cfg_attr(docsrs, doc(cfg(feature = "lua")))] +//! Provides access to `StackGraph` instances from Lua. +//! +//! With the `lua` feature enabled, you can add [`StackGraph`] instances to a [`Lua`][mlua::Lua] +//! interpreter. You might typically use this to _create_ stack graphs from Lua, by calling a Lua +//! function with an empty stack graph as a parameter. Note that you'll almost certainly need to +//! use `mlua`'s [scoped values](mlua::Lua::scope) mechanism so that you can still use the +//! [`StackGraph`] on the Rust side once the Lua function has finished. +//! +//! ``` +//! # use mlua::Lua; +//! # use stack_graphs::graph::StackGraph; +//! # fn main() -> Result<(), mlua::Error> { +//! let lua = Lua::new(); +//! let chunk = r#" +//! function process_graph(graph) +//! local file = graph:file("test.py") +//! local def = file:definition_node("foo") +//! def:add_edge_from(graph:root_node()) +//! end +//! "#; +//! lua.load(chunk).set_name("stack graph chunk").exec()?; +//! let process_graph: mlua::Function = lua.globals().get("process_graph")?; +//! +//! let mut graph = StackGraph::new(); +//! lua.scope(|scope| { +//! let graph = scope.create_userdata_ref_mut(&mut graph); +//! process_graph.call(graph) +//! })?; +//! assert_eq!(graph.iter_nodes().count(), 3); +//! # Ok(()) +//! # } +//! ``` +//! +//! ## Building +//! +//! Lua support is only enabled if you compile with the `lua` feature. This feature is not enough +//! on its own, because the `mlua` crate supports multiple Lua versions, and can either link +//! against a system-installed copy of Lua, or build its own copy from vendored Lua source. These +//! choices are all controlled via additional features on the `mlua` crate. +//! +//! When building and testing this crate, make sure to provide all necessary features on the +//! command line: +//! +//! ``` console +//! $ cargo test --features lua,mlua/lua54,mlua/vendored +//! ``` +//! +//! When building a crate that depends on this crate, add a dependency on `mlua` so that you can +//! set its feature flags: +//! +//! ``` toml +//! [dependencies] +//! stack-graphs = { version="0.13", features=["lua"] } +//! mlua = { version="0.9", features=["lua54", "vendored"] } +//! ``` + +// Implementation notes: Stack graphs, files, and nodes can live inside the Lua interpreter as +// objects. They are each wrapped in a userdata, with a metatable defining the methods that are +// available. With mlua, the UserData trait is the way to define these metatables and methods. +// +// Complicating matters is that files and nodes need to be represented by a _pair_ of Lua values: +// the handle of the file or node, and a reference to the StackGraph that the file or node lives +// in. We need both because some of the methods need to dereference the handle to get e.g. the +// `Node` instance. It's not safe to dereference the handle when we create the userdata, because +// the resulting pointer is not guaranteed to be stable. (If you add another node, the arena's +// storage might get resized, moving the node instances around in memory.) +// +// To handle this, we leverage Lua's ability to associate “user values” with each userdata. For +// files and nodes, we store the graph's userdata (i.e. its Lua representation) as the user value +// of each file and node userdata. +// +// That, in turn, means that we must use `add_function` to define each metatable method, since that +// gives us an `mlua::AnyUserData`, which lets us access the userdata's underlying Rust value _and_ +// its user value. (Typically, you would use the more ergonomic `add_method` or `add_method_mut`, +// which take care of unwrapping the userdata and giving you a &ref or &mut ref to the underlying +// Rust type. But then you don't have access to the userdata's user value.) + +use std::fmt::Write; +use std::num::NonZeroU32; + +use controlled_option::ControlledOption; +use lsp_positions::Span; +use mlua::AnyUserData; +use mlua::UserData; +use mlua::UserDataMethods; + +use crate::arena::Handle; +use crate::graph::File; +use crate::graph::Node; +use crate::graph::StackGraph; + +impl UserData for StackGraph { + fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_function("file", |l, (graph_ud, name): (AnyUserData, String)| { + let file = { + let mut graph = graph_ud.borrow_mut::()?; + graph.get_or_create_file(&name) + }; + let file_ud = l.create_userdata(file)?; + file_ud.set_user_value(graph_ud)?; + Ok(file_ud) + }); + + methods.add_function("jump_to_node", |l, graph_ud: AnyUserData| { + let node = StackGraph::jump_to_node(); + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + + methods.add_function("nodes", |l, graph_ud: AnyUserData| { + let iter = l.create_function( + |l, (graph_ud, prev_node_ud): (AnyUserData, Option)| { + let prev_index = match prev_node_ud { + Some(prev_node_ud) => { + let prev_node = prev_node_ud.borrow::>()?; + prev_node.as_u32() + } + None => 0, + }; + let node_index = { + let graph = graph_ud.borrow::()?; + let node_count = graph.nodes.len() as u32; + if prev_index == node_count - 1 { + return Ok(None); + } + unsafe { NonZeroU32::new_unchecked(prev_index + 1) } + }; + let node = Handle::new(node_index); + let node_ud = l.create_userdata::>(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(Some(node_ud)) + }, + )?; + Ok((iter, graph_ud, None::)) + }); + + methods.add_function("root_node", |l, graph_ud: AnyUserData| { + let node = StackGraph::root_node(); + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + } +} + +impl UserData for Handle { + fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_function( + "definition_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_pop_symbol_node(node_id, symbol, true) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function("drop_scopes_node", |l, file_ud: AnyUserData| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let node_id = graph.new_node_id(file); + graph + .add_drop_scopes_node(node_id) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + + methods.add_function("exported_scope_node", |l, file_ud: AnyUserData| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let node_id = graph.new_node_id(file); + graph + .add_scope_node(node_id, true) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + + methods.add_function("internal_scope_node", |l, file_ud: AnyUserData| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let node_id = graph.new_node_id(file); + graph + .add_scope_node(node_id, false) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + + methods.add_function( + "pop_scoped_symbol_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_pop_scoped_symbol_node(node_id, symbol, false) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "pop_symbol_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_pop_symbol_node(node_id, symbol, false) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "push_scoped_symbol_node", + |l, (file_ud, symbol, scope_ud): (AnyUserData, String, AnyUserData)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let scope = *scope_ud.borrow::>()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let scope_id = { + let scope = &graph[scope]; + if !scope.is_exported_scope() { + return Err(mlua::Error::RuntimeError( + "Can only push exported scope nodes".to_string(), + )); + } + scope.id() + }; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_push_scoped_symbol_node(node_id, symbol, scope_id, false) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "push_symbol_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_push_symbol_node(node_id, symbol, false) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "reference_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_push_symbol_node(node_id, symbol, true) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "scoped_definition_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_pop_scoped_symbol_node(node_id, symbol, true) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "scoped_reference_node", + |l, (file_ud, symbol, scope_ud): (AnyUserData, String, AnyUserData)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let scope = *scope_ud.borrow::>()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let scope_id = { + let scope = &graph[scope]; + if !scope.is_exported_scope() { + return Err(mlua::Error::RuntimeError( + "Can only push exported scope nodes".to_string(), + )); + } + scope.id() + }; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_push_scoped_symbol_node(node_id, symbol, scope_id, true) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + } +} + +impl UserData for Handle { + fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_function( + "add_edge_from", + |_, (this_ud, from_ud, precedence): (AnyUserData, AnyUserData, Option)| { + let this = *this_ud.borrow::>()?; + let from = *from_ud.borrow::>()?; + let graph_ud = this_ud.user_value::()?; + let mut graph = graph_ud.borrow_mut::()?; + let precedence = precedence.unwrap_or(0); + graph.add_edge(from, this, precedence); + Ok(()) + }, + ); + + methods.add_function( + "add_edge_to", + |_, (this_ud, to_ud, precedence): (AnyUserData, AnyUserData, Option)| { + let this = *this_ud.borrow::>()?; + let to = *to_ud.borrow::>()?; + let graph_ud = this_ud.user_value::()?; + let mut graph = graph_ud.borrow_mut::()?; + let precedence = precedence.unwrap_or(0); + graph.add_edge(this, to, precedence); + Ok(()) + }, + ); + + methods.add_function("debug_info", |l, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let debug_info = match graph.node_debug_info(node) { + Some(debug_info) => debug_info, + None => return Ok(None), + }; + let result = l.create_table()?; + for entry in debug_info.iter() { + result.set(&graph[entry.key], &graph[entry.value])?; + } + Ok(Some(result)) + }); + + methods.add_function("definiens_span", |_, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let source_info = match graph.source_info(node) { + Some(source_info) => source_info, + None => return Ok(None), + }; + Ok(Some(source_info.definiens_span.clone())) + }); + + methods.add_function("local_id", |_, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + Ok(graph[node].id().local_id()) + }); + + methods.add_function( + "set_debug_info", + |_, (node_ud, k, v): (AnyUserData, String, String)| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let mut graph = graph_ud.borrow_mut::()?; + let k = graph.add_string(&k); + let v = graph.add_string(&v); + graph.node_debug_info_mut(node).add(k, v); + Ok(()) + }, + ); + + methods.add_function( + "set_definiens_span", + |_, (node_ud, definiens_span): (AnyUserData, Span)| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let mut graph = graph_ud.borrow_mut::()?; + graph.source_info_mut(node).definiens_span = definiens_span; + Ok(()) + }, + ); + + methods.add_function("set_span", |_, (node_ud, span): (AnyUserData, Span)| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let mut graph = graph_ud.borrow_mut::()?; + graph.source_info_mut(node).span = span; + Ok(()) + }); + + methods.add_function( + "set_syntax_type", + |_, (node_ud, syntax_type): (AnyUserData, String)| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let mut graph = graph_ud.borrow_mut::()?; + let syntax_type = graph.add_string(&syntax_type); + graph.source_info_mut(node).syntax_type = ControlledOption::some(syntax_type); + Ok(()) + }, + ); + + methods.add_function("span", |_, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let source_info = match graph.source_info(node) { + Some(source_info) => source_info, + None => return Ok(None), + }; + Ok(Some(source_info.span.clone())) + }); + + methods.add_function("syntax_type", |_, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let source_info = match graph.source_info(node) { + Some(source_info) => source_info, + None => return Ok(None), + }; + let syntax_type = match source_info.syntax_type.into_option() { + Some(syntax_type) => syntax_type, + None => return Ok(None), + }; + Ok(Some(graph[syntax_type].to_string())) + }); + + methods.add_meta_function(mlua::MetaMethod::ToString, |_, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let mut display = graph[node].display(&graph).to_string(); + if let Some(source_info) = graph.source_info(node) { + display.pop(); // remove the trailing ] + if let Some(syntax_type) = source_info.syntax_type.into_option() { + write!(&mut display, " ({})", syntax_type.display(&graph)).unwrap(); + } + if source_info.span != Span::default() { + write!( + &mut display, + " at {}:{}-{}:{}", + source_info.span.start.line, + source_info.span.start.column.utf8_offset, + source_info.span.end.line, + source_info.span.end.column.utf8_offset, + ) + .unwrap(); + } + if source_info.definiens_span != Span::default() { + write!( + &mut display, + " def {}:{}-{}:{}", + source_info.definiens_span.start.line, + source_info.definiens_span.start.column.utf8_offset, + source_info.definiens_span.end.line, + source_info.definiens_span.end.column.utf8_offset, + ) + .unwrap(); + } + display.push(']'); + } + Ok(display) + }); + } +} diff --git a/stack-graphs/tests/it/lua.rs b/stack-graphs/tests/it/lua.rs new file mode 100644 index 000000000..34f4dc585 --- /dev/null +++ b/stack-graphs/tests/it/lua.rs @@ -0,0 +1,295 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use std::collections::HashSet; + +use maplit::hashset; +use stack_graphs::graph::NodeID; +use stack_graphs::graph::StackGraph; + +const TEST_PRELUDE: &str = r#" + function assert_eq(thing, expected, actual) + if expected ~= actual then + error("Expected "..thing.." "..expected..", got "..actual) + end + end + + function deepeq(t1, t2, prefix) + prefix = prefix or "" + local ty1 = type(t1) + local ty2 = type(t2) + if ty1 ~= ty2 then + local msg = "different types for lhs"..prefix.." ("..ty1..") and rhs"..prefix.." ("..ty2..")" + return false, {msg} + end + + -- non-table types can be directly compared + if ty1 ~= 'table' and ty2 ~= 'table' then + if t1 ~= t2 then + local msg = "different values for lhs"..prefix.." ("..t1..") and rhs"..prefix.." ("..t2..")" + return false, {msg} + end + return true, {} + end + + equal = true + diffs = {} + for k2, v2 in pairs(t2) do + local v1 = t1[k2] + if v1 == nil then + equal = false + diffs[#diffs+1] = "missing lhs"..prefix.."."..k2 + else + local e, d = deepeq(v1, v2, prefix.."."..k2) + equal = equal and e + table.move(d, 1, #d, #diffs+1, diffs) + end + end + for k1, v1 in pairs(t1) do + local v2 = t2[k1] + if v2 == nil then + equal = false + diffs[#diffs+1] = "missing rhs"..prefix.."."..k1 + end + end + return equal, diffs + end + + function assert_deepeq(thing, expected, actual) + local eq, diffs = deepeq(expected, actual) + if not eq then + error("Unexpected "..thing..": "..table.concat(diffs, ", ")) + end + end +"#; + +fn new_lua() -> mlua::Lua { + let l = mlua::Lua::new(); + l.load(TEST_PRELUDE) + .set_name("test prelude") + .exec() + .expect("Error loading test prelude"); + l +} + +trait CheckLua { + /// Executes a chunk of Lua code. If it returns a string, interprets that string as an + /// error message, and translates that into an `anyhow` error. + fn check_without_graph(&self, chunk: &str) -> Result<(), mlua::Error>; + fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error>; +} + +impl CheckLua for mlua::Lua { + fn check_without_graph(&self, chunk: &str) -> Result<(), mlua::Error> { + self.load(chunk).set_name("test chunk").exec() + } + + fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error> { + self.scope(|scope| { + let graph = scope.create_userdata_ref_mut(graph); + self.load(chunk).set_name("test chunk").call(graph) + }) + } +} + +#[test] +fn can_deepeq_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua(); + l.check_without_graph( + r#" + function check_deepeq(lhs, rhs, expected, expected_diffs) + local actual, actual_diffs = deepeq(lhs, rhs) + actual_diffs = table.concat(actual_diffs, ", ") + assert_eq("deepeq", expected, actual) + assert_eq("differences", expected_diffs, actual_diffs) + end + + check_deepeq(0, 0, true, "") + check_deepeq(0, 1, false, "different values for lhs (0) and rhs (1)") + + check_deepeq({"a", "b", "c"}, {"a", "b", "c"}, true, "") + check_deepeq({"a", "b", "c"}, {"a", "b"}, false, "missing rhs.3") + check_deepeq({"a", "b", "c"}, {"a", "b", "d"}, false, "different values for lhs.3 (c) and rhs.3 (d)") + + check_deepeq({a=1, b=2, c=3}, {a=1, b=2, c=3}, true, "") + check_deepeq({a=1, b=2, c=3}, {a=1, b=2}, false, "missing rhs.c") + check_deepeq({a=1, b=2, c=3}, {a=1, b=2, c=4}, false, "different values for lhs.c (3) and rhs.c (4)") + check_deepeq({a=1, b=2, c=3}, {a=1, b=2, d=3}, false, "missing lhs.d, missing rhs.c") + "#, + )?; + Ok(()) +} + +#[test] +fn can_create_nodes_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua(); + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local file = graph:file("test.py") + local n0 = file:internal_scope_node() + local n1 = file:internal_scope_node() + assert_eq("local ID", 0, n0:local_id()) + assert_eq("local ID", 1, n1:local_id()) + "#, + )?; + + let node_count = graph.iter_nodes().count(); + assert_eq!(node_count, 4); // Include the predefined ROOT and JUMP TO nodes in the count + + let file = graph.get_file("test.py").expect("Cannot find file"); + let n0 = graph.node_for_id(NodeID::new_in_file(file, 0)); + assert!(n0.is_some(), "Cannot find node 0"); + let n1 = graph.node_for_id(NodeID::new_in_file(file, 1)); + assert!(n1.is_some(), "Cannot find node 1"); + + Ok(()) +} + +#[test] +fn can_set_source_info_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua(); + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local file = graph:file("test.py") + local n0 = file:internal_scope_node() + + n0:set_syntax_type("function") + assert_eq("syntax type", "function", n0:syntax_type()) + + n0:set_span { + start={line=1, column={utf8_offset=1}}, + ["end"]={line=1, column={utf8_offset=19}}, + } + assert_eq("start line", 1, n0:span().start.line) + assert_eq("start column", 1, n0:span().start.column.utf8_offset) + assert_eq("end line", 1, n0:span()["end"].line) + assert_eq("end column", 19, n0:span()["end"].column.utf8_offset) + + n0:set_definiens_span { + start={line=2, column={utf8_offset=1}}, + ["end"]={line=78, column={utf8_offset=24}}, + } + assert_eq("start line", 2, n0:definiens_span().start.line) + assert_eq("start column", 1, n0:definiens_span().start.column.utf8_offset) + assert_eq("end line", 78, n0:definiens_span()["end"].line) + assert_eq("end column", 24, n0:definiens_span()["end"].column.utf8_offset) + + assert_eq("node", "[test.py(0) scope (function) at 1:1-1:19 def 2:1-78:24]", tostring(n0)) + "#, + )?; + Ok(()) +} + +#[test] +fn can_set_debug_info_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua(); + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local file = graph:file("test.py") + local n0 = file:internal_scope_node() + n0:set_debug_info("k1", "v1") + n0:set_debug_info("k2", "v2") + local expected = { k1="v1", k2="v2" } + assert_deepeq("debug info", expected, n0:debug_info()) + "#, + )?; + Ok(()) +} + +#[test] +fn can_create_edges_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua(); + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local file = graph:file("test.py") + local n0 = file:internal_scope_node() + local n1 = file:internal_scope_node() + n0:add_edge_to(n1) + n0:add_edge_from(n1, 10) + "#, + )?; + + let file = graph.get_file("test.py").expect("Cannot find file"); + let n0 = graph + .node_for_id(NodeID::new_in_file(file, 0)) + .expect("Cannot find node 0"); + let n1 = graph + .node_for_id(NodeID::new_in_file(file, 1)) + .expect("Cannot find node 1"); + + let edges_from_n0 = graph + .outgoing_edges(n0) + .map(|edge| (edge.sink, edge.precedence)) + .collect::>(); + assert_eq!(edges_from_n0, hashset! {(n1, 0)}); + + let edges_from_n1 = graph + .outgoing_edges(n1) + .map(|edge| (edge.sink, edge.precedence)) + .collect::>(); + assert_eq!(edges_from_n1, hashset! {(n0, 10)}); + + Ok(()) +} + +#[test] +fn can_create_all_node_types_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua(); + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local root = graph:root_node() + local jump_to = graph:jump_to_node() + local file = graph:file("test.py") + local drop_scopes = file:drop_scopes_node() + local exported = file:exported_scope_node() + local internal = file:internal_scope_node() + local pop_scoped_symbol = file:pop_scoped_symbol_node("foo") + local scoped_definition = file:scoped_definition_node("bar") + local pop_symbol = file:pop_symbol_node("foo") + local definition = file:definition_node("bar") + local push_scoped_symbol = file:push_scoped_symbol_node("foo", exported) + local scoped_reference = file:scoped_reference_node("bar", exported) + local push_symbol = file:push_symbol_node("foo") + local reference = file:reference_node("bar") + local actual = {} + for node in graph:nodes() do + table.insert(actual, tostring(node)) + end + assert_deepeq("nodes", { + "[root]", + "[jump to scope]", + "[test.py(0) drop scopes]", + "[test.py(1) exported scope]", + "[test.py(2) scope]", + "[test.py(3) pop scoped foo]", + "[test.py(4) scoped definition bar]", + "[test.py(5) pop foo]", + "[test.py(6) definition bar]", + "[test.py(7) push scoped foo test.py(1)]", + "[test.py(8) scoped reference bar test.py(1)]", + "[test.py(9) push foo]", + "[test.py(10) reference bar]", + }, actual) + "#, + )?; + Ok(()) +} diff --git a/stack-graphs/tests/it/main.rs b/stack-graphs/tests/it/main.rs index bea8a6030..f5d12306c 100644 --- a/stack-graphs/tests/it/main.rs +++ b/stack-graphs/tests/it/main.rs @@ -21,6 +21,8 @@ mod can_jump_to_definition; mod can_jump_to_definition_with_forward_partial_path_stitching; mod cycles; mod graph; +#[cfg(feature = "lua")] +mod lua; mod partial; #[cfg(feature = "serde")] mod serde; From 91e44c6604d5c221ce10a8b67041f757e31d0615 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Sat, 18 Nov 2023 16:28:12 -0500 Subject: [PATCH 02/10] Document Lua API --- stack-graphs/src/lua.rs | 272 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) diff --git a/stack-graphs/src/lua.rs b/stack-graphs/src/lua.rs index 8831ea242..408c3399b 100644 --- a/stack-graphs/src/lua.rs +++ b/stack-graphs/src/lua.rs @@ -61,6 +61,278 @@ //! stack-graphs = { version="0.13", features=["lua"] } //! mlua = { version="0.9", features=["lua54", "vendored"] } //! ``` +//! +//! ## Lua API +//! +//! ### Stack graphs +//! +//! The following Lua methods are available on a stack graph instance: +//! +//! #### `file` +//! +//! ``` lua +//! local file = graph:file(name) +//! ``` +//! +//! Returns the file in the stack graph with the given name, creating it if necessary. +//! +//! #### `jump_to_node` +//! +//! ``` lua +//! local node = graph:jump_to_node() +//! ``` +//! +//! Returns the graph's jump-to node. +//! +//! #### `nodes` +//! +//! ``` lua +//! for node in graph:nodes() do +//! -- whatever +//! end +//! ``` +//! +//! Returns an iterator of every node in the stack graph. +//! +//! #### `root_node` +//! +//! ``` lua +//! local node = graph:root_node() +//! ``` +//! +//! Returns the graph's root node. +//! +//! ### Files +//! +//! The following Lua methods are available on a file instance: +//! +//! #### `definition_node` +//! +//! ``` lua +//! local node = file:definition_node(symbol) +//! ``` +//! +//! Adds a new definition node to this file. `symbol` must be a string, or an instance that can be +//! converted to a string via its `tostring` method. +//! +//! #### `drop_scopes_node` +//! +//! ``` lua +//! local node = file:drop_scopes_node() +//! ``` +//! +//! Adds a new drop scopes node to this file. +//! +//! #### `exported_scope_node` +//! +//! ``` lua +//! local node = file:exported_scope_node() +//! ``` +//! +//! Adds a new exported scope node to this file. +//! +//! #### `internal_scope_node` +//! +//! ``` lua +//! local node = file:internal_scope_node() +//! ``` +//! +//! Adds a new internal scope node to this file. +//! +//! #### `pop_scoped_symbol_node` +//! +//! ``` lua +//! local node = file:pop_scoped_symbol_node(symbol) +//! ``` +//! +//! Adds a new pop scoped symbol node to this file. `symbol` must be a string, or an instance that +//! can be converted to a string via its `tostring` method. +//! +//! #### `pop_symbol_node` +//! +//! ``` lua +//! local node = file:pop_symbol_node(symbol) +//! ``` +//! +//! Adds a new pop symbol node to this file. `symbol` must be a string, or an instance that can be +//! converted to a string via its `tostring` method. +//! +//! #### `push_scoped_symbol_node` +//! +//! ``` lua +//! local node = file:push_scoped_symbol_node(symbol, scope) +//! ``` +//! +//! Adds a new push scoped symbol node to this file. `symbol` must be a string, or an instance +//! that can be converted to a string via its `tostring` method. `scope` must be an exported scope +//! node. +//! +//! #### `push_symbol_node` +//! +//! ``` lua +//! local node = file:push_symbol_node(symbol) +//! ``` +//! +//! Adds a new push symbol node to this file. `symbol` must be a string, or an instance that can +//! be converted to a string via its `tostring` method. +//! +//! #### `reference_node` +//! +//! ``` lua +//! local node = file:reference_node(symbol) +//! ``` +//! +//! Adds a new definition node to this file. `symbol` must be a string, or an instance that can be +//! converted to a string via its `tostring` method. +//! +//! #### `scoped_definition_node` +//! +//! ``` lua +//! local node = file:scoped_definition_node(symbol) +//! ``` +//! +//! Adds a new scoped definition node to this file. `symbol` must be a string, or an instance that +//! can be converted to a string via its `tostring` method. +//! +//! #### `scoped_reference_node` +//! +//! ``` lua +//! local node = file:scoped_reference_node(symbol) +//! ``` +//! +//! Adds a new scoped reference node to this file. `symbol` must be a string, or an instance that +//! can be converted to a string via its `tostring` method. +//! +//! ### Nodes +//! +//! The following Lua methods are available on a node instance: +//! +//! #### `add_edge_from` +//! +//! ``` lua +//! node:add_edge_from(other, precedence) +//! ``` +//! +//! Adds an edge from another node to this node. `precedence` is optional; it defaults to 0 if not +//! given. +//! +//! #### `add_edge_to` +//! +//! ``` lua +//! node:add_edge_to(other, precedence) +//! ``` +//! +//! Adds an edge from this node to another node. `precedence` is optional; it defaults to 0 if not +//! given. +//! +//! #### `debug_info` +//! +//! ``` lua +//! let info = node:debug_info() +//! ``` +//! +//! Returns a Lua table containing all of the debug info added to this node. +//! +//! #### `definiens_span` +//! +//! ``` lua +//! let span = node:definiens_span() +//! ``` +//! +//! Returns the definiens span of this node. (See [`set_definiens_span`](#set_definiens_span) for +//! the structure of a span.) +//! +//! #### `local_id` +//! +//! ``` lua +//! let local_id = node:local_id() +//! ``` +//! +//! Returns the local ID of this node within its file. +//! +//! #### `set_debug_info` +//! +//! ``` lua +//! node:add_debug_info(key, value) +//! ``` +//! +//! Adds a new debug info to this node. `key` and `value` must each be a string, or an instance +//! that can be converted to a string via its `tostring` method. +//! +//! #### `set_definiens_span` +//! +//! ``` lua +//! node:set_definiens_span { +//! start = { +//! line = 1, +//! column = { utf8_offset = 1, utf16_offset = 1, grapheme_offset = 1 }, +//! -- UTF-8 offsets within the source file of the line containing the span +//! containing_line = { start = 1, end = 14 }, +//! -- UTF-8 offsets within the source file of the line containing the span, with leading and +//! -- trailing whitespace removed +//! trimmed_line = { start = 2, end = 12 }, +//! }, +//! end = { +//! line = 2, +//! column = { utf8_offset = 12, utf16_offset = 10, grapheme_offset = 8 }, +//! containing_line = { start = 1, end = 14 }, +//! trimmed_line = { start = 1, end = 14 }, +//! }, +//! } +//! ``` +//! +//! Sets the definiens span of this node. All entries in the table are optional, and default to 0 +//! if not provided. +//! +//! #### `set_span` +//! +//! ``` lua +//! node:set_span { +//! start = { +//! line = 1, +//! column = { utf8_offset = 1, utf16_offset = 1, grapheme_offset = 1 }, +//! -- UTF-8 offsets within the source file of the line containing the span +//! containing_line = { start = 1, end = 14 }, +//! -- UTF-8 offsets within the source file of the line containing the span, with leading and +//! -- trailing whitespace removed +//! trimmed_line = { start = 2, end = 12 }, +//! }, +//! end = { +//! line = 2, +//! column = { utf8_offset = 12, utf16_offset = 10, grapheme_offset = 8 }, +//! containing_line = { start = 1, end = 14 }, +//! trimmed_line = { start = 1, end = 14 }, +//! }, +//! } +//! ``` +//! +//! Sets the span of this node. All entries in the table are optional, and default to 0 if not +//! provided. +//! +//! #### `set_syntax_type` +//! +//! ``` lua +//! node:set_syntax_type(syntax_type) +//! ``` +//! +//! Sets the syntax type of this node. `syntax_type` must be a string, or an instance that can be +//! converted to a string via its `tostring` method. +//! +//! #### `span` +//! +//! ``` lua +//! let span = node:span() +//! ``` +//! +//! Returns the span of this node. (See [`set_span`](#set_span) for the structure of a span.) +//! +//! #### `syntax_type` +//! +//! ``` lua +//! let syntax_type = node:syntax_type() +//! ``` +//! +//! Returns the syntax type of this node. // Implementation notes: Stack graphs, files, and nodes can live inside the Lua interpreter as // objects. They are each wrapped in a userdata, with a metatable defining the methods that are From b450a8731f772d65df96e5954c08deca32c298bf Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Sat, 18 Nov 2023 20:12:25 -0500 Subject: [PATCH 03/10] Access root and jump-to nodes from file's in Lua --- stack-graphs/src/lua.rs | 32 ++++++++++++++++++++++++++++++++ stack-graphs/tests/it/lua.rs | 2 ++ 2 files changed, 34 insertions(+) diff --git a/stack-graphs/src/lua.rs b/stack-graphs/src/lua.rs index 408c3399b..d37ea52f8 100644 --- a/stack-graphs/src/lua.rs +++ b/stack-graphs/src/lua.rs @@ -139,6 +139,14 @@ //! //! Adds a new internal scope node to this file. //! +//! #### `jump_to_node` +//! +//! ``` lua +//! local node = file:jump_to_node() +//! ``` +//! +//! Returns the root node of the graph containing this file. +//! //! #### `pop_scoped_symbol_node` //! //! ``` lua @@ -185,6 +193,14 @@ //! Adds a new definition node to this file. `symbol` must be a string, or an instance that can be //! converted to a string via its `tostring` method. //! +//! #### `root_node` +//! +//! ``` lua +//! local node = file:root_node() +//! ``` +//! +//! Returns the root node of the graph containing this file. +//! //! #### `scoped_definition_node` //! //! ``` lua @@ -490,6 +506,14 @@ impl UserData for Handle { Ok(node_ud) }); + methods.add_function("jump_to_node", |l, file_ud: AnyUserData| { + let graph_ud = file_ud.user_value::()?; + let node = StackGraph::jump_to_node(); + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + methods.add_function( "pop_scoped_symbol_node", |l, (file_ud, symbol): (AnyUserData, String)| { @@ -595,6 +619,14 @@ impl UserData for Handle { }, ); + methods.add_function("root_node", |l, file_ud: AnyUserData| { + let graph_ud = file_ud.user_value::()?; + let node = StackGraph::root_node(); + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + methods.add_function( "scoped_definition_node", |l, (file_ud, symbol): (AnyUserData, String)| { diff --git a/stack-graphs/tests/it/lua.rs b/stack-graphs/tests/it/lua.rs index 34f4dc585..66d425a95 100644 --- a/stack-graphs/tests/it/lua.rs +++ b/stack-graphs/tests/it/lua.rs @@ -259,6 +259,8 @@ fn can_create_all_node_types_from_lua() -> Result<(), anyhow::Error> { local root = graph:root_node() local jump_to = graph:jump_to_node() local file = graph:file("test.py") + local file_root = file:root_node() + local file_jump_to = file:jump_to_node() local drop_scopes = file:drop_scopes_node() local exported = file:exported_scope_node() local internal = file:internal_scope_node() From 9ae6a87ad865dd4c3c247aff37054848a6c7fb82 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Mon, 20 Nov 2023 16:05:48 -0500 Subject: [PATCH 04/10] Iterate nodes belonging to file --- stack-graphs/src/lua.rs | 48 +++++++++++++++++++++++++++++++++ stack-graphs/tests/it/lua.rs | 52 ++++++++++++++++++++++++++++++++---- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/stack-graphs/src/lua.rs b/stack-graphs/src/lua.rs index d37ea52f8..d493ea7ec 100644 --- a/stack-graphs/src/lua.rs +++ b/stack-graphs/src/lua.rs @@ -514,6 +514,54 @@ impl UserData for Handle { Ok(node_ud) }); + methods.add_function("nodes", |l, file_ud: AnyUserData| { + let iter = l.create_function( + |l, (file_ud, prev_node_ud): (AnyUserData, Option)| { + // Pull out the node handle from the previous iteration. + let prev_index = match prev_node_ud { + Some(prev_node_ud) => { + let prev_node = prev_node_ud.borrow::>()?; + prev_node.as_u32() + } + None => 0, + }; + + // Loop through the next node handles until we find one belonging to the file. + let graph_ud = file_ud.user_value::()?; + let node = { + let file = *file_ud.borrow::>()?; + let graph = graph_ud.borrow::()?; + let node_count = graph.nodes.len() as u32; + let mut node_index = unsafe { NonZeroU32::new_unchecked(prev_index + 1) }; + loop { + let handle = Handle::::new(node_index); + + // If we reach the end without finding a matching node, return nil + // to terminate the iterator. + if node_index.get() == node_count { + return Ok(None); + } + + // If the node belongs to the file, break out of the loop to use this + // node as the next result of the iterator. + if graph[handle].file().map(|f| f == file).unwrap_or(false) { + break handle; + } + + // Otherwise try the next node. + node_index = node_index.checked_add(1).unwrap(); + } + }; + + // Wrap up the node handle that we just found. + let node_ud = l.create_userdata::>(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(Some(node_ud)) + }, + )?; + Ok((iter, file_ud, None::)) + }); + methods.add_function( "pop_scoped_symbol_node", |l, (file_ud, symbol): (AnyUserData, String)| { diff --git a/stack-graphs/tests/it/lua.rs b/stack-graphs/tests/it/lua.rs index 66d425a95..bb0a019d7 100644 --- a/stack-graphs/tests/it/lua.rs +++ b/stack-graphs/tests/it/lua.rs @@ -65,6 +65,14 @@ const TEST_PRELUDE: &str = r#" error("Unexpected "..thing..": "..table.concat(diffs, ", ")) end end + + function iter_tostring(...) + local result = {} + for element in ... do + table.insert(result, tostring(element)) + end + return result + end "#; fn new_lua() -> mlua::Lua { @@ -272,10 +280,7 @@ fn can_create_all_node_types_from_lua() -> Result<(), anyhow::Error> { local scoped_reference = file:scoped_reference_node("bar", exported) local push_symbol = file:push_symbol_node("foo") local reference = file:reference_node("bar") - local actual = {} - for node in graph:nodes() do - table.insert(actual, tostring(node)) - end + assert_deepeq("nodes", { "[root]", "[jump to scope]", @@ -290,7 +295,44 @@ fn can_create_all_node_types_from_lua() -> Result<(), anyhow::Error> { "[test.py(8) scoped reference bar test.py(1)]", "[test.py(9) push foo]", "[test.py(10) reference bar]", - }, actual) + }, iter_tostring(graph:nodes())) + "#, + )?; + Ok(()) +} + +#[test] +fn can_iterate_nodes_in_file() -> Result<(), anyhow::Error> { + let l = new_lua(); + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local file1 = graph:file("test1.py") + local file2 = graph:file("test2.py") + file1:internal_scope_node() + file2:internal_scope_node() + file1:internal_scope_node() + file2:internal_scope_node() + file2:internal_scope_node() + file1:internal_scope_node() + file2:internal_scope_node() + file1:internal_scope_node() + + assert_deepeq("nodes", { + "[test1.py(0) scope]", + "[test1.py(1) scope]", + "[test1.py(2) scope]", + "[test1.py(3) scope]", + }, iter_tostring(file1:nodes())) + + assert_deepeq("nodes", { + "[test2.py(0) scope]", + "[test2.py(1) scope]", + "[test2.py(2) scope]", + "[test2.py(3) scope]", + }, iter_tostring(file2:nodes())) "#, )?; Ok(()) From 9a30497f2fb8086be3504228dfcbe7279073ef29 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Mon, 20 Nov 2023 16:43:32 -0500 Subject: [PATCH 05/10] Add Lua wrappers for edges You can't do anything with them other than call `tostring` on them, but even that is helpful in test cases! --- stack-graphs/src/lua.rs | 135 ++++++++++++++++++++++++++++++++--- stack-graphs/tests/it/lua.rs | 63 +++++++++------- 2 files changed, 164 insertions(+), 34 deletions(-) diff --git a/stack-graphs/src/lua.rs b/stack-graphs/src/lua.rs index d493ea7ec..67a5201e7 100644 --- a/stack-graphs/src/lua.rs +++ b/stack-graphs/src/lua.rs @@ -68,6 +68,14 @@ //! //! The following Lua methods are available on a stack graph instance: //! +//! #### `edges` +//! +//! ``` lua +//! let edges = graph:edges() +//! ``` +//! +//! Returns an array containing all of the edges in the graph. +//! //! #### `file` //! //! ``` lua @@ -123,6 +131,14 @@ //! //! Adds a new drop scopes node to this file. //! +//! #### `edges` +//! +//! ``` lua +//! let edges = file:edges() +//! ``` +//! +//! Returns an array containing all of the edges starting from or leaving a node in this file. +//! //! #### `exported_scope_node` //! //! ``` lua @@ -266,6 +282,14 @@ //! //! Returns the local ID of this node within its file. //! +//! #### `outgoing_edges` +//! +//! ``` lua +//! let edges = node:outgoing_edges() +//! ``` +//! +//! Returns an array containing all of the edges leaving this node. +//! //! #### `set_debug_info` //! //! ``` lua @@ -381,12 +405,26 @@ use mlua::UserData; use mlua::UserDataMethods; use crate::arena::Handle; +use crate::graph::Edge; use crate::graph::File; use crate::graph::Node; use crate::graph::StackGraph; impl UserData for StackGraph { fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_function("edges", |l, graph_ud: AnyUserData| { + let graph = graph_ud.borrow::()?; + let mut edges = Vec::new(); + for node in graph.iter_nodes() { + for edge in graph.outgoing_edges(node) { + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud.clone())?; + edges.push(edge_ud); + } + } + Ok(edges) + }); + methods.add_function("file", |l, (graph_ud, name): (AnyUserData, String)| { let file = { let mut graph = graph_ud.borrow_mut::()?; @@ -476,6 +514,39 @@ impl UserData for Handle { Ok(node_ud) }); + methods.add_function("edges", |l, file_ud: AnyUserData| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let mut edges = Vec::new(); + // First find any edges from the singleton nodes _to_ a node in this file. + for edge in graph.outgoing_edges(StackGraph::root_node()) { + if !graph[edge.sink].file().map(|f| f == file).unwrap_or(false) { + continue; + } + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud.clone())?; + edges.push(edge_ud); + } + for edge in graph.outgoing_edges(StackGraph::jump_to_node()) { + if !graph[edge.sink].file().map(|f| f == file).unwrap_or(false) { + continue; + } + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud.clone())?; + edges.push(edge_ud); + } + // Then find any edges _starting_ from a node in this file. + for node in graph.nodes_for_file(file) { + for edge in graph.outgoing_edges(node) { + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud.clone())?; + edges.push(edge_ud); + } + } + Ok(edges) + }); + methods.add_function("exported_scope_node", |l, file_ud: AnyUserData| { let file = *file_ud.borrow::>()?; let graph_ud = file_ud.user_value::()?; @@ -729,27 +800,45 @@ impl UserData for Handle { fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { methods.add_function( "add_edge_from", - |_, (this_ud, from_ud, precedence): (AnyUserData, AnyUserData, Option)| { + |l, (this_ud, from_ud, precedence): (AnyUserData, AnyUserData, Option)| { let this = *this_ud.borrow::>()?; let from = *from_ud.borrow::>()?; let graph_ud = this_ud.user_value::()?; - let mut graph = graph_ud.borrow_mut::()?; let precedence = precedence.unwrap_or(0); - graph.add_edge(from, this, precedence); - Ok(()) + { + let mut graph = graph_ud.borrow_mut::()?; + graph.add_edge(from, this, precedence); + } + let edge = Edge { + source: from, + sink: this, + precedence, + }; + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud)?; + Ok(edge_ud) }, ); methods.add_function( "add_edge_to", - |_, (this_ud, to_ud, precedence): (AnyUserData, AnyUserData, Option)| { + |l, (this_ud, to_ud, precedence): (AnyUserData, AnyUserData, Option)| { let this = *this_ud.borrow::>()?; let to = *to_ud.borrow::>()?; let graph_ud = this_ud.user_value::()?; - let mut graph = graph_ud.borrow_mut::()?; let precedence = precedence.unwrap_or(0); - graph.add_edge(this, to, precedence); - Ok(()) + { + let mut graph = graph_ud.borrow_mut::()?; + graph.add_edge(this, to, precedence); + } + let edge = Edge { + source: this, + sink: to, + precedence, + }; + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud)?; + Ok(edge_ud) }, ); @@ -786,6 +875,19 @@ impl UserData for Handle { Ok(graph[node].id().local_id()) }); + methods.add_function("outgoing_edges", |l, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let mut edges = Vec::new(); + for edge in graph.outgoing_edges(node) { + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud.clone())?; + edges.push(edge_ud); + } + Ok(edges) + }); + methods.add_function( "set_debug_info", |_, (node_ud, k, v): (AnyUserData, String, String)| { @@ -894,3 +996,20 @@ impl UserData for Handle { }); } } + +impl UserData for Edge { + fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_meta_function(mlua::MetaMethod::ToString, |_, edge_ud: AnyUserData| { + let edge = *edge_ud.borrow::()?; + let graph_ud = edge_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let display = format!( + "{} -{}-> {}", + edge.source.display(&graph), + edge.precedence, + edge.sink.display(&graph), + ); + Ok(display) + }); + } +} diff --git a/stack-graphs/tests/it/lua.rs b/stack-graphs/tests/it/lua.rs index bb0a019d7..3cbd2415e 100644 --- a/stack-graphs/tests/it/lua.rs +++ b/stack-graphs/tests/it/lua.rs @@ -5,9 +5,6 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ -use std::collections::HashSet; - -use maplit::hashset; use stack_graphs::graph::NodeID; use stack_graphs::graph::StackGraph; @@ -66,6 +63,11 @@ const TEST_PRELUDE: &str = r#" end end + function values(t) + local i = 0 + return function() i = i + 1; return t[i] end + end + function iter_tostring(...) local result = {} for element in ... do @@ -225,34 +227,43 @@ fn can_create_edges_from_lua() -> Result<(), anyhow::Error> { &mut graph, r#" local graph = ... + local root = graph:root_node() local file = graph:file("test.py") local n0 = file:internal_scope_node() local n1 = file:internal_scope_node() - n0:add_edge_to(n1) - n0:add_edge_from(n1, 10) + local e0 = n0:add_edge_to(n1) + local e1 = n0:add_edge_from(n1, 10) + local e2 = n0:add_edge_to(root) + local e3 = n0:add_edge_from(root) + assert_eq("edge", "[test.py(0) scope] -0-> [test.py(1) scope]", tostring(e0)) + assert_eq("edge", "[test.py(1) scope] -10-> [test.py(0) scope]", tostring(e1)) + + assert_deepeq("node edges", { + "[test.py(0) scope] -0-> [root]", + "[test.py(0) scope] -0-> [test.py(1) scope]", + }, iter_tostring(values(n0:outgoing_edges()))) + assert_deepeq("node edges", { + "[test.py(1) scope] -10-> [test.py(0) scope]", + }, iter_tostring(values(n1:outgoing_edges()))) + assert_deepeq("node edges", { + "[root] -0-> [test.py(0) scope]", + }, iter_tostring(values(root:outgoing_edges()))) + + assert_deepeq("file edges", { + "[root] -0-> [test.py(0) scope]", + "[test.py(0) scope] -0-> [root]", + "[test.py(0) scope] -0-> [test.py(1) scope]", + "[test.py(1) scope] -10-> [test.py(0) scope]", + }, iter_tostring(values(file:edges()))) + + assert_deepeq("graph edges", { + "[root] -0-> [test.py(0) scope]", + "[test.py(0) scope] -0-> [root]", + "[test.py(0) scope] -0-> [test.py(1) scope]", + "[test.py(1) scope] -10-> [test.py(0) scope]", + }, iter_tostring(values(graph:edges()))) "#, )?; - - let file = graph.get_file("test.py").expect("Cannot find file"); - let n0 = graph - .node_for_id(NodeID::new_in_file(file, 0)) - .expect("Cannot find node 0"); - let n1 = graph - .node_for_id(NodeID::new_in_file(file, 1)) - .expect("Cannot find node 1"); - - let edges_from_n0 = graph - .outgoing_edges(n0) - .map(|edge| (edge.sink, edge.precedence)) - .collect::>(); - assert_eq!(edges_from_n0, hashset! {(n1, 0)}); - - let edges_from_n1 = graph - .outgoing_edges(n1) - .map(|edge| (edge.sink, edge.precedence)) - .collect::>(); - assert_eq!(edges_from_n1, hashset! {(n0, 10)}); - Ok(()) } From ea2e20350df4dd6fb685bc01c2703338e47914ef Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Mon, 20 Nov 2023 14:08:12 -0500 Subject: [PATCH 06/10] Move Lua test helpers out into separate (unpublished) package --- Cargo.toml | 1 + lua-helpers/Cargo.toml | 7 ++ lua-helpers/src/lib.rs | 119 +++++++++++++++++++++++++++++++++ stack-graphs/Cargo.toml | 1 + stack-graphs/tests/it/lua.rs | 126 ++--------------------------------- 5 files changed, 135 insertions(+), 119 deletions(-) create mode 100644 lua-helpers/Cargo.toml create mode 100644 lua-helpers/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 587e70d61..cd9b3374f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "1" members = [ # library projects "lsp-positions", + "lua-helpers", "stack-graphs", "tree-sitter-stack-graphs", "languages/*", diff --git a/lua-helpers/Cargo.toml b/lua-helpers/Cargo.toml new file mode 100644 index 000000000..cca90bf07 --- /dev/null +++ b/lua-helpers/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "lua-helpers" +version = "0.0.0" +description = "Unpublished helper methods for our Lua test cases" + +[dependencies] +mlua = { version = "0.9" } diff --git a/lua-helpers/src/lib.rs b/lua-helpers/src/lib.rs new file mode 100644 index 000000000..92a9b34d5 --- /dev/null +++ b/lua-helpers/src/lib.rs @@ -0,0 +1,119 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +const TEST_PRELUDE: &str = r#" + function assert_eq(thing, expected, actual) + if expected ~= actual then + error("Expected "..thing.." "..expected..", got "..actual) + end + end + + function deepeq(t1, t2, prefix) + prefix = prefix or "" + local ty1 = type(t1) + local ty2 = type(t2) + if ty1 ~= ty2 then + local msg = "different types for lhs"..prefix.." ("..ty1..") and rhs"..prefix.." ("..ty2..")" + return false, {msg} + end + + -- non-table types can be directly compared + if ty1 ~= 'table' and ty2 ~= 'table' then + if t1 ~= t2 then + local msg = "different values for lhs"..prefix.." ("..t1..") and rhs"..prefix.." ("..t2..")" + return false, {msg} + end + return true, {} + end + + local equal = true + local diffs = {} + for k2, v2 in pairs(t2) do + local v1 = t1[k2] + if v1 == nil then + equal = false + diffs[#diffs+1] = "missing lhs"..prefix.."."..k2 + else + local e, d = deepeq(v1, v2, prefix.."."..k2) + equal = equal and e + table.move(d, 1, #d, #diffs+1, diffs) + end + end + for k1, v1 in pairs(t1) do + local v2 = t2[k1] + if v2 == nil then + equal = false + diffs[#diffs+1] = "missing rhs"..prefix.."."..k1 + end + end + return equal, diffs + end + + function assert_deepeq(thing, expected, actual) + local eq, diffs = deepeq(expected, actual) + if not eq then + error("Unexpected "..thing..": "..table.concat(diffs, ", ")) + end + end + + function values(t) + local i = 0 + return function() i = i + 1; return t[i] end + end + + function iter_tostring(...) + local result = {} + for element in ... do + table.insert(result, tostring(element)) + end + return result + end +"#; + +pub fn new_lua() -> Result { + let l = mlua::Lua::new(); + l.load(TEST_PRELUDE).set_name("test prelude").exec()?; + Ok(l) +} + +pub trait CheckLua { + fn check(&self, chunk: &str) -> Result<(), mlua::Error>; +} + +impl CheckLua for mlua::Lua { + fn check(&self, chunk: &str) -> Result<(), mlua::Error> { + self.load(chunk).set_name("test chunk").exec() + } +} + +#[test] +fn can_deepeq_from_lua() -> Result<(), mlua::Error> { + let l = new_lua()?; + l.check( + r#" + function check_deepeq(lhs, rhs, expected, expected_diffs) + local actual, actual_diffs = deepeq(lhs, rhs) + actual_diffs = table.concat(actual_diffs, ", ") + assert_eq("deepeq", expected, actual) + assert_eq("differences", expected_diffs, actual_diffs) + end + + check_deepeq(0, 0, true, "") + check_deepeq(0, 1, false, "different values for lhs (0) and rhs (1)") + + check_deepeq({"a", "b", "c"}, {"a", "b", "c"}, true, "") + check_deepeq({"a", "b", "c"}, {"a", "b"}, false, "missing rhs.3") + check_deepeq({"a", "b", "c"}, {"a", "b", "d"}, false, "different values for lhs.3 (c) and rhs.3 (d)") + + check_deepeq({a=1, b=2, c=3}, {a=1, b=2, c=3}, true, "") + check_deepeq({a=1, b=2, c=3}, {a=1, b=2}, false, "missing rhs.c") + check_deepeq({a=1, b=2, c=3}, {a=1, b=2, c=4}, false, "different values for lhs.c (3) and rhs.c (4)") + check_deepeq({a=1, b=2, c=3}, {a=1, b=2, d=3}, false, "missing lhs.d, missing rhs.c") + "#, + )?; + Ok(()) +} diff --git a/stack-graphs/Cargo.toml b/stack-graphs/Cargo.toml index c52131288..31c89ae09 100644 --- a/stack-graphs/Cargo.toml +++ b/stack-graphs/Cargo.toml @@ -46,6 +46,7 @@ thiserror = { version = "1.0" } anyhow = "1.0" assert-json-diff = "2" itertools = "0.10" +lua-helpers = { path = "../lua-helpers" } maplit = "1.0" pretty_assertions = "0.7" serde_json = { version = "1.0" } diff --git a/stack-graphs/tests/it/lua.rs b/stack-graphs/tests/it/lua.rs index 3cbd2415e..02c7c3de3 100644 --- a/stack-graphs/tests/it/lua.rs +++ b/stack-graphs/tests/it/lua.rs @@ -5,99 +5,15 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ +use lua_helpers::new_lua; use stack_graphs::graph::NodeID; use stack_graphs::graph::StackGraph; -const TEST_PRELUDE: &str = r#" - function assert_eq(thing, expected, actual) - if expected ~= actual then - error("Expected "..thing.." "..expected..", got "..actual) - end - end - - function deepeq(t1, t2, prefix) - prefix = prefix or "" - local ty1 = type(t1) - local ty2 = type(t2) - if ty1 ~= ty2 then - local msg = "different types for lhs"..prefix.." ("..ty1..") and rhs"..prefix.." ("..ty2..")" - return false, {msg} - end - - -- non-table types can be directly compared - if ty1 ~= 'table' and ty2 ~= 'table' then - if t1 ~= t2 then - local msg = "different values for lhs"..prefix.." ("..t1..") and rhs"..prefix.." ("..t2..")" - return false, {msg} - end - return true, {} - end - - equal = true - diffs = {} - for k2, v2 in pairs(t2) do - local v1 = t1[k2] - if v1 == nil then - equal = false - diffs[#diffs+1] = "missing lhs"..prefix.."."..k2 - else - local e, d = deepeq(v1, v2, prefix.."."..k2) - equal = equal and e - table.move(d, 1, #d, #diffs+1, diffs) - end - end - for k1, v1 in pairs(t1) do - local v2 = t2[k1] - if v2 == nil then - equal = false - diffs[#diffs+1] = "missing rhs"..prefix.."."..k1 - end - end - return equal, diffs - end - - function assert_deepeq(thing, expected, actual) - local eq, diffs = deepeq(expected, actual) - if not eq then - error("Unexpected "..thing..": "..table.concat(diffs, ", ")) - end - end - - function values(t) - local i = 0 - return function() i = i + 1; return t[i] end - end - - function iter_tostring(...) - local result = {} - for element in ... do - table.insert(result, tostring(element)) - end - return result - end -"#; - -fn new_lua() -> mlua::Lua { - let l = mlua::Lua::new(); - l.load(TEST_PRELUDE) - .set_name("test prelude") - .exec() - .expect("Error loading test prelude"); - l -} - trait CheckLua { - /// Executes a chunk of Lua code. If it returns a string, interprets that string as an - /// error message, and translates that into an `anyhow` error. - fn check_without_graph(&self, chunk: &str) -> Result<(), mlua::Error>; fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error>; } impl CheckLua for mlua::Lua { - fn check_without_graph(&self, chunk: &str) -> Result<(), mlua::Error> { - self.load(chunk).set_name("test chunk").exec() - } - fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error> { self.scope(|scope| { let graph = scope.create_userdata_ref_mut(graph); @@ -106,37 +22,9 @@ impl CheckLua for mlua::Lua { } } -#[test] -fn can_deepeq_from_lua() -> Result<(), anyhow::Error> { - let l = new_lua(); - l.check_without_graph( - r#" - function check_deepeq(lhs, rhs, expected, expected_diffs) - local actual, actual_diffs = deepeq(lhs, rhs) - actual_diffs = table.concat(actual_diffs, ", ") - assert_eq("deepeq", expected, actual) - assert_eq("differences", expected_diffs, actual_diffs) - end - - check_deepeq(0, 0, true, "") - check_deepeq(0, 1, false, "different values for lhs (0) and rhs (1)") - - check_deepeq({"a", "b", "c"}, {"a", "b", "c"}, true, "") - check_deepeq({"a", "b", "c"}, {"a", "b"}, false, "missing rhs.3") - check_deepeq({"a", "b", "c"}, {"a", "b", "d"}, false, "different values for lhs.3 (c) and rhs.3 (d)") - - check_deepeq({a=1, b=2, c=3}, {a=1, b=2, c=3}, true, "") - check_deepeq({a=1, b=2, c=3}, {a=1, b=2}, false, "missing rhs.c") - check_deepeq({a=1, b=2, c=3}, {a=1, b=2, c=4}, false, "different values for lhs.c (3) and rhs.c (4)") - check_deepeq({a=1, b=2, c=3}, {a=1, b=2, d=3}, false, "missing lhs.d, missing rhs.c") - "#, - )?; - Ok(()) -} - #[test] fn can_create_nodes_from_lua() -> Result<(), anyhow::Error> { - let l = new_lua(); + let l = new_lua()?; let mut graph = StackGraph::new(); l.check( &mut graph, @@ -164,7 +52,7 @@ fn can_create_nodes_from_lua() -> Result<(), anyhow::Error> { #[test] fn can_set_source_info_from_lua() -> Result<(), anyhow::Error> { - let l = new_lua(); + let l = new_lua()?; let mut graph = StackGraph::new(); l.check( &mut graph, @@ -202,7 +90,7 @@ fn can_set_source_info_from_lua() -> Result<(), anyhow::Error> { #[test] fn can_set_debug_info_from_lua() -> Result<(), anyhow::Error> { - let l = new_lua(); + let l = new_lua()?; let mut graph = StackGraph::new(); l.check( &mut graph, @@ -221,7 +109,7 @@ fn can_set_debug_info_from_lua() -> Result<(), anyhow::Error> { #[test] fn can_create_edges_from_lua() -> Result<(), anyhow::Error> { - let l = new_lua(); + let l = new_lua()?; let mut graph = StackGraph::new(); l.check( &mut graph, @@ -269,7 +157,7 @@ fn can_create_edges_from_lua() -> Result<(), anyhow::Error> { #[test] fn can_create_all_node_types_from_lua() -> Result<(), anyhow::Error> { - let l = new_lua(); + let l = new_lua()?; let mut graph = StackGraph::new(); l.check( &mut graph, @@ -314,7 +202,7 @@ fn can_create_all_node_types_from_lua() -> Result<(), anyhow::Error> { #[test] fn can_iterate_nodes_in_file() -> Result<(), anyhow::Error> { - let l = new_lua(); + let l = new_lua()?; let mut graph = StackGraph::new(); l.check( &mut graph, From b82ec8b73f93de1bd85b13cde7b3db32182f32c3 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Mon, 20 Nov 2023 13:12:44 -0500 Subject: [PATCH 07/10] Bring in (not-yet-released) tree-sitter updates --- Cargo.toml | 9 +++++++++ tree-sitter-stack-graphs/src/loader.rs | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index cd9b3374f..e44137d10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,12 @@ default-members = [ "stack-graphs", "tree-sitter-stack-graphs", ] + +[patch.crates-io] +# TODO: Revert to regular versioned dependencies once tree-sitter#2773 has been +# merged. +tree-sitter = { git="https://github.com/dcreager/tree-sitter", branch="rust-linking" } +tree-sitter-graph = { git="https://github.com/tree-sitter/tree-sitter-graph", branch="ts-bump" } +tree-sitter-highlight = { git="https://github.com/dcreager/tree-sitter", branch="rust-linking" } +tree-sitter-loader = { git="https://github.com/dcreager/tree-sitter", branch="rust-linking" } +tree-sitter-tags = { git="https://github.com/dcreager/tree-sitter", branch="rust-linking" } diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index 0feb6a28c..2744bcc16 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -743,7 +743,7 @@ impl SupplementedTsLoader { .map_err(LoadError::TreeSitter)?; let configurations = self .0 - .find_language_configurations_at_path(&path) + .find_language_configurations_at_path(&path, false) .map_err(LoadError::TreeSitter)?; let languages = languages .into_iter() From ff1e99d00a7c1578192b7df771e06dd291939d32 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Sat, 18 Nov 2023 20:17:21 -0500 Subject: [PATCH 08/10] Add Lua+tree-sitter stack graph builder This is the spackle that parses a source file using tree-sitter, and calls a Lua function with it and an empty stack graph. The Lua function can do whatever it wants to walk the parse tree and add nodes and edges to the graph. --- stack-graphs/src/lua.rs | 33 ++++++- stack-graphs/tests/it/lua.rs | 2 +- tree-sitter-stack-graphs/Cargo.toml | 8 ++ tree-sitter-stack-graphs/src/lib.rs | 48 ++++++---- tree-sitter-stack-graphs/src/lua.rs | 103 ++++++++++++++++++++++ tree-sitter-stack-graphs/tests/it/lua.rs | 64 ++++++++++++++ tree-sitter-stack-graphs/tests/it/main.rs | 3 + 7 files changed, 241 insertions(+), 20 deletions(-) create mode 100644 tree-sitter-stack-graphs/src/lua.rs create mode 100644 tree-sitter-stack-graphs/tests/it/lua.rs diff --git a/stack-graphs/src/lua.rs b/stack-graphs/src/lua.rs index 67a5201e7..0e9cce12b 100644 --- a/stack-graphs/src/lua.rs +++ b/stack-graphs/src/lua.rs @@ -31,7 +31,7 @@ //! //! let mut graph = StackGraph::new(); //! lua.scope(|scope| { -//! let graph = scope.create_userdata_ref_mut(&mut graph); +//! let graph = graph.lua_ref_mut(&scope)?; //! process_graph.call(graph) //! })?; //! assert_eq!(graph.iter_nodes().count(), 3); @@ -401,6 +401,8 @@ use std::num::NonZeroU32; use controlled_option::ControlledOption; use lsp_positions::Span; use mlua::AnyUserData; +use mlua::Lua; +use mlua::Scope; use mlua::UserData; use mlua::UserDataMethods; @@ -410,6 +412,35 @@ use crate::graph::File; use crate::graph::Node; use crate::graph::StackGraph; +impl StackGraph { + // Returns a Lua wrapper for this stack graph. Takes ownership of the stack graph. If you + // want to access the stack graph after your Lua code is done with it, use [`lua_ref_mut`] + // instead. + pub fn lua_value<'lua>(self, lua: &'lua Lua) -> Result, mlua::Error> { + lua.create_userdata(self) + } + + // Returns a scoped Lua wrapper for this stack graph. + pub fn lua_ref_mut<'lua, 'scope>( + &'scope mut self, + scope: &Scope<'lua, 'scope>, + ) -> Result, mlua::Error> { + scope.create_userdata_ref_mut(self) + } + + // Returns a scoped Lua wrapper for a file in this stack graph. + pub fn file_lua_ref_mut<'lua, 'scope>( + &'scope mut self, + file: Handle, + scope: &Scope<'lua, 'scope>, + ) -> Result, mlua::Error> { + let graph_ud = self.lua_ref_mut(scope)?; + let file_ud = scope.create_userdata(file)?; + file_ud.set_user_value(graph_ud)?; + Ok(file_ud) + } +} + impl UserData for StackGraph { fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { methods.add_function("edges", |l, graph_ud: AnyUserData| { diff --git a/stack-graphs/tests/it/lua.rs b/stack-graphs/tests/it/lua.rs index 02c7c3de3..fe4e96d7e 100644 --- a/stack-graphs/tests/it/lua.rs +++ b/stack-graphs/tests/it/lua.rs @@ -16,7 +16,7 @@ trait CheckLua { impl CheckLua for mlua::Lua { fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error> { self.scope(|scope| { - let graph = scope.create_userdata_ref_mut(graph); + let graph = graph.lua_ref_mut(&scope)?; self.load(chunk).set_name("test chunk").call(graph) }) } diff --git a/tree-sitter-stack-graphs/Cargo.toml b/tree-sitter-stack-graphs/Cargo.toml index 2de26a86f..1302d597b 100644 --- a/tree-sitter-stack-graphs/Cargo.toml +++ b/tree-sitter-stack-graphs/Cargo.toml @@ -47,6 +47,11 @@ lsp = [ "tokio", "tower-lsp", ] +lua = [ + "dep:mlua", + "dep:mlua-tree-sitter", + "stack-graphs/lua", +] [dependencies] anyhow = "1.0" @@ -63,6 +68,8 @@ indoc = { version = "1.0", optional = true } itertools = "0.10" log = "0.4" lsp-positions = { version="0.3", path="../lsp-positions", features=["tree-sitter"] } +mlua = { version = "0.9", optional = true } +mlua-tree-sitter = { version = "0.1", git="https://github.com/dcreager/mlua-tree-sitter", optional = true } once_cell = "1" pathdiff = { version = "0.2.1", optional = true } regex = "1" @@ -81,5 +88,6 @@ tree-sitter-loader = "0.20" walkdir = { version = "2.3", optional = true } [dev-dependencies] +lua-helpers = { path = "../lua-helpers" } pretty_assertions = "0.7" tree-sitter-python = "0.19.1" diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index 86651b6be..99ebf660a 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -357,6 +357,7 @@ use std::time::Duration; use std::time::Instant; use thiserror::Error; use tree_sitter::Parser; +use tree_sitter::Tree; use tree_sitter_graph::functions::Functions; use tree_sitter_graph::graph::Edge; use tree_sitter_graph::graph::Graph; @@ -375,6 +376,8 @@ pub mod ci; pub mod cli; pub mod functions; pub mod loader; +#[cfg(feature = "lua")] +pub mod lua; pub mod test; mod util; @@ -578,6 +581,29 @@ impl StackGraphLanguage { } } +pub(crate) fn parse_file( + language: tree_sitter::Language, + source: &str, + cancellation_flag: &dyn CancellationFlag, +) -> Result { + let tree = { + let mut parser = Parser::new(); + parser.set_language(language)?; + let ts_cancellation_flag = TreeSitterCancellationFlag::from(cancellation_flag); + // The parser.set_cancellation_flag` is unsafe, because it does not tie the + // lifetime of the parser to the lifetime of the cancellation flag in any way. + // To make it more obvious that the parser does not outlive the cancellation flag, + // it is put into its own block here, instead of extending to the end of the method. + unsafe { parser.set_cancellation_flag(Some(ts_cancellation_flag.as_ref())) }; + parser.parse(source, None).ok_or(BuildError::ParseError)? + }; + let parse_errors = ParseError::into_all(tree); + if parse_errors.errors().len() > 0 { + return Err(BuildError::ParseErrors(parse_errors)); + } + Ok(parse_errors.into_tree()) +} + pub struct Builder<'a> { sgl: &'a StackGraphLanguage, stack_graph: &'a mut StackGraph, @@ -615,24 +641,7 @@ impl<'a> Builder<'a> { globals: &'a Variables<'a>, cancellation_flag: &dyn CancellationFlag, ) -> Result<(), BuildError> { - let tree = { - let mut parser = Parser::new(); - parser.set_language(self.sgl.language)?; - let ts_cancellation_flag = TreeSitterCancellationFlag::from(cancellation_flag); - // The parser.set_cancellation_flag` is unsafe, because it does not tie the - // lifetime of the parser to the lifetime of the cancellation flag in any way. - // To make it more obvious that the parser does not outlive the cancellation flag, - // it is put into its own block here, instead of extending to the end of the method. - unsafe { parser.set_cancellation_flag(Some(ts_cancellation_flag.as_ref())) }; - parser - .parse(self.source, None) - .ok_or(BuildError::ParseError)? - }; - let parse_errors = ParseError::into_all(tree); - if parse_errors.errors().len() > 0 { - return Err(BuildError::ParseErrors(parse_errors)); - } - let tree = parse_errors.into_tree(); + let tree = parse_file(self.sgl.language, self.source, cancellation_flag)?; let mut globals = Variables::nested(globals); if globals.get(&ROOT_NODE_VAR.into()).is_none() { @@ -826,6 +835,9 @@ pub enum BuildError { LanguageError(#[from] tree_sitter::LanguageError), #[error("Expected exported symbol scope in {0}, got {1}")] SymbolScopeError(String, String), + #[cfg(feature = "lua")] + #[error(transparent)] + LuaError(#[from] mlua::Error), } impl From for BuildError { diff --git a/tree-sitter-stack-graphs/src/lua.rs b/tree-sitter-stack-graphs/src/lua.rs new file mode 100644 index 000000000..a0e9ace9c --- /dev/null +++ b/tree-sitter-stack-graphs/src/lua.rs @@ -0,0 +1,103 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +//! Construct stack graphs using a Lua script that consumes a tree-sitter parse tree + +use std::borrow::Cow; + +use mlua::Lua; +use mlua_tree_sitter::Module; +use mlua_tree_sitter::WithSource; +use stack_graphs::arena::Handle; +use stack_graphs::graph::File; +use stack_graphs::graph::StackGraph; + +use crate::parse_file; +use crate::BuildError; +use crate::CancellationFlag; + +/// Holds information about how to construct stack graphs for a particular language. +pub struct StackGraphLanguageLua { + language: tree_sitter::Language, + lua_source: Cow<'static, [u8]>, + lua_source_name: String, +} + +impl StackGraphLanguageLua { + /// Creates a new stack graph language for the given language, loading the Lua stack graph + /// construction rules from a static string. + pub fn from_static_str( + language: tree_sitter::Language, + lua_source: &'static [u8], + lua_source_name: &str, + ) -> StackGraphLanguageLua { + StackGraphLanguageLua { + language, + lua_source: Cow::from(lua_source), + lua_source_name: lua_source_name.to_string(), + } + } + + /// Creates a new stack graph language for the given language, loading the Lua stack graph + /// construction rules from a string. + pub fn from_str( + language: tree_sitter::Language, + lua_source: &[u8], + lua_source_name: &str, + ) -> StackGraphLanguageLua { + StackGraphLanguageLua { + language, + lua_source: Cow::from(lua_source.to_vec()), + lua_source_name: lua_source_name.to_string(), + } + } + + pub fn language(&self) -> tree_sitter::Language { + self.language + } + + pub fn lua_source_name(&self) -> &str { + &self.lua_source_name + } + + pub fn lua_source(&self) -> &Cow<'static, [u8]> { + &self.lua_source + } + + /// Executes the graph construction rules for this language against a source file, creating new + /// nodes and edges in `stack_graph`. Any new nodes that we create will belong to `file`. + /// (The source file must be implemented in this language, otherwise you'll probably get a + /// parse error.) + pub fn build_stack_graph_into<'a>( + &'a self, + stack_graph: &'a mut StackGraph, + file: Handle, + source: &'a str, + cancellation_flag: &'a dyn CancellationFlag, + ) -> Result<(), BuildError> { + // Create a Lua environment and load the language's stack graph rules. + // TODO: Sandbox the Lua environment + let lua = Lua::new(); + lua.open_ltreesitter()?; + lua.load(self.lua_source.as_ref()) + .set_name(&self.lua_source_name) + .exec()?; + let process: mlua::Function = lua.globals().get("process")?; + + // Parse the source using the requested grammar. + let tree = parse_file(self.language, source, cancellation_flag)?; + let tree = tree.with_source(source.as_bytes()); + + // Invoke the Lua `process` function with the parsed tree and the stack graph file. + // TODO: Add a debug hook that checks the cancellation flag during execution + lua.scope(|scope| { + let file = stack_graph.file_lua_ref_mut(file, scope)?; + process.call((tree, file)) + })?; + Ok(()) + } +} diff --git a/tree-sitter-stack-graphs/tests/it/lua.rs b/tree-sitter-stack-graphs/tests/it/lua.rs new file mode 100644 index 000000000..dc8d01e4f --- /dev/null +++ b/tree-sitter-stack-graphs/tests/it/lua.rs @@ -0,0 +1,64 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use lua_helpers::new_lua; +use stack_graphs::graph::StackGraph; +use tree_sitter_stack_graphs::lua::StackGraphLanguageLua; +use tree_sitter_stack_graphs::NoCancellation; + +trait CheckLua { + fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error>; +} + +impl CheckLua for mlua::Lua { + fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error> { + self.scope(|scope| { + let graph = graph.lua_ref_mut(&scope)?; + self.load(chunk).set_name("test chunk").call(graph) + }) + } +} + +// This doesn't build a very _interesting_ stack graph, but it does test that the end-to-end +// spackle all works correctly. +#[test] +fn can_build_stack_graph_from_lua() -> Result<(), anyhow::Error> { + const LUA: &[u8] = br#" + function process(parsed, file) + -- TODO: fill in the definiens span from the parse tree root + local module = file:internal_scope_node() + module:add_edge_from(file:root_node()) + end + "#; + + let code = r#" + def double(x): + return x * 2 + "#; + let mut graph = StackGraph::new(); + let file = graph.get_or_create_file("test.py"); + let language = + StackGraphLanguageLua::from_static_str(tree_sitter_python::language(), LUA, "test"); + language.build_stack_graph_into(&mut graph, file, code, &NoCancellation)?; + + let l = new_lua()?; + l.check( + &mut graph, + r#" + local graph = ... + local file = graph:file("test.py") + assert_deepeq("nodes", { + "[test.py(0) scope]", + }, iter_tostring(file:nodes())) + assert_deepeq("edges", { + "[root] -0-> [test.py(0) scope]", + }, iter_tostring(values(file:edges()))) + "#, + )?; + + Ok(()) +} diff --git a/tree-sitter-stack-graphs/tests/it/main.rs b/tree-sitter-stack-graphs/tests/it/main.rs index c1e37a40e..c4e5d4bc9 100644 --- a/tree-sitter-stack-graphs/tests/it/main.rs +++ b/tree-sitter-stack-graphs/tests/it/main.rs @@ -19,6 +19,9 @@ mod loader; mod nodes; mod test; +#[cfg(feature = "lua")] +mod lua; + pub(self) fn build_stack_graph( python_source: &str, tsg_source: &str, From 0c639ffb1ab9ff3a7f84bb5bb38916422d9e0190 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Mon, 20 Nov 2023 14:48:31 -0500 Subject: [PATCH 09/10] Add Lua bindings for `SpanCalculator` This one is fun, because `SpanCalculator` holds a reference to the file's source code, while the `mlua::UserData` works best for Rust types that are 'static. To get around this, we make sure to only ever create `SpanCalculator` wrappers for source data that is owned by the Lua interpreter, and add that source data as a user value of the Lua wrapper that we create. That should cause Lua's garbage collector to ensure that the source code outlives the `SpanCalculator`, making it safe for us to transmute the source reference to a 'static lifetime. --- lsp-positions/Cargo.toml | 8 +++- lsp-positions/src/lib.rs | 2 +- lsp-positions/src/lua.rs | 73 ++++++++++++++++++++++++++++ lsp-positions/tests/it/lua.rs | 88 ++++++++++++++++++++++++++++++++++ lsp-positions/tests/it/main.rs | 3 ++ 5 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 lsp-positions/tests/it/lua.rs diff --git a/lsp-positions/Cargo.toml b/lsp-positions/Cargo.toml index f3bd926b9..db874de99 100644 --- a/lsp-positions/Cargo.toml +++ b/lsp-positions/Cargo.toml @@ -18,13 +18,19 @@ test = false [features] bincode = ["dep:bincode"] -lua = ["dep:mlua"] +lua = ["tree-sitter", "dep:mlua", "dep:mlua-tree-sitter"] tree-sitter = ["dep:tree-sitter"] [dependencies] memchr = "2.4" mlua = { version = "0.9", optional = true } +mlua-tree-sitter = { version = "0.1", git="https://github.com/dcreager/mlua-tree-sitter", optional = true } tree-sitter = { version=">= 0.19", optional=true } unicode-segmentation = { version="1.8" } serde = { version="1", optional=true, features=["derive"] } bincode = { version="2.0.0-rc.3", optional=true } + +[dev-dependencies] +anyhow = { version = "1.0" } +lua-helpers = { path = "../lua-helpers" } +tree-sitter-python = { version = "0.19.1" } diff --git a/lsp-positions/src/lib.rs b/lsp-positions/src/lib.rs index 91dadd00a..d9fa74199 100644 --- a/lsp-positions/src/lib.rs +++ b/lsp-positions/src/lib.rs @@ -34,7 +34,7 @@ use memchr::memchr; use unicode_segmentation::UnicodeSegmentation as _; #[cfg(feature = "lua")] -mod lua; +pub mod lua; fn grapheme_len(string: &str) -> usize { string.graphemes(true).count() diff --git a/lsp-positions/src/lua.rs b/lsp-positions/src/lua.rs index 047814280..f71e7a3ad 100644 --- a/lsp-positions/src/lua.rs +++ b/lsp-positions/src/lua.rs @@ -11,11 +11,63 @@ use mlua::Error; use mlua::FromLua; use mlua::IntoLua; use mlua::Lua; +use mlua::UserData; +use mlua::UserDataMethods; use mlua::Value; +use mlua_tree_sitter::TSNode; +use mlua_tree_sitter::TreeWithSource; use crate::Offset; use crate::Position; use crate::Span; +use crate::SpanCalculator; + +/// An extension trait that lets you load the `lsp_positions` module into a Lua environment. +pub trait Module { + /// Loads the `lsp_positions` module into a Lua environment. + fn open_lsp_positions(&self) -> Result<(), mlua::Error>; +} + +impl Module for Lua { + fn open_lsp_positions(&self) -> Result<(), mlua::Error> { + let exports = self.create_table()?; + let sc_type = self.create_table()?; + + let function = self.create_function(|lua, source_value: mlua::String| { + // We are going to add the Lua string as a user value of the SpanCalculator's Lua + // wrapper. That will ensure that the string is not garbage collected before the + // SpanCalculator, which makes it safe to transmute into a 'static reference. + let source = source_value.to_str()?; + let source: &'static str = unsafe { std::mem::transmute(source) }; + let sc = SpanCalculator::new(source); + let sc = lua.create_userdata(sc)?; + sc.set_user_value(source_value)?; + Ok(sc) + })?; + sc_type.set("new", function)?; + + #[cfg(feature = "tree-sitter")] + { + let function = self.create_function(|lua, tws_value: Value| { + // We are going to add the tree-sitter treee as a user value of the + // SpanCalculator's Lua wrapper. That will ensure that the tree is not garbage + // collected before the SpanCalculator, which makes it safe to transmute into a + // 'static reference. + let tws = TreeWithSource::from_lua(tws_value.clone(), lua)?; + let source: &'static str = unsafe { std::mem::transmute(tws.src) }; + let sc = SpanCalculator::new(source); + let sc = lua.create_userdata(sc)?; + sc.set_user_value(tws_value)?; + Ok(sc) + })?; + sc_type.set("new_from_tree", function)?; + } + + exports.set("SpanCalculator", sc_type)?; + self.globals().set("lsp_positions", exports)?; + Ok(()) + } +} impl<'lua> FromLua<'lua> for Offset { fn from_lua(value: Value<'lua>, _: &'lua Lua) -> Result { @@ -142,3 +194,24 @@ impl<'lua> IntoLua<'lua> for Span { Ok(Value::Table(result)) } } + +impl UserData for SpanCalculator<'static> { + fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_method_mut( + "for_line_and_column", + |_, sc, (line, line_utf8_offset, column_utf8_offset)| { + Ok(sc.for_line_and_column(line, line_utf8_offset, column_utf8_offset)) + }, + ); + + methods.add_method_mut( + "for_line_and_grapheme", + |_, sc, (line, line_utf8_offset, column_grapheme_offset)| { + Ok(sc.for_line_and_grapheme(line, line_utf8_offset, column_grapheme_offset)) + }, + ); + + #[cfg(feature = "tree-sitter")] + methods.add_method_mut("for_node", |_, sc, node: TSNode| Ok(sc.for_node(&node))); + } +} diff --git a/lsp-positions/tests/it/lua.rs b/lsp-positions/tests/it/lua.rs new file mode 100644 index 000000000..42b7663d5 --- /dev/null +++ b/lsp-positions/tests/it/lua.rs @@ -0,0 +1,88 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use lsp_positions::lua::Module; +use lua_helpers::new_lua; +use lua_helpers::CheckLua; + +#[test] +fn can_calculate_positions_from_lua() -> Result<(), mlua::Error> { + let l = new_lua()?; + l.open_lsp_positions()?; + l.check( + r#" + local source = " from a import * " + local sc = lsp_positions.SpanCalculator.new(source) + local position = sc:for_line_and_column(0, 0, 9) + local expected = { + line=0, + column={ + utf8_offset=9, + utf16_offset=9, + grapheme_offset=9, + }, + containing_line={start=0, ["end"]=21}, + trimmed_line={start=3, ["end"]=18}, + } + assert_deepeq("position", expected, position) + "#, + )?; + Ok(()) +} + +#[cfg(feature = "tree-sitter")] +#[test] +fn can_calculate_tree_sitter_spans_from_lua() -> Result<(), anyhow::Error> { + let code = br#" + def double(x): + return x * 2 + "#; + let mut parser = tree_sitter::Parser::new(); + parser.set_language(tree_sitter_python::language()).unwrap(); + let parsed = parser.parse(code, None).unwrap(); + + use mlua_tree_sitter::Module; + use mlua_tree_sitter::WithSource; + let l = new_lua()?; + l.open_lsp_positions()?; + l.open_ltreesitter()?; + l.globals().set("parsed", parsed.with_source(code))?; + + l.check( + r#" + local module = parsed:root() + local double = module:child(0) + local name = double:child_by_field_name("name") + local sc = lsp_positions.SpanCalculator.new_from_tree(parsed) + local position = sc:for_node(name) + local expected = { + start={ + line=1, + column={ + utf8_offset=10, + utf16_offset=10, + grapheme_offset=10, + }, + containing_line={start=1, ["end"]=21}, + trimmed_line={start=7, ["end"]=21}, + }, + ["end"]={ + line=1, + column={ + utf8_offset=16, + utf16_offset=16, + grapheme_offset=16, + }, + containing_line={start=1, ["end"]=21}, + trimmed_line={start=7, ["end"]=21}, + }, + } + assert_deepeq("position", expected, position) + "#, + )?; + Ok(()) +} diff --git a/lsp-positions/tests/it/main.rs b/lsp-positions/tests/it/main.rs index 44dd4230f..8a3e340e7 100644 --- a/lsp-positions/tests/it/main.rs +++ b/lsp-positions/tests/it/main.rs @@ -9,6 +9,9 @@ use unicode_segmentation::UnicodeSegmentation as _; use lsp_positions::Offset; +#[cfg(feature = "lua")] +mod lua; + fn check_offsets(line: &str) { let offsets = Offset::all_chars(line).collect::>(); assert!(!offsets.is_empty()); From 39ee6863103f6734114fdee162443023cc3ee9bf Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Mon, 20 Nov 2023 16:44:50 -0500 Subject: [PATCH 10/10] Calculate spans in Lua stack graph scripts The stack graph builder now imports the `lsp-position` module before handing control to your Lua script. That lets you create a span calculator, and use that to fill in spans and definiens for the stack graph nodes that you create. --- tree-sitter-stack-graphs/Cargo.toml | 2 ++ tree-sitter-stack-graphs/src/lua.rs | 4 +++- tree-sitter-stack-graphs/tests/it/lua.rs | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tree-sitter-stack-graphs/Cargo.toml b/tree-sitter-stack-graphs/Cargo.toml index 1302d597b..d6e3b911c 100644 --- a/tree-sitter-stack-graphs/Cargo.toml +++ b/tree-sitter-stack-graphs/Cargo.toml @@ -50,6 +50,8 @@ lsp = [ lua = [ "dep:mlua", "dep:mlua-tree-sitter", + "lsp-positions/lua", + "lsp-positions/tree-sitter", "stack-graphs/lua", ] diff --git a/tree-sitter-stack-graphs/src/lua.rs b/tree-sitter-stack-graphs/src/lua.rs index a0e9ace9c..7b0390bf3 100644 --- a/tree-sitter-stack-graphs/src/lua.rs +++ b/tree-sitter-stack-graphs/src/lua.rs @@ -9,8 +9,9 @@ use std::borrow::Cow; +use lsp_positions::lua::Module as _; use mlua::Lua; -use mlua_tree_sitter::Module; +use mlua_tree_sitter::Module as _; use mlua_tree_sitter::WithSource; use stack_graphs::arena::Handle; use stack_graphs::graph::File; @@ -82,6 +83,7 @@ impl StackGraphLanguageLua { // Create a Lua environment and load the language's stack graph rules. // TODO: Sandbox the Lua environment let lua = Lua::new(); + lua.open_lsp_positions()?; lua.open_ltreesitter()?; lua.load(self.lua_source.as_ref()) .set_name(&self.lua_source_name) diff --git a/tree-sitter-stack-graphs/tests/it/lua.rs b/tree-sitter-stack-graphs/tests/it/lua.rs index dc8d01e4f..a4184a042 100644 --- a/tree-sitter-stack-graphs/tests/it/lua.rs +++ b/tree-sitter-stack-graphs/tests/it/lua.rs @@ -29,8 +29,10 @@ impl CheckLua for mlua::Lua { fn can_build_stack_graph_from_lua() -> Result<(), anyhow::Error> { const LUA: &[u8] = br#" function process(parsed, file) - -- TODO: fill in the definiens span from the parse tree root + local sc = lsp_positions.SpanCalculator.new_from_tree(parsed) + local module_ast = parsed:root() local module = file:internal_scope_node() + module:set_definiens_span(sc:for_node(module_ast)) module:add_edge_from(file:root_node()) end "#; @@ -52,7 +54,7 @@ fn can_build_stack_graph_from_lua() -> Result<(), anyhow::Error> { local graph = ... local file = graph:file("test.py") assert_deepeq("nodes", { - "[test.py(0) scope]", + "[test.py(0) scope def 1:6-3:4]", }, iter_tostring(file:nodes())) assert_deepeq("edges", { "[root] -0-> [test.py(0) scope]",