Skip to content

Commit f87929d

Browse files
vbaruavbarua
authored
feat(isthmus): additional parsing and unparsing utils for SQL (#430)
The SubstraitCreateStatementParser can be used to turn one or more SQL CREATE statements into a CalciteCatalogReader It is based on internal logic that was used to convert CREATE statements in methods like SqlToSubstrait#execute(String sql, List tables) into catalogs for processing This class has been introduced to allow users to access this logic, so that they can provide Isthmus with catalogs directly, enabling the removal of methods that performed extra processing on user input to produce catalogs The generation of catalogs is a Calcite concern, and as such is considered out of scope of the API for Isthmus. SubstraitCreateStatementParser is provided as a convenience to users. The SubstraitSqlDialect is now provided as a convenience for producing SQL from a Calcite RelNode. It replaces SubstraitToSql#toSql(RelNode root). BREAKING CHANGE: removed SqlToSubstrait#execute(String sql, List tables) BREAKING CHANGE: removed SqlToSubstrait#execute(String sql, String name, Schema schema) BREAKING CHANGE: removed SubstraitToSql#substraitRelToCalciteRel(Rel relRoot, List tables) BREAKING CHANGE: removed SubstraitToSql#toSql(RelNode root)
1 parent b66913a commit f87929d

25 files changed

+345
-402
lines changed

‎isthmus-cli/src/main/java/io/substrait/isthmus/cli/IsthmusEntryPoint.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,14 @@
1313
import io.substrait.isthmus.ImmutableFeatureBoard;
1414
import io.substrait.isthmus.SqlExpressionToSubstrait;
1515
import io.substrait.isthmus.SqlToSubstrait;
16+
import io.substrait.isthmus.sql.SubstraitCreateStatementParser;
1617
import io.substrait.proto.ExtendedExpression;
1718
import io.substrait.proto.Plan;
1819
import java.io.IOException;
1920
import java.util.List;
2021
import java.util.concurrent.Callable;
2122
import org.apache.calcite.avatica.util.Casing;
23+
import org.apache.calcite.prepare.Prepare;
2224
import picocli.CommandLine;
2325

2426
@Command(
@@ -40,7 +42,7 @@ public class IsthmusEntryPoint implements Callable<Integer> {
4042
names = {"-c", "--create"},
4143
description =
4244
"One or multiple create table statements e.g. CREATE TABLE T1(foo int, bar bigint)")
43-
private List<String> createStatements;
45+
private List<String> createStatements = List.of();
4446

4547
@Option(
4648
names = {"--outputformat"},
@@ -90,7 +92,10 @@ public Integer call() throws Exception {
9092
printMessage(extendedExpression);
9193
} else { // by default Isthmus image are parsing SQL Query
9294
SqlToSubstrait converter = new SqlToSubstrait(featureBoard);
93-
Plan plan = converter.execute(sql, createStatements);
95+
Prepare.CatalogReader catalog =
96+
SubstraitCreateStatementParser.processCreateStatementsToCatalog(
97+
createStatements.toArray(String[]::new));
98+
Plan plan = converter.execute(sql, catalog);
9499
printMessage(plan);
95100
}
96101
return 0;

‎isthmus-cli/src/test/java/io/substrait/isthmus/cli/IsthmusEntryPointTest.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,12 @@ void canProcessQuery() {
1414
int statusCode = cli.execute("SELECT 1;");
1515
assertEquals(0, statusCode);
1616
}
17+
18+
@Test
19+
void canProcessQueryWithCreates() {
20+
IsthmusEntryPoint isthmusEntryPoint = new IsthmusEntryPoint();
21+
CommandLine cli = new CommandLine(isthmusEntryPoint);
22+
int statusCode = cli.execute("SELECT * FROM foo", "--create", "CREATE TABLE foo(id INT)");
23+
assertEquals(0, statusCode);
24+
}
1725
}

‎isthmus/src/main/java/io/substrait/isthmus/SqlConverterBase.java

Lines changed: 0 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,21 @@
11
package io.substrait.isthmus;
22

33
import io.substrait.extension.SimpleExtension;
4-
import io.substrait.isthmus.calcite.SubstraitOperatorTable;
5-
import io.substrait.isthmus.calcite.SubstraitTable;
6-
import java.util.ArrayList;
7-
import java.util.List;
84
import org.apache.calcite.config.CalciteConnectionConfig;
95
import org.apache.calcite.config.CalciteConnectionProperty;
10-
import org.apache.calcite.jdbc.CalciteSchema;
116
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
127
import org.apache.calcite.plan.Contexts;
138
import org.apache.calcite.plan.RelOptCluster;
149
import org.apache.calcite.plan.RelOptCostImpl;
1510
import org.apache.calcite.plan.volcano.VolcanoPlanner;
16-
import org.apache.calcite.prepare.CalciteCatalogReader;
1711
import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
1812
import org.apache.calcite.rel.metadata.ProxyingMetadataHandlerProvider;
1913
import org.apache.calcite.rel.metadata.RelMetadataQuery;
20-
import org.apache.calcite.rel.type.RelDataType;
2114
import org.apache.calcite.rel.type.RelDataTypeFactory;
2215
import org.apache.calcite.rex.RexBuilder;
23-
import org.apache.calcite.schema.Schema;
24-
import org.apache.calcite.sql.SqlNode;
25-
import org.apache.calcite.sql.SqlNodeList;
26-
import org.apache.calcite.sql.SqlOperatorTable;
27-
import org.apache.calcite.sql.ddl.SqlColumnDeclaration;
28-
import org.apache.calcite.sql.ddl.SqlCreateTable;
29-
import org.apache.calcite.sql.ddl.SqlKeyConstraint;
30-
import org.apache.calcite.sql.parser.SqlParseException;
3116
import org.apache.calcite.sql.parser.SqlParser;
32-
import org.apache.calcite.sql.parser.SqlParserPos;
3317
import org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl;
3418
import org.apache.calcite.sql.validate.SqlConformanceEnum;
35-
import org.apache.calcite.sql.validate.SqlValidator;
36-
import org.apache.calcite.sql.validate.SqlValidatorCatalogReader;
37-
import org.apache.calcite.sql.validate.SqlValidatorImpl;
3819
import org.apache.calcite.sql2rel.SqlToRelConverter;
3920

4021
class SqlConverterBase {
@@ -71,106 +52,4 @@ protected SqlConverterBase(FeatureBoard features) {
7152

7253
protected static final SimpleExtension.ExtensionCollection EXTENSION_COLLECTION =
7354
SimpleExtension.loadDefaults();
74-
75-
CalciteCatalogReader registerCreateTables(List<String> tables) throws SqlParseException {
76-
CalciteSchema rootSchema = CalciteSchema.createRootSchema(false);
77-
CalciteCatalogReader catalogReader =
78-
new CalciteCatalogReader(rootSchema, List.of(), factory, config);
79-
SqlValidator validator = Validator.create(factory, catalogReader, SqlValidator.Config.DEFAULT);
80-
if (tables != null) {
81-
for (String tableDef : tables) {
82-
List<SubstraitTable> tList = parseCreateTable(factory, validator, tableDef);
83-
for (SubstraitTable t : tList) {
84-
rootSchema.add(t.getName(), t);
85-
}
86-
}
87-
}
88-
return catalogReader;
89-
}
90-
91-
CalciteCatalogReader registerSchema(String name, Schema schema) {
92-
CalciteSchema rootSchema = CalciteSchema.createRootSchema(false);
93-
if (schema != null) {
94-
rootSchema.add(name, schema);
95-
rootSchema = rootSchema.getSubSchema(name, false);
96-
}
97-
return new CalciteCatalogReader(rootSchema, List.of(), factory, config);
98-
}
99-
100-
protected List<SubstraitTable> parseCreateTable(
101-
RelDataTypeFactory factory, SqlValidator validator, String sql) throws SqlParseException {
102-
SqlParser parser = SqlParser.create(sql, parserConfig);
103-
List<SubstraitTable> tableList = new ArrayList<>();
104-
105-
SqlNodeList nodeList = parser.parseStmtList();
106-
for (SqlNode parsed : nodeList) {
107-
if (!(parsed instanceof SqlCreateTable)) {
108-
throw fail("Not a valid CREATE TABLE statement.");
109-
}
110-
111-
SqlCreateTable create = (SqlCreateTable) parsed;
112-
if (create.name.names.size() > 1) {
113-
throw fail("Only simple table names are allowed.", create.name.getParserPosition());
114-
}
115-
116-
if (create.query != null) {
117-
throw fail("CTAS not supported.", create.name.getParserPosition());
118-
}
119-
120-
List<String> names = new ArrayList<>();
121-
List<RelDataType> columnTypes = new ArrayList<>();
122-
123-
for (SqlNode node : create.columnList) {
124-
if (!(node instanceof SqlColumnDeclaration)) {
125-
if (node instanceof SqlKeyConstraint) {
126-
// key constraints declarations, like primary key declaration, are valid and should not
127-
// result in parse exceptions. Ignore the constraint declaration.
128-
continue;
129-
}
130-
131-
throw fail("Unexpected column list construction.", node.getParserPosition());
132-
}
133-
134-
SqlColumnDeclaration col = (SqlColumnDeclaration) node;
135-
if (col.name.names.size() != 1) {
136-
throw fail("Expected simple column names.", col.name.getParserPosition());
137-
}
138-
139-
names.add(col.name.names.get(0));
140-
columnTypes.add(col.dataType.deriveType(validator));
141-
}
142-
143-
tableList.add(
144-
new SubstraitTable(
145-
create.name.names.get(0), factory.createStructType(columnTypes, names)));
146-
}
147-
148-
return tableList;
149-
}
150-
151-
protected static SqlParseException fail(String text, SqlParserPos pos) {
152-
return new SqlParseException(text, pos, null, null, new RuntimeException("fake lineage"));
153-
}
154-
155-
protected static SqlParseException fail(String text) {
156-
return fail(text, SqlParserPos.ZERO);
157-
}
158-
159-
protected static final class Validator extends SqlValidatorImpl {
160-
161-
private Validator(
162-
SqlOperatorTable opTab,
163-
SqlValidatorCatalogReader catalogReader,
164-
RelDataTypeFactory typeFactory,
165-
Config config) {
166-
super(opTab, catalogReader, typeFactory, config);
167-
}
168-
169-
public static Validator create(
170-
RelDataTypeFactory factory,
171-
SqlValidatorCatalogReader validatorCatalog,
172-
SqlValidator.Config config) {
173-
return new Validator(SubstraitOperatorTable.INSTANCE, validatorCatalog, factory, config);
174-
}
175-
}
17655
}

‎isthmus/src/main/java/io/substrait/isthmus/SqlExpressionToSubstrait.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
import io.substrait.isthmus.calcite.SubstraitTable;
88
import io.substrait.isthmus.expression.RexExpressionConverter;
99
import io.substrait.isthmus.expression.ScalarFunctionConverter;
10+
import io.substrait.isthmus.sql.SubstraitCreateStatementParser;
11+
import io.substrait.isthmus.sql.SubstraitSqlValidator;
1012
import io.substrait.type.NamedStruct;
1113
import io.substrait.type.Type;
1214
import java.util.ArrayList;
@@ -140,10 +142,10 @@ private Result registerCreateTablesForExtendedExpression(List<String> tables)
140142
CalciteSchema rootSchema = CalciteSchema.createRootSchema(false);
141143
CalciteCatalogReader catalogReader =
142144
new CalciteCatalogReader(rootSchema, List.of(), factory, config);
143-
SqlValidator validator = Validator.create(factory, catalogReader, SqlValidator.Config.DEFAULT);
144145
if (tables != null) {
145146
for (String tableDef : tables) {
146-
List<SubstraitTable> tList = parseCreateTable(factory, validator, tableDef);
147+
List<SubstraitTable> tList =
148+
SubstraitCreateStatementParser.processCreateStatements(tableDef);
147149
for (SubstraitTable t : tList) {
148150
rootSchema.add(t.getName(), t);
149151
for (RelDataTypeField field : t.getRowType(factory).getFieldList()) {
@@ -165,6 +167,7 @@ private Result registerCreateTablesForExtendedExpression(List<String> tables)
165167
}
166168
}
167169
}
170+
SqlValidator validator = new SubstraitSqlValidator(catalogReader);
168171
return new Result(validator, catalogReader, nameToTypeMap, nameToNodeMap);
169172
}
170173

‎isthmus/src/main/java/io/substrait/isthmus/SqlToSubstrait.java

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
11
package io.substrait.isthmus;
22

33
import com.google.common.annotations.VisibleForTesting;
4+
import io.substrait.isthmus.sql.SubstraitSqlValidator;
45
import io.substrait.plan.Plan.Version;
56
import io.substrait.plan.PlanProtoConverter;
67
import io.substrait.proto.Plan;
78
import java.util.List;
89
import org.apache.calcite.plan.hep.HepPlanner;
910
import org.apache.calcite.plan.hep.HepProgram;
10-
import org.apache.calcite.prepare.CalciteCatalogReader;
1111
import org.apache.calcite.prepare.Prepare;
1212
import org.apache.calcite.rel.RelRoot;
13-
import org.apache.calcite.schema.Schema;
1413
import org.apache.calcite.sql.SqlNode;
1514
import org.apache.calcite.sql.parser.SqlParseException;
1615
import org.apache.calcite.sql.parser.SqlParser;
@@ -29,27 +28,15 @@ public SqlToSubstrait(FeatureBoard features) {
2928
super(features);
3029
}
3130

32-
public Plan execute(String sql, List<String> tables) throws SqlParseException {
33-
CalciteCatalogReader catalogReader = registerCreateTables(tables);
34-
SqlValidator validator = Validator.create(factory, catalogReader, SqlValidator.Config.DEFAULT);
35-
return executeInner(sql, validator, catalogReader);
36-
}
37-
38-
public Plan execute(String sql, String name, Schema schema) throws SqlParseException {
39-
CalciteCatalogReader catalogReader = registerSchema(name, schema);
40-
SqlValidator validator = Validator.create(factory, catalogReader, SqlValidator.Config.DEFAULT);
41-
return executeInner(sql, validator, catalogReader);
42-
}
43-
4431
public Plan execute(String sql, Prepare.CatalogReader catalogReader) throws SqlParseException {
45-
SqlValidator validator = Validator.create(factory, catalogReader, SqlValidator.Config.DEFAULT);
32+
SqlValidator validator = new SubstraitSqlValidator(catalogReader);
4633
return executeInner(sql, validator, catalogReader);
4734
}
4835

4936
// Package protected for testing
50-
List<RelRoot> sqlToRelNode(String sql, List<String> tables) throws SqlParseException {
51-
Prepare.CatalogReader catalogReader = registerCreateTables(tables);
52-
SqlValidator validator = Validator.create(factory, catalogReader, SqlValidator.Config.DEFAULT);
37+
List<RelRoot> sqlToRelNode(String sql, Prepare.CatalogReader catalogReader)
38+
throws SqlParseException {
39+
SqlValidator validator = new SubstraitSqlValidator(catalogReader);
5340
return sqlToRelNode(sql, validator, catalogReader);
5441
}
5542

‎isthmus/src/main/java/io/substrait/isthmus/SubstraitToSql.java

Lines changed: 0 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,16 @@
11
package io.substrait.isthmus;
22

33
import io.substrait.relation.Rel;
4-
import java.util.List;
5-
import java.util.function.UnaryOperator;
6-
import org.apache.calcite.prepare.CalciteCatalogReader;
74
import org.apache.calcite.prepare.Prepare;
85
import org.apache.calcite.rel.RelNode;
9-
import org.apache.calcite.rel.rel2sql.RelToSqlConverter;
10-
import org.apache.calcite.sql.SqlDialect;
11-
import org.apache.calcite.sql.SqlNode;
12-
import org.apache.calcite.sql.SqlWriterConfig;
13-
import org.apache.calcite.sql.parser.SqlParseException;
146

157
public class SubstraitToSql extends SqlConverterBase {
168

179
public SubstraitToSql() {
1810
super(FEATURES_DEFAULT);
1911
}
2012

21-
public RelNode substraitRelToCalciteRel(Rel relRoot, List<String> tables)
22-
throws SqlParseException {
23-
CalciteCatalogReader catalogReader = registerCreateTables(tables);
24-
return SubstraitRelNodeConverter.convert(relRoot, relOptCluster, catalogReader, parserConfig);
25-
}
26-
2713
public RelNode substraitRelToCalciteRel(Rel relRoot, Prepare.CatalogReader catalog) {
2814
return SubstraitRelNodeConverter.convert(relRoot, relOptCluster, catalog, parserConfig);
2915
}
30-
31-
// DEFAULT_SQL_DIALECT uses Calcite's EMPTY_CONTEXT with setting:
32-
// identifierQuoteString : null, identifierEscapeQuoteString : null
33-
// quotedCasing : UNCHANGED, unquotedCasing : TO_UPPER
34-
// caseSensitive: true
35-
// supportsApproxCountDistinct is true
36-
private static final SqlDialect DEFAULT_SQL_DIALECT =
37-
new SqlDialect(SqlDialect.EMPTY_CONTEXT) {
38-
@Override
39-
public boolean supportsApproxCountDistinct() {
40-
return true;
41-
}
42-
};
43-
44-
public static String toSql(RelNode root) {
45-
return toSql(root, DEFAULT_SQL_DIALECT);
46-
}
47-
48-
public static String toSql(RelNode root, SqlDialect dialect) {
49-
return toSql(
50-
root,
51-
dialect,
52-
c ->
53-
c.withAlwaysUseParentheses(false)
54-
.withSelectListItemsOnSeparateLines(false)
55-
.withUpdateSetListNewline(false)
56-
.withIndentation(0));
57-
}
58-
59-
private static String toSql(
60-
RelNode root, SqlDialect dialect, UnaryOperator<SqlWriterConfig> transform) {
61-
final RelToSqlConverter converter = new RelToSqlConverter(dialect);
62-
final SqlNode sqlNode = converter.visitRoot(root).asStatement();
63-
return sqlNode.toSqlString(c -> transform.apply(c.withDialect(dialect))).getSql();
64-
}
6516
}

0 commit comments

Comments
 (0)