Skip to content

Option to enable or disable the ANTLR-based parser configuration. #507

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ if (NOT WIN32)
endif ()


option(ENABLE_ANTLR "Enable Antlr for path" OFF)

# All libs will be stored here, including libtsfile, compress-encoding lib.
set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)

Expand Down
21 changes: 16 additions & 5 deletions cpp/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ message("Running in src directory")
if (${COV_ENABLED})
add_compile_options(-fprofile-arcs -ftest-coverage)
endif ()
add_definitions(-DANTLR4CPP_STATIC)
set(ANTLR4_WITH_STATIC_CRT OFF)



set(PROJECT_INCLUDE_DIR
Expand All @@ -40,13 +39,21 @@ set(PROJECT_INCLUDE_DIR
${CMAKE_SOURCE_DIR}/third_party/lzokay
${CMAKE_SOURCE_DIR}/third_party/zlib-1.2.13
${CMAKE_SOURCE_DIR}/third_party/google_snappy
${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src

)



include_directories(${PROJECT_INCLUDE_DIR})

add_subdirectory(parser)
if (ENABLE_ANTLR)
add_definitions(-DANTLR4CPP_STATIC)
set(ANTLR4_WITH_STATIC_CRT OFF)
include_directories(${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src)
endif (ENABLE_ANTLR)

add_subdirectory(common)
add_subdirectory(parser)
add_subdirectory(compress)
add_subdirectory(cwrapper)
add_subdirectory(encoding)
Expand All @@ -57,12 +64,16 @@ add_subdirectory(writer)


set(COMPRESSION_LIBS snappy LZ4 lzokay zlibstatic)
target_link_libraries(parser_obj antlr4_static)

target_link_libraries(compress_obj ${COMPRESSION_LIBS})
target_link_libraries(common_obj ${COMPRESSION_LIBS})
target_link_libraries(read_obj ${COMPRESSION_LIBS})
target_link_libraries(write_obj ${COMPRESSION_LIBS})

if (ENABLE_ANTLR)
target_link_libraries(parser_obj antlr4_static)
endif (ENABLE_ANTLR)

add_library(tsfile SHARED)
if (${COV_ENABLED})
message("Enable code cov...")
Expand Down
4 changes: 3 additions & 1 deletion cpp/src/common/path.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@
#include <string>

#include "common/device_id.h"
#include "parser/generated/PathParser.h"
#include "parser/path_nodes_generator.h"
#include "utils/errno_define.h"
#ifdef ENABLE_ANTLR
#include "parser/generated/PathParser.h"
#endif

namespace storage {

Expand Down
3 changes: 3 additions & 0 deletions cpp/src/cwrapper/tsfile_cwrapper.cc
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@

#include "cwrapper/tsfile_cwrapper.h"

#include <set>
#include <string>

#include <file/write_file.h>
#include <reader/qds_without_timegenerator.h>
#include <unistd.h>
Expand Down
6 changes: 5 additions & 1 deletion cpp/src/parser/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@ under the License.
]]
message("Running in src/parser directory")
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
file(GLOB_RECURSE PARSER_SRC_LIST "*.cpp")
if (ENABLE_ANTLR)
file(GLOB_RECURSE PARSER_SRC_LIST "*.cpp")
else()
set(PARSER_SRC_LIST path_nodes_generator.cpp)
endif (ENABLE_ANTLR)
add_library(parser_obj OBJECT ${PARSER_SRC_LIST})

file(GLOB_RECURSE HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h")
Expand Down
61 changes: 40 additions & 21 deletions cpp/src/parser/path_nodes_generator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,30 +18,49 @@
*/
#include <string>
#include <vector>

#include "path_nodes_generator.h"
#include "utils/errno_define.h"
#include "generated/PathLexer.h"
#include "generated/PathParser.h"
#include "path_parser_error.h"
#include "path_visitor.h"

#ifdef ENABLE_ANTLR
#include "utils/errno_define.h"
#include "generated/PathLexer.h"
#include "generated/PathParser.h"
#include "path_parser_error.h"
#include "path_visitor.h"
#else
#endif


namespace storage {
std::vector<std::string> PathNodesGenerator::invokeParser(const std::string& path) {
antlr4::ANTLRInputStream inputStream(path);
PathLexer lexer(&inputStream);
lexer.removeErrorListeners();
lexer.addErrorListener(&PathParseError::getInstance());
antlr4::CommonTokenStream tokens(&lexer);
PathParser parser(&tokens);
parser.removeErrorListeners();
parser.addErrorListener(&PathParseError::getInstance());
parser.getInterpreter<antlr4::atn::ParserATNSimulator>()->setPredictionMode(antlr4::atn::PredictionMode::LL);
/* if use SLL Mode to parse path, it will throw exception
but c++ tsfile forbid throw exception, so we use LL Mode
to parse path.
*/
PathVisitor path_visitor;
return path_visitor.visit(parser.path()).as<std::vector<std::string>>();
#ifdef ENABLE_ANTLR
antlr4::ANTLRInputStream inputStream(path);
PathLexer lexer(&inputStream);
lexer.removeErrorListeners();
lexer.addErrorListener(&PathParseError::getInstance());
antlr4::CommonTokenStream tokens(&lexer);
PathParser parser(&tokens);
parser.removeErrorListeners();
parser.addErrorListener(&PathParseError::getInstance());
parser.getInterpreter<antlr4::atn::ParserATNSimulator>()->setPredictionMode(antlr4::atn::PredictionMode::LL);
/* if use SLL Mode to parse path, it will throw exception
but c++ tsfile forbid throw exception, so we use LL Mode
to parse path.
*/
PathVisitor path_visitor;
return path_visitor.visit(parser.path()).as<std::vector<std::string>>();
#else
std::vector<std::string> result;
size_t start = 0;
size_t end = path.find('.');

while (end != std::string::npos) {
result.emplace_back(path.substr(start, end - start));
start = end + 1;
end = path.find('.', start);
}

result.emplace_back(path.substr(start));
return result;
#endif
}
}
2 changes: 2 additions & 0 deletions cpp/src/reader/column_mapping.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
#ifndef READER_COLUMN_MAPPING_H
#define READER_COLUMN_MAPPING_H

#include <unordered_set>

#include "common/schema.h"
#include "expression.h"
namespace storage
Expand Down
10 changes: 7 additions & 3 deletions cpp/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,14 @@ include_directories(
${CMAKE_SOURCE_DIR}/third_party/google_snappy
${CMAKE_SOURCE_DIR}/third_party/lzokay
${CMAKE_SOURCE_DIR}/third_party/zlib-1.2.13
${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src
)

if (ENABLE_ANTLR)
include_directories(${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src)
add_definitions(-DANTLR4CPP_STATIC)
set(ANTLR4_WITH_STATIC_CRT OFF)
endif (ENABLE_ANTLR)

enable_testing()

file(GLOB_RECURSE TEST_SRCS
Expand All @@ -106,8 +111,7 @@ if (${COV_ENABLED})
add_compile_options(-fprofile-arcs -ftest-coverage)
endif ()

add_definitions(-DANTLR4CPP_STATIC)
set(ANTLR4_WITH_STATIC_CRT OFF)


add_executable(TsFile_Test ${TEST_SRCS})
target_link_libraries(
Expand Down
4 changes: 3 additions & 1 deletion cpp/third_party/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
]]
add_subdirectory(antlr4-cpp-runtime-4)
if (ENABLE_ANTLR)
add_subdirectory(antlr4-cpp-runtime-4)
endif (ENABLE_ANTLR)
add_subdirectory(google_snappy)
add_subdirectory(lz4)
add_subdirectory(lzokay)
Expand Down
Loading
Loading