add cache when generate parser in cmake process

sunxuan01 · Dec 5, 2024 · 443063a · 443063a
1 parent b3984bd
commit 443063a
Show file tree

Hide file tree

Showing 7 changed files with 82 additions and 20 deletions.
diff --git a/.gitignore b/.gitignore
@@ -164,6 +164,7 @@ src/sql/parser/sql_parser_oracle_single_byte_mode_tab.h
 src/sql/parser/non_reserved_keywords_oracle_single_byte_mode.c
 src/sql/parser/_gen_parser.output
 src/sql/parser/_gen_parser.error
+src/sql/parser/_MD5
 src/pl/parser/pl_parser_mysql_mode_lex.c
 src/pl/parser/pl_parser_mysql_mode_lex.h
 src/pl/parser/pl_parser_mysql_mode_tab.c
@@ -174,6 +175,7 @@ src/pl/parser/pl_parser_oracle_mode_tab.c
 src/pl/parser/pl_parser_oracle_mode_tab.h
 src/pl/parser/_gen_parser.error
 src/pl/parser/_gen_pl_parser.output
+src/pl/parser/_MD5
 src/pl/parser/pl_parser_mysql_mode.output
 src/pl/parser/pl_parser_oracle_mode.output
 src/share/inner_table/sys_package/system_package.cpp

diff --git a/cmake/Env.cmake b/cmake/Env.cmake
@@ -27,6 +27,7 @@ ob_define(OB_BUILD_CCLS OFF)
 ob_define(LTO_JOBS all)
 ob_define(LTO_CACHE_DIR "${CMAKE_BINARY_DIR}/cache")
 ob_define(LTO_CACHE_POLICY cache_size=100%:cache_size_bytes=0k:cache_size_files=0:prune_after=0s:prune_interval=72h)
+ob_define(NEED_PARSER_CACHE ON)
 # get compiler from build.sh
 ob_define(OB_CC "")
 ob_define(OB_CXX "")

diff --git a/src/objit/src/ob_llvm_di_helper.cpp b/src/objit/src/ob_llvm_di_helper.cpp
@@ -40,9 +40,9 @@ uint64_t ObLLVMDIType::get_align_bits()
 {
   return OB_ISNULL(v_) ? 0 : v_->getAlignInBits();
 }
-
-int ObLLVMDIHelper::init(core::JitContext *jc)
-{
+
+int ObLLVMDIHelper::init(core::JitContext *jc)
+{
   int ret = OB_SUCCESS;
   if (OB_ISNULL(jc)) {
     ret = OB_ERR_UNEXPECTED;
@@ -155,9 +155,9 @@ int ObLLVMDIHelper::create_local_variable(const ObString &name, uint32_t arg_no,
   ObDISubprogram *sp = NULL;
   ObDILocalVariable *var_ptr;
   if (name.empty() || OB_ISNULL(jc_) || OB_ISNULL(file = jc_->file_)
-      || OB_ISNULL(sp = jc_->sp_) || OB_ISNULL(type.get_v())) {
-    ret = OB_NOT_INIT;
-    LOG_WARN("jc or file or sp or type is NULL", K(name), K(ret));
+      || OB_ISNULL(sp = jc_->sp_) || OB_ISNULL(type.get_v())) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("jc or file or sp or type is NULL", K(name), K(ret));
   } else {
     var_ptr = (arg_no > 0) ?
         jc_->dbuilder_.createParameterVariable(sp, StringRef(name.ptr(), name.length()),
@@ -227,9 +227,9 @@ int ObLLVMDIHelper::create_pointer_type(ObLLVMDIType &pointee_type,
   } else if (OB_ISNULL(ptr_type = jc_->dbuilder_.createPointerType(pte_type, 64, 64))) {
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("failed to create pointer type", K(ret));
-  } else {
-    pointer_type.set_v(ptr_type);
-  }
+  } else {
+    pointer_type.set_v(ptr_type);
+  }
   return ret;
 }
 

diff --git a/src/pl/CMakeLists.txt b/src/pl/CMakeLists.txt
@@ -1,5 +1,5 @@
 execute_process(
-  COMMAND bash gen_parser.sh
+  COMMAND env NEED_PARSER_CACHE=${NEED_PARSER_CACHE} bash gen_parser.sh
   RESULT_VARIABLE RET
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src/pl/parser
   OUTPUT_FILE _gen_pl_parser.output

diff --git a/src/pl/parser/gen_parser.sh b/src/pl/parser/gen_parser.sh
@@ -9,6 +9,8 @@ CURDIR="$(dirname $(readlink -f "$0"))"
 #export PATH=/usr/local/bin:$PATH
 export PATH=$CURDIR/../../../deps/3rd/usr/local/oceanbase/devtools/bin/:$PATH
 export BISON_PKGDATADIR=$CURDIR/../../../deps/3rd/usr/local/oceanbase/devtools/share/bison
+CACHE_MD5_FILE=$CURDIR/_MD5
+TEMP_FILE=$(mktemp)
 
 BISON_VERSION=`bison -V| grep 'bison (GNU Bison)'|awk '{ print  $4;}'`
 NEED_VERSION='2.4.1'
@@ -34,17 +36,43 @@ then
 fi
 }
 
+cat ../../../src/pl/parser/pl_parser_mysql_mode.y >> $TEMP_FILE
+cat ../../../src/pl/parser/pl_parser_mysql_mode.l >> $TEMP_FILE
+if [ -d "../../../close_modules/oracle_pl/pl/parser/" ]; then
+  cat ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.y >> $TEMP_FILE
+  cat ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.l >> $TEMP_FILE
+fi
+
+md5sum_value=$(md5sum "$TEMP_FILE" | awk '{ print $1 }')
+
+function generate_parser {
 # generate pl_parser
-bison_parser ../../../src/pl/parser/pl_parser_mysql_mode.y ../../../src/pl/parser/pl_parser_mysql_mode_tab.c
-flex -o ../../../src/pl/parser/pl_parser_mysql_mode_lex.c ../../../src/pl/parser/pl_parser_mysql_mode.l ../../../src/pl/parser/pl_parser_mysql_mode_tab.h
+  bison_parser ../../../src/pl/parser/pl_parser_mysql_mode.y ../../../src/pl/parser/pl_parser_mysql_mode_tab.c
+  flex -o ../../../src/pl/parser/pl_parser_mysql_mode_lex.c ../../../src/pl/parser/pl_parser_mysql_mode.l ../../../src/pl/parser/pl_parser_mysql_mode_tab.h
 
-# TODO  delete the following line at 9.30
-rm -rf ../../../src/pl/parser/pl_parser_oracle_mode_lex.c ../../../src/pl/parser/pl_parser_oracle_mode_tab.c ../../../src/pl/parser/pl_parser_oracle_mode_tab.h
+  # TODO  delete the following line at 9.30
+  rm -rf ../../../src/pl/parser/pl_parser_oracle_mode_lex.c ../../../src/pl/parser/pl_parser_oracle_mode_tab.c ../../../src/pl/parser/pl_parser_oracle_mode_tab.h
 
-if [ -d "../../../close_modules/oracle_pl/pl/parser/" ]; then
-  bison_parser ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.y ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.c
-  flex -o ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_lex.c ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.l ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.h
-fi
+  if [ -d "../../../close_modules/oracle_pl/pl/parser/" ]; then
+    bison_parser ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.y ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.c
+    flex -o ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_lex.c ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.l ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.h
+  fi
+
+  #./gen_type_name.sh ob_item_type.h >type_name.c
+  echo "$md5sum_value" > $CACHE_MD5_FILE
+}
 
-#./gen_type_name.sh ob_item_type.h >type_name.c
+if [[ -n "$NEED_PARSER_CACHE" && "$NEED_PARSER_CACHE" == "ON" ]]; then
+    echo "generate pl parser with cache"
+    origin_md5sum_value=$(<$CACHE_MD5_FILE)
+    if [[ "$md5sum_value" == "$origin_md5sum_value" ]]; then
+      echo "hit the md5 cache"
+    else
+      generate_parser
+    fi
+else
+    echo "generate pl parser without cache"
+    generate_parser
+fi
 
+rm -rf $TEMP_FILE
diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt
@@ -1536,7 +1536,7 @@ target_link_libraries(ob_sql_static
 endif()
 
 execute_process(
-  COMMAND bash gen_parser.sh
+  COMMAND env NEED_PARSER_CACHE=${NEED_PARSER_CACHE} bash gen_parser.sh
   RESULT_VARIABLE RET
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src/sql/parser
   OUTPUT_FILE _gen_parser.output

diff --git a/src/sql/parser/gen_parser.sh b/src/sql/parser/gen_parser.sh
@@ -9,6 +9,9 @@ CURDIR="$(dirname $(readlink -f "$0"))"
 #export PATH=/usr/local/bin:$PATH
 export PATH=$CURDIR/../../../deps/3rd/usr/local/oceanbase/devtools/bin/:$PATH
 export BISON_PKGDATADIR=$CURDIR/../../../deps/3rd/usr/local/oceanbase/devtools/share/bison
+CACHE_MD5_FILE=$CURDIR/_MD5
+TEMP_FILE=$(mktemp)
+
 BISON_VERSION=`bison -V| grep 'bison (GNU Bison)'|awk '{ print  $4;}'`
 NEED_VERSION='2.4.1'
 
@@ -17,6 +20,15 @@ if [ "$BISON_VERSION" != "$NEED_VERSION" ]; then
   exit 1
 fi
 
+cat ../../../src/sql/parser/sql_parser_mysql_mode.y >> $TEMP_FILE
+cat ../../../src/sql/parser/sql_parser_mysql_mode.l >> $TEMP_FILE
+if [ -d "../../../close_modules/oracle_pl/pl/parser/" ]; then
+  cat ../../../close_modules/oracle_parser/sql/parser/sql_parser_oracle_mode.y >> $TEMP_FILE
+  cat ../../../close_modules/oracle_parser/sql/parser/sql_parser_oracle_mode.l >> $TEMP_FILE
+fi
+
+md5sum_value=$(md5sum "$TEMP_FILE" | awk '{ print $1 }')
+
 bison_parser() {
 BISON_OUTPUT="$(bison -v -Werror -d $1 -o $2 2>&1)"
 BISON_RETURN="$?"
@@ -33,6 +45,7 @@ then
 fi
 }
 
+function generate_parser {
 # generate mysql sql_parser
 bison_parser ../../../src/sql/parser/sql_parser_mysql_mode.y ../../../src/sql/parser/sql_parser_mysql_mode_tab.c
 flex -Cfa -B -8 -o ../../../src/sql/parser/sql_parser_mysql_mode_lex.c ../../../src/sql/parser/sql_parser_mysql_mode.l ../../../src/sql/parser/sql_parser_mysql_mode_tab.h
@@ -264,3 +277,21 @@ fi
 
 # generate type name
 ./gen_type_name.sh ../../../src/objit/include/objit/common/ob_item_type.h > type_name.c
+
+echo "$md5sum_value" > $CACHE_MD5_FILE
+}
+
+if [[ -n "$NEED_PARSER_CACHE" && "$NEED_PARSER_CACHE" == "ON" ]]; then
+    echo "generate pl parser with cache"
+    origin_md5sum_value=$(<$CACHE_MD5_FILE)
+    if [[ "$md5sum_value" == "$origin_md5sum_value" ]]; then
+      echo "hit the md5 cache"
+    else
+      generate_parser
+    fi
+else
+    echo "generate pl parser without cache"
+    generate_parser
+fi
+
+rm -rf $TEMP_FILE