Fix/get local ip

.github/actions/confWriter/action.yml

@@ -17,6 +17,10 @@ inputs:
         description: "make the storage engine read-only"
         required: false
         default: "false"
+    Has-Data:
+        description: "does the storage engine has data"
+        required: false
+        default: "false"
     Push-Down:
         description: "make the IGinX push down filter"
         required: false
@@ -29,6 +33,10 @@ inputs:
         description: "the path of IGinX root directory"
         required: false
         default: "${GITHUB_WORKSPACE}"
+    zookeeper-port:
+        description: "zookeeper service port"
+        required: false
+        default: "2181"
 
 runs:
     using: "composite" # Mandatory parameter
@@ -74,6 +82,21 @@ runs:
           run: |
               echo "${{ inputs.DB-name }}" > ${{ inputs.Root-Dir-Path }}/test/src/test/resources/DBName.txt
 
+        - if: inputs.zookeeper-port!='2181'
+          name: Change Zookeeper Port
+          shell: bash
+          run: |
+              if [ "$RUNNER_OS" == "Linux" ]; then
+                sudo sed -i 's/zookeeperConnectionString=127.0.0.1:2181/zookeeperConnectionString=127.0.0.1:${{ inputs.zookeeper-port }}/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              elif [ "$RUNNER_OS" == "Windows" ]; then
+                sed -i 's/zookeeperConnectionString=127.0.0.1:2181/zookeeperConnectionString=127.0.0.1:${{ inputs.zookeeper-port }}/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              elif [ "$RUNNER_OS" == "macOS" ]; then
+                sudo sed -i '' 's/zookeeperConnectionString=127.0.0.1:2181/zookeeperConnectionString=127.0.0.1:${{ inputs.zookeeper-port }}/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              else
+                echo "$RUNNER_OS is not supported"
+                exit 1
+              fi
+
         - name: Change UDF conf
           shell: bash
           run: |
@@ -103,6 +126,22 @@ runs:
                 echo "$RUNNER_OS is not supported"
                 exit 1
               fi
+
+        - if: inputs.Has-Data=='true'
+          name: Set Has-Data
+          shell: bash
+          run: |
+              if [ "$RUNNER_OS" == "Linux" ]; then
+                sudo sed -i 's/has_data=false/has_data=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              elif [ "$RUNNER_OS" == "Windows" ]; then
+                sed -i 's/has_data=false/has_data=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              elif [ "$RUNNER_OS" == "macOS" ]; then
+                sudo sed -i '' 's/has_data=false/has_data=true/' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              else
+                echo "$RUNNER_OS is not supported"
+                exit 1
+              fi
+
         - if: inputs.Push-Down=='true'
           name: Change push_down
           shell: bash

.github/actions/dbConfWriter/action.yml

@@ -5,6 +5,10 @@ inputs:
         description: "DB name"
         required: false
         default: IoTDB12
+    DB-port:
+        description: "DB port"
+        required: false
+        default: "0"
     Root-Dir-Path:
         description: "the path of IGinX root directory"
         required: false
@@ -97,3 +101,23 @@ runs:
           with:
               paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
               statements: s|^#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path|storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false#meta_properties_path=${{ steps.mysql-properties.outputs.path }}|g
+
+        # use regex
+        - if: inputs.DB-port!='0'
+          name: Change db port
+          shell: bash
+          run: |
+              if [ "$RUNNER_OS" == "Linux" ]; then
+                sudo sed -i -E "s/^(storageEngineList=127\.0\.0\.1#)[0-9]+(#.*)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+                sudo sed -i -E "s/(mongodb:\/\/127\.0\.0\.1:)[0-9]+(\/)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              elif [ "$RUNNER_OS" == "Windows" ]; then
+                sed -i -E "s/^(storageEngineList=127\.0\.0\.1#)[0-9]+(#.*)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+                sed -i -E "s/(mongodb:\/\/127\.0\.0\.1:)[0-9]+(\/)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              elif [ "$RUNNER_OS" == "macOS" ]; then
+                sudo sed -i '' -E "s/^(storageEngineList=127\.0\.0\.1#)[0-9]+(#.*)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+                sudo sed -i '' -E "s/(mongodb:\/\/127\.0\.0\.1:)[0-9]+(\/)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              else
+                echo "$RUNNER_OS is not supported"
+                exit 1
+              fi
+              cat ${{ inputs.Root-Dir-Path }}/conf/config.properties

.github/actions/service/portMapper/action.yml

@@ -0,0 +1,66 @@
+name: "Database Ports Mapper"
+description: "Map database name to corresponding ports"
+inputs:
+    DB-name:
+        description: "Name of the database to get ports for"
+        required: true
+
+outputs:
+    port1:
+        description: "port1"
+        value: ${{ steps.get-ports.outputs.port1 }}
+    port2:
+        description: "port2"
+        value: ${{ steps.get-ports.outputs.port2 }}
+    port3:
+        description: "port3"
+        value: ${{ steps.get-ports.outputs.port3 }}
+
+runs:
+    using: "composite"
+    steps:
+        - id: get-ports
+          shell: bash
+          run: |
+              case "${{ inputs.DB-name }}" in
+                "InfluxDB")
+                  echo "port1=8086" >> $GITHUB_OUTPUT
+                  echo "port2=8087" >> $GITHUB_OUTPUT
+                  echo "port3=8088" >> $GITHUB_OUTPUT
+                  ;;
+                "IoTDB12")
+                  echo "port1=6667" >> $GITHUB_OUTPUT
+                  echo "port2=6668" >> $GITHUB_OUTPUT
+                  echo "port3=6669" >> $GITHUB_OUTPUT
+                  ;;
+                "FileSystem")
+                  echo "port1=6667" >> $GITHUB_OUTPUT
+                  echo "port2=6668" >> $GITHUB_OUTPUT
+                  echo "port3=6669" >> $GITHUB_OUTPUT
+                  ;;
+                "MySQL")
+                  echo "port1=3306" >> $GITHUB_OUTPUT
+                  echo "port2=3307" >> $GITHUB_OUTPUT
+                  echo "port3=3308" >> $GITHUB_OUTPUT
+                  ;;
+                "PostgreSQL")
+                  echo "port1=5432" >> $GITHUB_OUTPUT
+                  echo "port2=5433" >> $GITHUB_OUTPUT
+                  echo "port3=5434" >> $GITHUB_OUTPUT
+                  ;;
+                "MongoDB")
+                  echo "port1=27017" >> $GITHUB_OUTPUT
+                  echo "port2=27018" >> $GITHUB_OUTPUT
+                  echo "port3=27019" >> $GITHUB_OUTPUT
+                  ;;
+                "Redis")
+                  echo "port1=6379" >> $GITHUB_OUTPUT
+                  echo "port2=6380" >> $GITHUB_OUTPUT
+                  echo "port3=6381" >> $GITHUB_OUTPUT
+                  ;;
+                *)
+                  echo "port1=0" >> $GITHUB_OUTPUT
+                  echo "port2=0" >> $GITHUB_OUTPUT
+                  echo "port3=0" >> $GITHUB_OUTPUT
+                  ;;
+              esac

.github/actions/service/zookeeper/action.yml

@@ -10,30 +10,35 @@ inputs:
     start:
         description: "whether to start"
         required: true
+    port:
+        description: "zk port"
+        required: false
+        default: "2181"
 
 runs:
     using: "composite"
     steps:
         - if: fromJSON(inputs.stop)
           name: Stop zookeeper
           shell: bash
-          working-directory: ${{ github.action_path }}
-          run: zkServer.sh --config conf stop
+          working-directory: ${{ runner.temp }}/zookeeper-${{ inputs.port }}
+          run: bin/zkServer.sh --config conf stop
 
         - if: fromJSON(inputs.clean)
           name: Clean zookeeper
           shell: bash
-          working-directory: ${{ github.action_path }}
+          working-directory: ${{ runner.temp }}/zookeeper-${{ inputs.port }}
           run: rm -rf data logs conf
 
         - if: fromJSON(inputs.start)
           name: Start zookeeper
           shell: bash
-          working-directory: ${{ github.action_path }}
+          working-directory: ${{ runner.temp }}/zookeeper-${{ inputs.port }}
           run: |
               mkdir -p conf
               echo "dataDir=data" > conf/zoo.cfg
               echo "dataLogDir=logs" >> conf/zoo.cfg
-              echo "clientPort=2181" >> conf/zoo.cfg
+              echo "clientPort=${{ inputs.port }}" >> conf/zoo.cfg
+              echo "admin.serverPort=$((${{ inputs.port }} + 10000))" >> conf/zoo.cfg
 
-              zkServer.sh --config conf start
+              bin/zkServer.sh --config conf start

.github/actions/setup/zookeeper/action.yml

@@ -5,6 +5,10 @@ inputs:
         description: "zookeeper version"
         required: false
         default: "3.7.2"
+    port:
+        description: "zk port"
+        required: false
+        default: "2181"
 
 runs:
     using: "composite"
@@ -32,10 +36,12 @@ runs:
               subdir: "apache-zookeeper-{version}-bin"
               ext: "tar.gz"
 
-        - name: Add zookeeper Bin to PATH
+        - name: Copy ZK to Port Dir
           shell: bash
-          working-directory: ${{ steps.base.outputs.tool-path }}
-          run: echo "$PWD/bin" >> $GITHUB_PATH
+          run: |
+              if [ ! -d "${{ runner.temp }}/zookeeper-${{ inputs.port }}" ]; then
+                cp -r "${{ steps.base.outputs.tool-path }}" "${{ runner.temp }}/zookeeper-${{ inputs.port }}"
+              fi
 
         - name: Save zookeeper Cache
           if: steps.restore.outputs.cache-hit != 'true'

.github/actions/tpchDataWriter/action.yml

@@ -9,10 +9,39 @@ inputs:
 runs:
     using: "composite" # Mandatory parameter
     steps:
+        - name: Start Old IGinX
+          shell: bash
+          run: |
+              cd IGinX/core/target/iginx-core-${VERSION}
+              pwd
+              export IGINX_HOME=$PWD
+              echo "IGinX home path: $IGINX_HOME"
+              cd ..
+              chmod +x iginx-core-${VERSION}/sbin/start_iginx.sh
+              nohup iginx-core-${VERSION}/sbin/start_iginx.sh > ../../iginx-${VERSION}.log 2>&1 &
+
+        # two IGinXs are using different ZKs, we need to insert data & register UDFs in both nodes
+        - name: Insert Data and Register UDF in Old IGinX
+          shell: bash
+          run: |
+              cd IGinX
+              mvn test -q -Dtest=TPCHDataGeneratorIT -DfailIfNoTests=false -P-format
+
+        - name: Show Old IGinX log
+          if: always()
+          shell: bash
+          run: cat IGinX/iginx-*.log
+
+        - name: Stop Old IGinX
+          uses: ./.github/actions/iginxRunner
+          with:
+              version: ${VERSION}
+              if-stop: "true"
+
         - name: Start New IGinX
           uses: ./.github/actions/iginxRunner
 
-        - name: Insert Data and Register UDF
+        - name: Insert Data and Register UDF in new IGinX
           shell: bash
           run: mvn test -q -Dtest=TPCHDataGeneratorIT -DfailIfNoTests=false -P-format
 

.github/actions/zookeeperRunner/action.yml

@@ -13,12 +13,18 @@ inputs:
         description: "to rerun the zookeeper"
         required: false
         default: "false"
+    port:
+        description: "zk port"
+        required: false
+        default: "2181"
 runs:
     using: "composite" # Mandatory parameter
     steps:
         - if: inputs.if-stop=='false' && inputs.if-rerun=='false'
           name: First Run ZooKeeper | Setup
           uses: ./.github/actions/setup/zookeeper
+          with:
+              port: ${{ inputs.port }}
 
         - if: inputs.if-stop=='false' && inputs.if-rerun=='false'
           name: First Run ZooKeeper
@@ -27,6 +33,7 @@ runs:
               stop: false
               clean: false
               start: true
+              port: ${{ inputs.port }}
 
         - if: inputs.if-rerun=='true'
           name: ReRun ZooKeeper
@@ -35,6 +42,7 @@ runs:
               stop: false
               clean: false
               start: true
+              port: ${{ inputs.port }}
 
         - if: inputs.if-stop=='true'
           name: Stop ZooKeeper
@@ -43,3 +51,4 @@ runs:
               stop: true
               clean: true
               start: false
+              port: ${{ inputs.port }}

.github/scripts/test/cli/test_infile.sh

@@ -32,6 +32,6 @@ bash -c "cat 'test/src/test/resources/fileReadAndWrite/csv/test.csv' > 'test/src
 
 bash -c "sed -i '1ikey,d m,b,[c],a' 'test/src/test/resources/fileReadAndWrite/csv/test1'"
 
-COMMAND1='LOAD DATA FROM INFILE "'"test/src/test/resources/fileReadAndWrite/csv/test1"'" AS CSV INTO t1;'
+COMMAND1='LOAD DATA FROM INFILE "'"test/src/test/resources/fileReadAndWrite/csv/test1"'" AS CSV INTO t1 AT 10;'
 
 bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}"

.github/scripts/test/cli/test_infile_macos.sh

@@ -28,4 +28,4 @@ sh -c "cat test/src/test/resources/fileReadAndWrite/csv/test.csv > test/src/test
 
 sh -c "sed -i '' '1s/^/key,d m,b,[c],a\n/' test/src/test/resources/fileReadAndWrite/csv/test1"
 
-sh -c "echo 'LOAD DATA FROM INFILE "'"test/src/test/resources/fileReadAndWrite/csv/test1"'" AS CSV INTO t1;' | xargs -0 -t -I F sh client/target/iginx-client-$1/sbin/start_cli.sh -e 'F'"
+sh -c "echo 'LOAD DATA FROM INFILE "'"test/src/test/resources/fileReadAndWrite/csv/test1"'" AS CSV INTO t1 AT 10;' | xargs -0 -t -I F sh client/target/iginx-client-$1/sbin/start_cli.sh -e 'F'"

.github/workflows/tpc-h-pushdown.yml

@@ -75,16 +75,13 @@ jobs:
                     systeminfo | findstr /C:"Total Physical Memory"
                   fi
 
-            - name: Download TPC-H Data
-              shell: bash
-              run: |
-                  wget https://github.com/IGinX-THU/IGinX-resources/raw/main/resources/tpc.7z
-                  sudo apt-get install p7zip-full
-                  7za x tpc.7z
-                  ls tpc
+            - name: Run ZooKeeper1
+              uses: ./.github/actions/zookeeperRunner
 
-            - name: Run ZooKeeper
+            - name: Run ZooKeeper2
               uses: ./.github/actions/zookeeperRunner
+              with:
+                  port: 2182
 
             - name: Run DB
               uses: ./.github/actions/dbRunner
@@ -101,18 +98,35 @@ jobs:
                   cp -f test/src/test/resources/testConfig.properties IGinX/test/src/test/resources/testConfig.properties
             # 新增tpc-h查询测试时，旧版本代码中没有这些查询，需从新分支复制过去，添加完所有的tpc_h测试后可删除这些复制命令
 
+            - name: Download TPC-H Data
+              shell: bash
+              run: |
+                  wget https://github.com/IGinX-THU/IGinX-resources/raw/main/resources/tpc.7z
+                  sudo apt-get install p7zip-full
+                  7za x tpc.7z
+                  ls tpc
+                  cp -r tpc IGinX/tpc
+                  ls IGinX/tpc
+
+            # let old iginx use second database service
+            - name: Find second port for DB
+              id: find-port
+              uses: ./.github/actions/service/portMapper
+              with:
+                  DB-name: ${{ matrix.DB-name }}
+
             - name: Rewrite DB Conf in Old IGinX
               uses: ./.github/actions/dbConfWriter
               with:
                   DB-name: ${{ matrix.DB-name }}
                   Root-Dir-Path: "IGinX"
+                  DB-port: ${{ steps.find-port.outputs.port2 }}
 
             - name: Install Old IGinX with Maven
               shell: bash
               run: |
                   cd IGinX
                   mvn clean package -DskipTests -P-format -q
-                  cp -rf ${GITHUB_WORKSPACE}/test/src/test/resources/tpch/udf/* core/target/iginx-core-${VERSION}/udf_funcs/python_scripts/
 
             - name: Change Old IGinX Config
               uses: ./.github/actions/confWriter
@@ -122,6 +136,7 @@ jobs:
                   Set-Filter-Fragment-OFF: "true"
                   Metadata: ${{ matrix.metadata }}
                   Root-Dir-Path: "IGinX"
+                  zookeeper-port: "2182"
 
             - name: Install New IGinX with Maven
               shell: bash