[test][enhance][refactor] support suite block to specify multiple group, suppo… (#8792)

support suite block to specify multiple groups. TestAction support compare result to iterator, local file and http stream. support print teamcity service message. abandon the logical: generate groovy file for sql file support 3 levels parrallel: script file, suite block, thread action support specify JAVA_OPTS for boot shell avoid jvm metaspace oom use -d to run the suite in some directories, instead of -g. and -g is used to specify groups
2026-11-04 07:31:17 +00:00 · 2022-04-01 20:59:01 +08:00
1 parent 9f80f6cf5e
commit decdc8e8b9
@@ -102,10 +102,12 @@ suitePath = "${DORIS_HOME}/regression-test/suites"
 dataPath = "${DORIS_HOME}/regression-test/data"

 // 默认会读所有的组,读多个组可以用半角逗号隔开，如: "demo,performance"
-// 一般不需要在配置文件中修改，而是通过run-regression-test.sh来动态指定和覆盖
+// 一般不需要在配置文件中修改，而是通过run-regression-test.sh --run -g来动态指定和覆盖
 testGroups = ""
-// 默认会读所有的用例, 同样可以使用run-regression-test.sh来动态指定和覆盖
+// 默认会读所有的用例, 同样可以使用run-regression-test.sh --run -s来动态指定和覆盖
 testSuites = ""
+// 默认会加载的用例目录, 可以通过run-regression-test.sh --run -d来动态指定和覆盖
+testDirectories = ""

 // 其他自定义配置
 customConf1 = "test_custom_conf_value"
@@ -128,30 +130,31 @@ sql action用于提交sql并获取结果，如果查询失败则会抛出异常

 下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/sql_action.groovy`:
 ```groovy
-// execute sql and ignore result
-sql "show databases"
+suite("sql_action", "demo") {
+    // execute sql and ignore result
+    sql "show databases"

-// execute sql and get result, outer List denote rows, inner List denote columns in a single row
-List<List<Object>> tables = sql "show tables"
+    // execute sql and get result, outer List denote rows, inner List denote columns in a single row
+    List<List<Object>> tables = sql "show tables"

-// assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
-assertTrue(tables.size() >= 0) // test rowCount >= 0
+    // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
+    assertTrue(tables.size() >= 0) // test rowCount >= 0

-// syntax error
-try {
-    sql "a b c d e"
-    throw new IllegalStateException("Should be syntax error")
-} catch (java.sql.SQLException t) {
-    assertTrue(true)
-}
+    // syntax error
+    try {
+        sql "a b c d e"
+        throw new IllegalStateException("Should be syntax error")
+    } catch (java.sql.SQLException t) {
+        assertTrue(true)
+    }

-def testTable = "test_sql_action1"
+    def testTable = "test_sql_action1"

-try {
-    sql "DROP TABLE IF EXISTS ${testTable}"
+    try {
+        sql "DROP TABLE IF EXISTS ${testTable}"

-    // multi-line sql
-    def result1 = sql """
+        // multi-line sql
+        def result1 = sql """
                        CREATE TABLE IF NOT EXISTS ${testTable} (
                            id int
                        )
@@ -161,40 +164,40 @@ try {
                        ) 
                        """

-    // DDL/DML return 1 row and 1 column, the only value is update row count
-    assertTrue(result1.size() == 1)
-    assertTrue(result1[0].size() == 1)
-    assertTrue(result1[0][0] == 0, "Create table should update 0 rows")
+        // DDL/DML return 1 row and 1 column, the only value is update row count
+        assertTrue(result1.size() == 1)
+        assertTrue(result1[0].size() == 1)
+        assertTrue(result1[0][0] == 0, "Create table should update 0 rows")

-    def result2 = sql "INSERT INTO test_sql_action1 values(1), (2), (3)"
-    assertTrue(result2.size() == 1)
-    assertTrue(result2[0].size() == 1)
-    assertTrue(result2[0][0] == 3, "Insert should update 3 rows")
-} finally {
-    /**
-     * try_xxx(args) means:
-     *
-     * try {
-     *    return xxx(args)
-     * } catch (Throwable t) {
-     *     // do nothing
-     *     return null
-     * }
-     */
-    try_sql("DROP TABLE IF EXISTS ${testTable}")
+        def result2 = sql "INSERT INTO test_sql_action1 values(1), (2), (3)"
+        assertTrue(result2.size() == 1)
+        assertTrue(result2[0].size() == 1)
+        assertTrue(result2[0][0] == 3, "Insert should update 3 rows")
+    } finally {
+        /**
+         * try_xxx(args) means:
+         *
+         * try {
+         *    return xxx(args)
+         * } catch (Throwable t) {
+         *     // do nothing
+         *     return null
+         * }
+         */
+        try_sql("DROP TABLE IF EXISTS ${testTable}")

-    // you can see the error sql will not throw exception and return
-    try {
-        def errorSqlResult = try_sql("a b c d e f g")
-        assertTrue(errorSqlResult == null)
-    } catch (Throwable t) {
-        assertTrue(false, "Never catch exception")
+        // you can see the error sql will not throw exception and return
+        try {
+            def errorSqlResult = try_sql("a b c d e f g")
+            assertTrue(errorSqlResult == null)
+        } catch (Throwable t) {
+            assertTrue(false, "Never catch exception")
+        }
    }
-}

-// order_sql(sqlStr) equals to sql(sqlStr, isOrder=true)
-// sort result by string dict
-def list = order_sql """
+    // order_sql(sqlStr) equals to sql(sqlStr, isOrder=true)
+    // sort result by string dict
+    def list = order_sql """
                select 2
                union all
                select 1
@@ -205,11 +208,13 @@ def list = order_sql """
                union all
                select 3
                """
-assertEquals(null, list[0][0])
-assertEquals(1, list[1][0])
-assertEquals(15, list[2][0])
-assertEquals(2, list[3][0])
-assertEquals(3, list[4][0])
+
+    assertEquals(null, list[0][0])
+    assertEquals(1, list[1][0])
+    assertEquals(15, list[2][0])
+    assertEquals(2, list[3][0])
+    assertEquals(3, list[4][0])
+}
 ```

 ### qt action
@@ -219,22 +224,23 @@ qt action用于提交sql，并使用对应的.out TSV文件来校验结果

 下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/qt_action.groovy`:
 ```groovy
-/**
- * qt_xxx sql equals to quickTest(xxx, sql) witch xxx is tag.
- * the result will be compare to the relate file: ${DORIS_HOME}/regression_test/data/qt_action.out.
- *
- * if you want to generate .out tsv file for real execute result. you can run with -genOut or -forceGenOut option.
- * e.g
- *   ${DORIS_HOME}/run-regression-test.sh --run qt_action -genOut
- *   ${DORIS_HOME}/run-regression-test.sh --run qt_action -forceGenOut
- */
-qt_select "select 1, 'beijing' union all select 2, 'shanghai'"
+suite("qt_action", "demo") {
+    /**
+     * qt_xxx sql equals to quickTest(xxx, sql) witch xxx is tag.
+     * the result will be compare to the relate file: ${DORIS_HOME}/regression_test/data/qt_action.out.
+     *
+     * if you want to generate .out tsv file for real execute result. you can run with -genOut or -forceGenOut option.
+     * e.g
+     *   ${DORIS_HOME}/run-regression-test.sh --run qt_action -genOut
+     *   ${DORIS_HOME}/run-regression-test.sh --run qt_action -forceGenOut
+     */
+    qt_select "select 1, 'beijing' union all select 2, 'shanghai'"

-qt_select2 "select 2"
+    qt_select2 "select 2"

-// order result by string dict then compare to .out file.
-// order_qt_xxx sql equals to quickTest(xxx, sql, true).
-order_qt_union_all  """
+    // order result by string dict then compare to .out file.
+    // order_qt_xxx sql equals to quickTest(xxx, sql, true).
+    order_qt_union_all  """
                select 2
                union all
                select 1
@@ -245,6 +251,7 @@ order_qt_union_all  """
                union all
                select 3
                """
+}
 ```

 ### test action
@@ -252,7 +259,9 @@ test action可以使用更复杂的校验规则来测试，比如验证行数、

 可用参数
 - String sql: 输入的sql字符串
- List<List<Object>> result: 提供一个List对象，用于校验真实查询结果对比是否相等
+- List<List<Object>> result: 提供一个List对象，用于比较真实查询结果与List对象是否相等
+- Iterator<Object> resultIterator: 提供一个Iterator对象，用于比较真实查询结果与Iterator是否相等
+- String resultFile: 提供一个文件Uri(可以是本地文件相对路径，或http(s)路径)，用于比较真实查询结果与http响应流是否相等，格式与.out文件格式类似，但没有块头和注释
 - String exception: 校验抛出的异常是否包含某些字符串
 - long rowNum: 验证结果行数
 - long time: 验证执行时间是否小于这个值，单位是毫秒
@@ -260,14 +269,15 @@ test action可以使用更复杂的校验规则来测试，比如验证行数、

 下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/qt_action.groovy`:
 ```groovy
-test {
-    sql "abcdefg"
-    // check exception message contains
-    exception "errCode = 2, detailMessage = Syntax error"
-}
+suite("test_action", "demo") {
+    test {
+        sql "abcdefg"
+        // check exception message contains
+        exception "errCode = 2, detailMessage = Syntax error"
+    }

-test {
-    sql """
+    test {
+        sql """
            select *
            from (
                select 1 id
@@ -276,33 +286,33 @@ test {
            ) a
            order by id"""

-    // multi check condition
+        // multi check condition

-    // check return 2 rows
-    rowNum 2
-    // execute time must <= 5000 millisecond
-    time 5000
-    // check result, must be 2 rows and 1 column, the first row is 1, second is 2
-    result(
-        [[1], [2]]
-    )
-}
-
-test {
-    sql "a b c d e f g"
-
-    // other check will not work because already declared a check callback
-    exception "aaaaaaaaa"
-
-    // callback
-    check { result, exception, startTime, endTime ->
-        // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
-        assertTrue(exception != null)
+        // check return 2 rows
+        rowNum 2
+        // execute time must <= 5000 millisecond
+        time 5000
+        // check result, must be 2 rows and 1 column, the first row is 1, second is 2
+        result(
+            [[1], [2]]
+        )
    }
-}

-test {
-    sql  """
+    test {
+        sql "a b c d e f g"
+
+        // other check will not work because already declared a check callback
+        exception "aaaaaaaaa"
+
+        // callback
+        check { result, exception, startTime, endTime ->
+            // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
+            assertTrue(exception != null)
+        }
+    }
+
+    test {
+        sql  """
                select 2
                union all
                select 1
@@ -314,15 +324,34 @@ test {
                select 3
                """

-    check { result, ex, startTime, endTime ->
-        // same as order_sql(sqlStr)
-        result = sortRows(result)
+        check { result, ex, startTime, endTime ->
+            // same as order_sql(sqlStr)
+            result = sortRows(result)

-        assertEquals(null, result[0][0])
-        assertEquals(1, result[1][0])
-        assertEquals(15, result[2][0])
-        assertEquals(2, result[3][0])
-        assertEquals(3, result[4][0])
+            assertEquals(null, result[0][0])
+            assertEquals(1, result[1][0])
+            assertEquals(15, result[2][0])
+            assertEquals(2, result[3][0])
+            assertEquals(3, result[4][0])
+        }
+    }
+
+    // execute sql and order query result, then compare to iterator
+    def selectValues = [1, 2, 3, 4]
+    test {
+        order true
+        sql selectUnionAll(selectValues)
+        resultIterator(selectValues.iterator())
+    }
+
+    // compare to data/demo/test_action.csv
+    test {
+        order true
+        sql selectUnionAll(selectValues)
+
+        // you can set to http://xxx or https://xxx
+        // and compare to http response body
+        resultFile "test_action.csv"
    }
 }
 ```
@@ -339,35 +368,37 @@ explain action用来校验explain返回的字符串是否包含某些字符串

 下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/explain_action.groovy`:
 ```groovy
-explain {
-    sql("select 100")
+suite("explain_action", "demo") {
+    explain {
+        sql("select 100")

-    // contains("OUTPUT EXPRS:<slot 0> 100\n") && contains("PARTITION: UNPARTITIONED\n")
-    contains "OUTPUT EXPRS:<slot 0> 100\n"
-    contains "PARTITION: UNPARTITIONED\n"
-}
+        // contains("OUTPUT EXPRS:<slot 0> 100\n") && contains("PARTITION: UNPARTITIONED\n")
+        contains "OUTPUT EXPRS:<slot 0> 100\n"
+        contains "PARTITION: UNPARTITIONED\n"
+    }

-explain {
-    sql("select 100")
+    explain {
+        sql("select 100")

-    // contains(" 100\n") && !contains("abcdefg") && !("1234567")
-    contains " 100\n"
-    notContains "abcdefg"
-    notContains "1234567"
-}
+        // contains(" 100\n") && !contains("abcdefg") && !("1234567")
+        contains " 100\n"
+        notContains "abcdefg"
+        notContains "1234567"
+    }

-explain {
-    sql("select 100")
-    // simple callback
-    check { explainStr -> explainStr.contains("abcdefg") || explainStr.contains(" 100\n") }
-}
+    explain {
+        sql("select 100")
+        // simple callback
+        check { explainStr -> explainStr.contains("abcdefg") || explainStr.contains(" 100\n") }
+    }

-explain {
-    sql("a b c d e")
-    // callback with exception and time
-    check { explainStr, exception, startTime, endTime ->
-        // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
-        assertTrue(exception != null)
+    explain {
+        sql("a b c d e")
+        // callback with exception and time
+        check { explainStr, exception, startTime, endTime ->
+            // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
+            assertTrue(exception != null)
+        }
    }
 }
 ```
@@ -387,63 +418,68 @@ streamLoad action用于导入数据

 下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/streamLoad_action.groovy`:
 ```groovy
-def tableName = "test_streamload_action1"
+suite("streamLoad_action", "demo") {

-sql """
-    CREATE TABLE IF NOT EXISTS ${tableName} (
-        id int,
-        name varchar(255)
-    )
-    DISTRIBUTED BY HASH(id) BUCKETS 1
-    PROPERTIES (
-      "replication_num" = "1"
-    ) 
-"""
+    def tableName = "test_streamload_action1"

-streamLoad {
-    // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy
-    // db 'regression_test'
-    table tableName
+    sql """
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                id int,
+                name varchar(255)
+            )
+            DISTRIBUTED BY HASH(id) BUCKETS 1
+            PROPERTIES (
+              "replication_num" = "1"
+            ) 
+        """

-    // default label is UUID:
-    // set 'label' UUID.randomUUID().toString()
+    streamLoad {
+        // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy
+        // db 'regression_test'
+        table tableName

-    // default column_separator is specify in doris fe config, usually is '\t'.
-    // this line change to ','
-    set 'column_separator', ','
+        // default label is UUID:
+        // set 'label' UUID.randomUUID().toString()

-    // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
-    // also, you can stream load a http stream, e.g. http://xxx/some.csv
-    file 'streamload_input.csv'
+        // default column_separator is specify in doris fe config, usually is '\t'.
+        // this line change to ','
+        set 'column_separator', ','

-    time 10000 // limit inflight 10s
+        // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
+        // also, you can stream load a http stream, e.g. http://xxx/some.csv
+        file 'streamload_input.csv'

-    // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
-}
+        time 10000 // limit inflight 10s
+
+        // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
+    }


-// stream load 100 rows
-def rowCount = 100
-def rowIt = java.util.stream.LongStream.range(0, rowCount) // [0, rowCount)
-        .mapToObj({i -> [i, "a_" + i]}) // change Long to List<Long, String>
-        .iterator()
+    // stream load 100 rows
+    def rowCount = 100
+    // range: [0, rowCount)
+    // or rangeClosed: [0, rowCount]
+    def rowIt = range(0, rowCount)
+            .mapToObj({i -> [i, "a_" + i]}) // change Long to List<Long, String>
+            .iterator()

-streamLoad {
-    table tableName
-    // also, you can upload a memory iterator
-    inputIterator rowIt
+    streamLoad {
+        table tableName
+        // also, you can upload a memory iterator
+        inputIterator rowIt

-    // if declared a check callback, the default check condition will ignore.
-    // So you must check all condition
-    check { result, exception, startTime, endTime ->
-        if (exception != null) {
-            throw exception
+        // if declared a check callback, the default check condition will ignore.
+        // So you must check all condition
+        check { result, exception, startTime, endTime ->
+            if (exception != null) {
+                throw exception
+            }
+            log.info("Stream load result: ${result}".toString())
+            def json = parseJson(result)
+            assertEquals("success", json.Status.toLowerCase())
+            assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+            assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
        }
-        log.info("Stream load result: ${result}".toString())
-        def json = parseJson(result)
-        assertEquals("success", json.Status.toLowerCase())
-        assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
-        assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
    }
 }
 ```
@@ -478,8 +514,14 @@ thread, lazyCheck, events, connect, selectUnionAll
 # 测试demo group下的sql_action
 ./run-regression-test.sh --run -g demo -s sql_action

+# 测试demo目录下的sql_action
+./run-regression-test.sh --run -d demo -s sql_action
+
 # 自定义配置
 ./run-regression-test.sh --run -conf a=b
+
+# 并发执行
+./run-regression-test.sh --run -parallel 5 -suiteParallel 10 -actionParallel 20
 ```

 ## 使用查询结果自动生成.out文件
@@ -489,4 +531,12 @@ thread, lazyCheck, events, connect, selectUnionAll

 # 使用查询结果自动生成sql_action用例的.out文件，如果.out文件存在则覆盖
 ./run-regression-test.sh --run sql_action -forceGenOut
+```
+
+## CI/CD的支持
+### TeamCity
+TeamCity可以通过stdout识别Service Message。当使用`--teamcity`参数启动回归测试框架时，回归测试框架就会在stdout打印TeamCity Service Message，TeamCity将会自动读取stdout中的事件日志，并在当前流水线中展示`Tests`，其中会展示测试的test及其日志。
+因此只需要配置下面一行启动回归测试框架的命令即可。其中`-Dteamcity.enableStdErr=false`可以让错误日志也打印到stdout中，方便按时间顺序分析日志。
+```shell
+JAVA_OPTS="-Dteamcity.enableStdErr=${enableStdErr}" ./run-regression-test.sh --teamcity --run
 ```
@@ -21,9 +21,18 @@ under the License.
    <timestamp key="LOG_TIME" datePattern="yyyyMMdd.HHmmss"/>

    <appender name="stdoutAppender" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n</pattern>
-        </encoder>
+        <if condition='property("stdoutAppenderType").equalsIgnoreCase("teamcity")'>
+            <then>
+                <encoder class="org.apache.doris.regression.logger.TeamcityServiceMessageEncoder">
+                    <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n</pattern>
+                </encoder>
+            </then>
+            <else>
+                <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+                    <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n</pattern>
+                </encoder>
+            </else>
+        </if>
    </appender>

    <appender name="rollingFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
@@ -38,5 +38,7 @@ dataPath = "${DORIS_HOME}/regression-test/data"
 testGroups = ""
 // empty suite will test all suite
 testSuites = ""
+// empty directories will test all directories
+testDirectories = ""

 customConf1 = "test_custom_conf_value"
@@ -0,0 +1,4 @@
+1
+2
+3
+4
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !test_sql_file --
+200
+100
+300
+
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !test_sql_file_order --
+100
+200
+300
+
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !avg_decimal --
+\N
+
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !agg_output_as_right_tale_left_outer_order --
+1	1
+2	2
+
@@ -130,6 +130,24 @@ under the License.
                        </configuration>
                    </execution>
                </executions>
+                <configuration>
+                    <shadedArtifactAttached>false</shadedArtifactAttached>
+                    <artifactSet>
+                        <includes>
+                            <include>*:*</include>
+                        </includes>
+                    </artifactSet>
+                    <filters>
+                        <filter>
+                            <artifact>*:*</artifact>
+                            <excludes>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                            </excludes>
+                        </filter>
+                    </filters>
+                </configuration>
            </plugin>
        </plugins>
        <pluginManagement>
@@ -215,5 +233,10 @@ under the License.
            <artifactId>guava</artifactId>
            <version>31.0.1-jre</version>
        </dependency>
+        <dependency>
+            <groupId>org.codehaus.janino</groupId>
+            <artifactId>janino</artifactId>
+            <version>3.1.6</version>
+        </dependency>
    </dependencies>
 </project>
@@ -27,6 +27,7 @@ import org.apache.doris.regression.util.JdbcUtils

 import java.sql.Connection
 import java.sql.DriverManager
+import java.util.function.Predicate

 import static org.apache.doris.regression.ConfigOptions.*

@@ -47,6 +48,7 @@ class Config {

    public String testGroups
    public String testSuites
+    public String testDirectories
    public boolean generateOutputFile
    public boolean forceGenerateOutputFile
    public boolean randomOrder
@@ -55,8 +57,10 @@ class Config {

    public Set<String> suiteWildcard = new HashSet<>()
    public Set<String> groups = new HashSet<>()
+    public Set<String> directories = new HashSet<>()
    public InetSocketAddress feHttpInetSocketAddress
    public Integer parallel
+    public Integer suiteParallel
    public Integer actionParallel
    public Integer times
    public boolean withOutLoadData
@@ -65,7 +69,7 @@ class Config {

    Config(String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword,
           String feHttpAddress, String feHttpUser, String feHttpPassword,
-           String suitePath, String dataPath, String testGroups, String testSuites) {
+           String suitePath, String dataPath, String testGroups, String testSuites, String testDirectories) {
        this.defaultDb = defaultDb
        this.jdbcUrl = jdbcUrl
        this.jdbcUser = jdbcUser
@@ -77,6 +81,7 @@ class Config {
        this.dataPath = dataPath
        this.testGroups = testGroups
        this.testSuites = testSuites
+        this.testDirectories = testDirectories
    }

    static Config fromCommandLine(CommandLine cmd) {
@@ -106,6 +111,11 @@ class Config {
                .collect({g -> g.trim()})
                .findAll({g -> g != null && g.length() > 0})
                .toSet()
+        config.directories = cmd.getOptionValue(directoriesOpt, config.testDirectories)
+                .split(",")
+                .collect({d -> d.trim()})
+                .findAll({d -> d != null && d.length() > 0})
+                .toSet()

        config.feHttpAddress = cmd.getOptionValue(feHttpAddressOpt, config.feHttpAddress)
        try {
@@ -116,7 +126,7 @@ class Config {
            throw new IllegalStateException("Can not parse stream load address: ${config.feHttpAddress}", t)
        }

-        config.defaultDb = cmd.getOptionValue(jdbcOpt, config.defaultDb)
+        config.defaultDb = cmd.getOptionValue(defaultDbOpt, config.defaultDb)
        config.jdbcUrl = cmd.getOptionValue(jdbcOpt, config.jdbcUrl)
        config.jdbcUser = cmd.getOptionValue(userOpt, config.jdbcUser)
        config.jdbcPassword = cmd.getOptionValue(passwordOpt, config.jdbcPassword)
@@ -125,6 +135,7 @@ class Config {
        config.generateOutputFile = cmd.hasOption(genOutOpt)
        config.forceGenerateOutputFile = cmd.hasOption(forceGenOutOpt)
        config.parallel = Integer.parseInt(cmd.getOptionValue(parallelOpt, "1"))
+        config.suiteParallel = Integer.parseInt(cmd.getOptionValue(suiteParallelOpt, "1"))
        config.actionParallel = Integer.parseInt(cmd.getOptionValue(actionParallelOpt, "10"))
        config.times = Integer.parseInt(cmd.getOptionValue(timesOpt, "1"))
        config.randomOrder = cmd.hasOption(randomOrderOpt)
@@ -151,7 +162,8 @@ class Config {
            configToString(obj.suitePath),
            configToString(obj.dataPath),
            configToString(obj.testGroups),
-            configToString(obj.testSuites)
+            configToString(obj.testSuites),
+            configToString(obj.testDirectories)
        )

        def declareFileNames = config.getClass()
@@ -218,6 +230,11 @@ class Config {
            log.info("Set testGroups to '${config.testGroups}' because not specify.".toString())
        }

+        if (config.testDirectories == null) {
+            config.testDirectories = ""
+            log.info("Set testDirectories to empty because not specify.".toString())
+        }
+
        if (config.testSuites == null) {
            config.testSuites = ""
            log.info("Set testSuites to empty because not specify.".toString())
@@ -228,6 +245,11 @@ class Config {
            log.info("Set parallel to 1 because not specify.".toString())
        }

+        if (config.suiteParallel == null) {
+            config.suiteParallel = 1
+            log.info("Set suiteParallel to 1 because not specify.".toString())
+        }
+
        if (config.actionParallel == null) {
            config.actionParallel = 10
            log.info("Set actionParallel to 10 because not specify.".toString())
@@ -265,6 +287,25 @@ class Config {
        return DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword)
    }

+    Predicate<String> getDirectoryFilter() {
+        return (Predicate<String>) { String directoryName ->
+            if (directories.isEmpty()) {
+                return true
+            }
+
+            String relativePath = new File(suitePath).relativePath(new File(directoryName))
+            List<String> allLevelPaths = new ArrayList<>()
+            String parentPath = ""
+            for (String pathName : relativePath.split(File.separator)) {
+                String currentPath = parentPath + pathName
+                allLevelPaths.add(currentPath)
+                parentPath = currentPath + File.separator
+            }
+
+            return allLevelPaths.any {directories.contains(it) }
+        }
+    }
+
    private void buildUrlWithDefaultDb() {
        String urlWithDb = jdbcUrl
        String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3)
@@ -39,10 +39,12 @@ class ConfigOptions {
    static Option dataOpt
    static Option suiteOpt
    static Option groupsOpt
+    static Option directoriesOpt
    static Option confOpt
    static Option genOutOpt
    static Option forceGenOutOpt
    static Option parallelOpt
+    static Option suiteParallelOpt
    static Option actionParallelOpt
    static Option randomOrderOpt
    static Option timesOpt
@@ -130,6 +132,15 @@ class ConfigOptions {
                .longOpt("groups")
                .desc("the suite group to be test")
                .build()
+        directoriesOpt = Option.builder("d")
+                .argName("directories")
+                .required(false)
+                .hasArg(true)
+                .optionalArg(true)
+                .type(String.class)
+                .longOpt("directories")
+                .desc("only the use cases in these directories can be executed")
+                .build()
        feHttpAddressOpt = Option.builder("ha")
                .argName("address")
                .required(false)
@@ -179,7 +190,15 @@ class ConfigOptions {
                .optionalArg(true)
                .type(String.class)
                .longOpt("parallel")
-                .desc("the num of threads running test")
+                .desc("the num of threads running scripts")
+                .build()
+        suiteParallelOpt = Option.builder("suiteParallel")
+                .argName("parallel")
+                .required(false)
+                .hasArg(true)
+                .type(String.class)
+                .longOpt("suiteParallel")
+                .desc("the num of threads running for suites")
                .build()
        actionParallelOpt = Option.builder("actionParallel")
                .argName("parallel")
@@ -221,6 +240,7 @@ class ConfigOptions {
                .addOption(confOpt)
                .addOption(suiteOpt)
                .addOption(groupsOpt)
+                .addOption(directoriesOpt)
                .addOption(feHttpAddressOpt)
                .addOption(feHttpUserOpt)
                .addOption(feHttpPasswordOpt)
@@ -228,8 +248,10 @@ class ConfigOptions {
                .addOption(confFileOpt)
                .addOption(forceGenOutOpt)
                .addOption(parallelOpt)
+                .addOption(suiteParallelOpt)
                .addOption(actionParallelOpt)
                .addOption(randomOrderOpt)
+                .addOption(timesOpt)
                .addOption(withOutLoadDataOpt)

        CommandLine cmd = new DefaultParser().parse(options, args, true)
@@ -14,22 +14,31 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
 package org.apache.doris.regression

+import com.google.common.collect.Lists
 import groovy.transform.CompileStatic
 import jodd.util.Wildcard
-import org.apache.doris.regression.suite.Suite
-import org.apache.doris.regression.suite.SuiteContext
+import org.apache.doris.regression.suite.event.EventListener
+import org.apache.doris.regression.suite.GroovyFileSource
+import org.apache.doris.regression.suite.ScriptContext
+import org.apache.doris.regression.suite.ScriptSource
+import org.apache.doris.regression.suite.SqlFileSource
+import org.apache.doris.regression.suite.event.RecorderEventListener
+import org.apache.doris.regression.suite.event.StackEventListeners
+import org.apache.doris.regression.suite.SuiteScript
+import org.apache.doris.regression.suite.event.TeamcityEventListener
 import org.apache.doris.regression.util.Recorder
 import groovy.util.logging.Slf4j
 import org.apache.commons.cli.*
-import org.apache.doris.regression.util.SuiteInfo
 import org.codehaus.groovy.control.CompilerConfiguration

+import java.beans.Introspector
 import java.util.concurrent.Executors
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Future
-import java.util.stream.Collectors
+import java.util.function.Predicate

@Slf4j
@CompileStatic
@@ -38,8 +47,11 @@ class RegressionTest {
    static ClassLoader classloader
    static CompilerConfiguration compileConfig
    static GroovyShell shell
-    static ExecutorService executorService
-    static ExecutorService actionExecutorService
+    static ExecutorService scriptExecutors
+    static ExecutorService suiteExecutors
+    static ExecutorService actionExecutors
+    static ThreadLocal<Integer> threadLoadedClassNum = new ThreadLocal<>()
+    static final int cleanLoadedClassesThreshold = 20

    static void main(String[] args) {
        CommandLine cmd = ConfigOptions.initCommands(args)
@@ -49,150 +61,114 @@ class RegressionTest {

        Config config = Config.fromCommandLine(cmd)
        initGroovyEnv(config)
+        boolean success = true
        for (int i = 0; i < config.times; i++) {
            log.info("=== run ${i} time ===")
-            Recorder recorder = runSuites(config)
-            printResult(config, recorder)
+            Recorder recorder = runScripts(config)
+            success = printResult(config, recorder)
+        }
+        actionExecutors.shutdown()
+        suiteExecutors.shutdown()
+        scriptExecutors.shutdown()
+        log.info("Test finished")
+        if (!success) {
+            System.exit(1)
        }
-        actionExecutorService.shutdown()
-        executorService.shutdown()
    }

    static void initGroovyEnv(Config config) {
-        log.info("parallel = ${config.parallel}")
+        log.info("parallel = ${config.parallel}, suiteParallel = ${config.suiteParallel}, actionParallel = ${config.actionParallel}")
        classloader = new GroovyClassLoader()
        compileConfig = new CompilerConfiguration()
-        compileConfig.setScriptBaseClass((Suite as Class).name)
+        compileConfig.setScriptBaseClass((SuiteScript as Class).name)
        shell = new GroovyShell(classloader, new Binding(), compileConfig)
-        log.info("starting ${config.parallel} threads")
-        executorService = Executors.newFixedThreadPool(config.parallel)
-        actionExecutorService = Executors.newFixedThreadPool(config.actionParallel)
+        scriptExecutors = Executors.newFixedThreadPool(config.parallel)
+        suiteExecutors = Executors.newFixedThreadPool(config.suiteParallel)
+        actionExecutors = Executors.newFixedThreadPool(config.actionParallel)
    }

-    static List<File> findSuiteFiles(String root) {
+    static List<ScriptSource> findScriptSources(String root, Predicate<String> directoryFilter,
+                                                Predicate<String> fileFilter) {
        if (root == null) {
            log.warn('Not specify suite path')
-            return new ArrayList<File>()
+            return new ArrayList<ScriptSource>()
        }
-        List<File> files = new ArrayList<>()
+        List<ScriptSource> sources = new ArrayList<>()
        // 1. generate groovy for sql, excluding ddl
-        new File(root).eachFileRecurse { f ->
-            if (f.isFile() && f.name.endsWith('.sql') && f.getParentFile().name != "ddl") {
-                genetate_groovy_from_sql(f)
+        def rootFile = new File(root)
+        rootFile.eachFileRecurse { f ->
+            if (f.isFile() && f.name.endsWith('.sql') && f.getParentFile().name != "ddl"
+                    && fileFilter.test(f.name) && directoryFilter.test(f.getParent())) {
+                sources.add(new SqlFileSource(rootFile, f))
            }
        }

-        // 2. collect groovy files.
-        new File(root).eachFileRecurse { f ->
-            if (f.isFile() && f.name.endsWith('.groovy')) {
-                files.add(f)
+        // 2. collect groovy sources.
+        rootFile.eachFileRecurse { f ->
+            if (f.isFile() && f.name.endsWith('.groovy') && fileFilter.test(f.name)
+                    && directoryFilter.test(f.getParent())) {
+                sources.add(new GroovyFileSource(f))
            }
        }
-        return files
+        return sources
    }

-    static void genetate_groovy_from_sql(File f) {
-        File groovy_file = new File(f.getAbsolutePath() + '.generated.groovy')
-        groovy_file.delete()
-        groovy_file.createNewFile()
-        int separatorIndex = f.name.lastIndexOf('.')
-        String action_name = f.name.substring(0, separatorIndex)
-        if (action_name.endsWith('order')) {
-             groovy_file.text = "order_qt_${action_name} \"\"\"${f.text}\"\"\""
-        } else {
-             groovy_file.text = "qt_${action_name} \"\"\"${f.text}\"\"\""
+    static void runScript(Config config, ScriptSource source, Recorder recorder) {
+        def suiteFilter = { String suiteName, String groupName ->
+            canRun(config, suiteName, groupName)
        }
-    }
-
-    static String parseGroup(Config config, File suiteFile) {
-        // ./run-regression-test.sh -g group_name runs all groups
-        // whose name starting with ${group_name}.
-        String group = new File(config.suitePath).relativePath(suiteFile)
-        int separatorIndex = group.lastIndexOf(File.separator)
-        String groups = ",";
-        while (separatorIndex != -1) {
-            group = group.substring(0, separatorIndex)
-            groups += "${group},"
-            separatorIndex = group.lastIndexOf(File.separator)
-        }
-        // remove ',' at head and trail
-        groups = groups.substring(1, groups.length() - 1);
-        return groups;
-    }
-
-    static Integer runSuite(Config config, SuiteFile sf, ExecutorService executorService, Recorder recorder) {
-        File file = sf.file
-        String suiteName = sf.suiteName
-        String group = sf.group
-        def suiteConn = config.getConnection()
-        new SuiteContext(file, suiteConn, executorService, config, recorder).withCloseable { context ->
-            Suite suite = null
+        def file = source.getFile()
+        def eventListeners = getEventListeners(config, recorder)
+        new ScriptContext(file, suiteExecutors, actionExecutors,
+                config, eventListeners, suiteFilter).start { scriptContext ->
            try {
-                log.info("Run ${suiteName} in $file".toString())
-                suite = shell.parse(file) as Suite
-                suite.init(suiteName, group, context)
-                suite.run()
-                suite.doLazyCheck()
-                suite.successCallbacks.each { it() }
-                recorder.onSuccess(new SuiteInfo(file, group, suiteName))
-                log.info("Run ${suiteName} in ${file.absolutePath} succeed".toString())
-            } catch (Throwable t) {
-                if (suite != null) {
-                    suite.failCallbacks.each { it() }
-                }
-                recorder.onFailure(new SuiteInfo(file, group, suiteName))
-                log.error("Run ${suiteName} in ${file.absolutePath} failed".toString(), t)
+                SuiteScript suiteScript = source.toScript(scriptContext, shell)
+                suiteScript.run()
            } finally {
-                if (suite != null) {
-                    suite.finishCallbacks.each { it() }
-                }
+                // avoid jvm metaspace oom
+                cleanLoadedClassesIfNecessary()
            }
-            shell.resetLoadedClasses()
        }
-
-        return 0
    }

-    static void runSuites(Config config, Recorder recorder, Closure suiteNameMatch) {
-        def files = findSuiteFiles(config.suitePath)
-        List<SuiteFile> runScripts = files.stream().map({ file ->
-            String suiteName = file.name.substring(0, file.name.lastIndexOf('.'))
-            String group = parseGroup(config, file)
-            return new SuiteFile(file, suiteName, group)
-        }).filter({ sf ->
-            suiteNameMatch(sf.suiteName) && canRun(config, sf.suiteName, sf.group)
-        }).collect(Collectors.toList())
-
+    static void runScripts(Config config, Recorder recorder,
+                           Predicate<String> directoryFilter, Predicate<String> fileNameFilter) {
+        def scriptSources = findScriptSources(config.suitePath, directoryFilter, fileNameFilter)
        if (config.randomOrder) {
-            Collections.shuffle(files)
+            Collections.shuffle(scriptSources)
        }
-        log.info('Start to run suites')
-        int totalFile = runScripts.size()
-        def futures = new ArrayList<Future>()
-        runScripts.eachWithIndex { sf, i ->
-            log.info("[${i + 1}/${totalFile}] Run ${sf.suiteName} in ${sf.file}".toString())
-            Future future = executorService.submit {
-                runSuite(config, sf, actionExecutorService, recorder)
+//        int totalFile = scriptSources.size()
+
+        List<Future> futures = Lists.newArrayList()
+        scriptSources.eachWithIndex { source, i ->
+//            log.info("Prepare scripts [${i + 1}/${totalFile}]".toString())
+            def future = scriptExecutors.submit {
+                runScript(config, source, recorder)
            }
            futures.add(future)
        }

-        for (Future<Integer> future : futures) {
+        // wait all scripts
+        for (Future future : futures) {
            try {
                future.get()
-            }
-            catch (Throwable t) {
-                log.info(" exception ${t.toString()}")
+            } catch (Throwable t) {
+                // do nothing, because already save to Recorder
            }
        }
    }

-    static Recorder runSuites(Config config) {
+    static Recorder runScripts(Config config) {
        def recorder = new Recorder()
+        def directoryFilter = config.getDirectoryFilter()
        if (!config.withOutLoadData) {
-            runSuites(config, recorder, {suiteName -> suiteName == "load" })
+            log.info('Start to run load scripts')
+            runScripts(config, recorder, directoryFilter,
+                    { fileName -> fileName.substring(0, fileName.lastIndexOf(".")) == "load" })
        }
-        runSuites(config, recorder, {suiteName -> suiteName != "load" })
+        log.info('Start to run scripts')
+        runScripts(config, recorder, directoryFilter,
+                { fileName -> fileName.substring(0, fileName.lastIndexOf(".")) != "load" })

        return recorder
    }
@@ -211,28 +187,67 @@ class RegressionTest {
        return false
    }

-    static void printResult(Config config, Recorder recorder) {
+    static List<EventListener> getEventListeners(Config config, Recorder recorder) {
+        StackEventListeners listeners = new StackEventListeners()
+
+        // RecorderEventListener **MUST BE** first listener
+        listeners.addListener(new RecorderEventListener(recorder))
+
+        // other listeners
+        String stdoutAppenderType = System.getProperty("stdoutAppenderType")
+        if (stdoutAppenderType != null && stdoutAppenderType.equalsIgnoreCase("teamcity")) {
+            listeners.addListener(new TeamcityEventListener())
+        }
+        return [listeners] as List<EventListener>
+    }
+
+    static void cleanLoadedClassesIfNecessary() {
+        Integer loadedClassNum = threadLoadedClassNum.get()
+        if (loadedClassNum == null) {
+            loadedClassNum = 0
+        }
+        loadedClassNum += 1
+        if (loadedClassNum >= cleanLoadedClassesThreshold) {
+            // release dynamic script class: ThreadGroupContext.getContext().beanInfoCache()
+            Introspector.flushCaches()
+            loadedClassNum = 0
+        }
+        threadLoadedClassNum.set(loadedClassNum)
+    }
+
+    static boolean printResult(Config config, Recorder recorder) {
        int allSuiteNum = recorder.successList.size() + recorder.failureList.size()
        int failedSuiteNum = recorder.failureList.size()
-        log.info("Test ${allSuiteNum} suites, failed ${failedSuiteNum} suites".toString())
+        int fatalScriptNum = recorder.fatalScriptList.size()
+        log.info("Test ${allSuiteNum} suites, failed ${failedSuiteNum} suites, fatal ${fatalScriptNum} scripts".toString())

        // print success list
-        {
+        if (!recorder.successList.isEmpty()) {
            String successList = recorder.successList.collect { info ->
                "${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}"
            }.join('\n')
-            log.info("successList suites:\n${successList}".toString())
+            log.info("SuccessList suites:\n${successList}".toString())
        }

        // print failure list
-        if (!recorder.failureList.isEmpty()) {
-            def failureList = recorder.failureList.collect() { info ->
-                "${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}"
-            }.join('\n')
-            log.info("Failure suites:\n${failureList}".toString())
+        if (!recorder.failureList.isEmpty() || !recorder.fatalScriptList.isEmpty()) {
+            if (!recorder.failureList.isEmpty()) {
+                def failureList = recorder.failureList.collect() { info ->
+                    "${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}"
+                }.join('\n')
+                log.info("Failure suites:\n${failureList}".toString())
+            }
+            if (!recorder.fatalScriptList.isEmpty()) {
+                def failureList = recorder.fatalScriptList.collect() { info ->
+                    "${info.file.absolutePath}"
+                }.join('\n')
+                log.info("Fatal scripts:\n${failureList}".toString())
+            }
            printFailed()
+            return false
        } else {
            printPassed()
+            return true
        }
    }

@@ -255,19 +270,4 @@ class RegressionTest {
                 ||_|/_/   \\_\\___|_____|_____|____/
                 |'''.stripMargin())
    }
-
-    static class SuiteFile {
-
-        File file
-        String suiteName
-        String group
-
-        SuiteFile(File file, String suiteName, String group) {
-            this.file = file
-            this.suiteName = suiteName
-            this.group = group
-        }
-
-    }
-
 }
@@ -80,15 +80,11 @@ class ExplainAction implements SuiteAction {
                }
            } catch (Throwable t) {
                log.error("Explain and custom check failed", t)
-                List resList = [context.file.getName(), 'explain', sql, t]
-                context.recorder.reportDiffResult(resList)
                throw t
            }
        } else if (result.exception != null) {
            String msg = "Explain failed"
            log.error(msg, result.exception)
-            List resList = [context.file.getName(), 'explain', sql, result.exception]
-            context.recorder.reportDiffResult(resList)
            throw new IllegalStateException(msg, result.exception)
        } else {
            for (String string : containsStrings) {
@@ -97,8 +93,6 @@ class ExplainAction implements SuiteAction {
                            + "but actual explain string is:\n${explainString}").toString()
                    log.info(msg)
                    def t = new IllegalStateException(msg)
-                    List resList = [context.file.getName(), 'explain', sql, t]
-                    context.recorder.reportDiffResult(resList)
                    throw t
                }
            }
@@ -108,8 +102,6 @@ class ExplainAction implements SuiteAction {
                            + "but actual explain string is:\n${explainString}").toString()
                    log.info(msg)
                    def t = new IllegalStateException(msg)
-                    List resList = [context.file.getName(), 'explain', sql, t]
-                    context.recorder.reportDiffResult(resList)
                    throw t
                }
            }
@@ -26,6 +26,8 @@ import org.apache.doris.regression.util.OutputUtils
 import groovy.json.JsonSlurper
 import groovy.util.logging.Slf4j
 import org.apache.http.HttpEntity
+import org.apache.http.HttpStatus
+import org.apache.http.client.methods.CloseableHttpResponse
 import org.apache.http.client.methods.RequestBuilder
 import org.apache.http.entity.FileEntity
 import org.apache.http.entity.InputStreamEntity
@@ -153,7 +155,14 @@ class StreamLoadAction implements SuiteAction {
    }

    private InputStream httpGetStream(CloseableHttpClient client, String url) {
-        return client.execute(RequestBuilder.get(url).build()).getEntity().getContent()
+        CloseableHttpResponse resp = client.execute(RequestBuilder.get(url).build())
+        int code = resp.getStatusLine().getStatusCode()
+        if (code != HttpStatus.SC_OK) {
+            String streamBody = EntityUtils.toString(resp.getEntity())
+            throw new IllegalStateException("Get http stream failed, status code is ${code}, body:\n${streamBody}")
+        }
+
+        return resp.getEntity().getContent()
    }

    private RequestBuilder prepareRequestHeader(RequestBuilder requestBuilder) {
@@ -207,9 +216,6 @@ class StreamLoadAction implements SuiteAction {
                def respCode = resp.getStatusLine().getStatusCode()
                // should redirect to backend
                if (respCode != 307) {
-                    List resList = [context.file.getName(), 'streamLoad', '', "Expect frontend stream load response code is 307, " +
-                            "but meet ${respCode}\nbody: ${body}"]
-                    context.recorder.reportDiffResult(resList)
                    throw new IllegalStateException("Expect frontend stream load response code is 307, " +
                            "but meet ${respCode}\nbody: ${body}")
                }
@@ -230,10 +236,6 @@ class StreamLoadAction implements SuiteAction {
                    String body = EntityUtils.toString(resp.getEntity())
                    def respCode = resp.getStatusLine().getStatusCode()
                    if (respCode != 200) {
-                        List resList = [context.file.getName(), 'streamLoad', '', "Expect backend stream load response code is 200, " +
-                                "but meet ${respCode}\nbody: ${body}"]
-                        context.recorder.reportDiffResult(resList)
-
                        throw new IllegalStateException("Expect backend stream load response code is 200, " +
                                "but meet ${respCode}\nbody: ${body}")
                    }
@@ -242,8 +244,6 @@ class StreamLoadAction implements SuiteAction {
            }
        } catch (Throwable t) {
            log.info("StreamLoadAction Exception: ", t)
-            List resList = [context.file.getName(), 'streamLoad', '', t]
-            context.recorder.reportDiffResult(resList)
        }
        return responseText
    }
@@ -253,8 +253,6 @@ class StreamLoadAction implements SuiteAction {
            check.call(responseText, ex, startTime, endTime)
        } else {
            if (ex != null) {
-                List resList = [context.file.getName(), 'streamLoad', '', ex]
-                context.recorder.reportDiffResult(resList)
                throw ex
            }

@@ -267,19 +265,13 @@ class StreamLoadAction implements SuiteAction {
                    String errorDetails = HttpClients.createDefault().withCloseable { client ->
                        httpGetString(client, errorUrl)
                    }
-                    List resList = [context.file.getName(), 'streamLoad', '', "Stream load failed:\n${responseText}\n${errorDetails}"]
-                    context.recorder.reportDiffResult(resList)
                    throw new IllegalStateException("Stream load failed:\n${responseText}\n${errorDetails}")
                }
-                List resList = [context.file.getName(), 'streamLoad', '', "Stream load failed:\n${responseText}"]
-                context.recorder.reportDiffResult(resList)
                throw new IllegalStateException("Stream load failed:\n${responseText}")
            }
            long numberTotalRows = result.NumberTotalRows.toLong()
            long numberLoadedRows = result.NumberLoadedRows.toLong()
            if (numberTotalRows != numberLoadedRows) {
-                List resList = [context.file.getName(), 'streamLoad', '', "Stream load rows mismatch:\n${responseText}"]
-                context.recorder.reportDiffResult(resList)
                throw new IllegalStateException("Stream load rows mismatch:\n${responseText}")

            }
@@ -289,8 +281,6 @@ class StreamLoadAction implements SuiteAction {
                try{
                    Assert.assertTrue("Expect elapsed <= ${time}, but meet ${elapsed}", elapsed <= time)
                } catch (Throwable t) {
-                    List resList = [context.file.getName(), 'streamLoad', '', "Expect elapsed <= ${time}, but meet ${elapsed}"]
-                    context.recorder.reportDiffResult(resList)
                    throw new IllegalStateException("Expect elapsed <= ${time}, but meet ${elapsed}")
                }
            }
@@ -17,18 +17,36 @@

 package org.apache.doris.regression.action

+import groovy.transform.CompileStatic
 import groovy.transform.stc.ClosureParams
 import groovy.transform.stc.FromString
 import groovy.util.logging.Slf4j
+import org.apache.commons.io.LineIterator
+import org.apache.doris.regression.util.DataUtils
+import org.apache.doris.regression.util.OutputUtils
+import org.apache.http.HttpStatus
+import org.apache.http.client.methods.CloseableHttpResponse
+import org.apache.http.client.methods.RequestBuilder
+import org.apache.http.impl.client.HttpClients
+import org.apache.http.util.EntityUtils
+
+import javax.swing.text.html.parser.Entity
+import java.nio.charset.StandardCharsets
 import java.sql.Connection

 import org.apache.doris.regression.suite.SuiteContext
 import org.apache.doris.regression.util.JdbcUtils
 import org.junit.Assert

+import java.util.function.Consumer
+
@Slf4j
+@CompileStatic
 class TestAction implements SuiteAction {
    private String sql
+    private boolean isOrder
+    private String resultFileUri
+    private Iterator<Object> resultIterator
    private Object result
    private long time
    private long rowNum = -1
@@ -70,22 +88,76 @@ class TestAction implements SuiteAction {
                if (this.result != null) {
                    Assert.assertEquals(this.result, result.result)
                }
+                if (this.resultIterator != null) {
+                    String errorMsg = OutputUtils.checkOutput(this.resultIterator, result.result.iterator(),
+                        { Object row ->
+                            if (row instanceof List) {
+                                return OutputUtils.toCsvString(row as List)
+                            } else {
+                                return OutputUtils.columnToCsvString(row)
+                            }
+                        },
+                            { List<Object> row -> OutputUtils.toCsvString(row) }, "Check failed")
+                    if (errorMsg != null) {
+                        throw new IllegalStateException(errorMsg)
+                    }
+                }
+                if (this.resultFileUri != null) {
+                    Consumer<InputStream> checkFunc = { InputStream inputStream ->
+                        def lineIt = new LineIterator(new InputStreamReader(inputStream, StandardCharsets.UTF_8))
+                        def csvIt = new OutputUtils.CsvParserIterator(lineIt)
+                        String errMsg = OutputUtils.checkOutput(csvIt, result.result.iterator(),
+                                { List<String> row -> OutputUtils.toCsvString(row as List<Object>) },
+                                { List<Object> row -> OutputUtils.toCsvString(row) },
+                                "Check failed compare to")
+                        if (errMsg != null) {
+                            throw new IllegalStateException(errMsg)
+                        }
+                    }
+
+                    if (this.resultFileUri.startsWith("http://") || this.resultFileUri.startsWith("https://")) {
+                        log.info("Compare to http stream: ${this.resultFileUri}")
+                        HttpClients.createDefault().withCloseable { client ->
+                            client.execute(RequestBuilder.get(this.resultFileUri).build()).withCloseable { CloseableHttpResponse resp ->
+                                int code = resp.getStatusLine().getStatusCode()
+                                if (code != HttpStatus.SC_OK) {
+                                    String streamBody = EntityUtils.toString(resp.getEntity())
+                                    throw new IllegalStateException("Get http stream failed, status code is ${code}, body:\n${streamBody}")
+                                }
+                                checkFunc(resp.entity.content)
+                            }
+                        }
+                    } else {
+                        String fileName = resultFileUri
+                        if (!new File(fileName).isAbsolute()) {
+                            fileName = new File(context.dataPath, fileName).getAbsolutePath()
+                        }
+                        def file = new File(fileName)
+                        if (!file.exists()) {
+                            log.warn("Result file not exists: ${file}".toString())
+                        }
+                        log.warn("Compare to local file: ${file}".toString())
+                        file.newInputStream().withCloseable { inputStream ->
+                            checkFunc(inputStream)
+                        }
+                    }
+                }
            }
        } catch (Throwable t) {
-            log.info("TestAction Exception: ", t)
-            List resList = [context.file.getName(), 'test', sql, t]
-            context.recorder.reportDiffResult(resList)
-            throw t
+            throw new IllegalStateException("TestAction failed, sql:\n${sql}", t)
        }
    }

    ActionResult doRun(Connection conn) {
-        Object result = null
+        List<List<Object>> result = null
        Throwable ex = null
        long startTime = System.currentTimeMillis()
        try {
-            log.info("Execute sql:\n${sql}".toString())
+            log.info("Execute ${isOrder ? "order_" : ""}sql:\n${sql}".toString())
            result = JdbcUtils.executeToList(conn, sql)
+            if (isOrder) {
+                result = DataUtils.sortByToString(result)
+            }
        } catch (Throwable t) {
            ex = t
        }
@@ -101,6 +173,10 @@ class TestAction implements SuiteAction {
        this.sql = sqlSupplier.call()
    }

+    void order(boolean isOrder) {
+        this.isOrder = isOrder
+    }
+
    void time(long time) {
        this.time = time
    }
@@ -125,6 +201,22 @@ class TestAction implements SuiteAction {
        this.result = resultSupplier.call()
    }

+    void resultIterator(Iterator<Object> resultIterator) {
+        this.resultIterator = resultIterator
+    }
+
+    void resultIterator(Closure<Iterator<Object>> resultIteratorSupplier) {
+        this.resultIterator = resultIteratorSupplier.call()
+    }
+
+    void resultFile(String resultFile) {
+        this.resultFileUri = resultFile
+    }
+
+    void resultFile(Closure<String> resultFileSupplier) {
+        this.resultFileUri = resultFileSupplier.call()
+    }
+
    void exception(String exceptionMsg) {
        this.exception = exceptionMsg
    }
@@ -138,12 +230,12 @@ class TestAction implements SuiteAction {
    }

    class ActionResult {
-        Object result
+        List<List<Object>> result
        Throwable exception
        long startTime
        long endTime

-        ActionResult(Object result, Throwable exception, long startTime, long endTime) {
+        ActionResult(List<List<Object>> result, Throwable exception, long startTime, long endTime) {
            this.result = result
            this.exception = exception
            this.startTime = startTime
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.logger
+
+import ch.qos.logback.classic.Level
+import ch.qos.logback.classic.encoder.PatternLayoutEncoder
+import ch.qos.logback.classic.spi.ILoggingEvent
+import org.apache.doris.regression.suite.SuiteContext
+import org.apache.doris.regression.util.TeamcityUtils
+
+class TeamcityServiceMessageEncoder extends PatternLayoutEncoder {
+    static final ThreadLocal<SuiteContext> CURRENT_SUITE_CONTEXT = new ThreadLocal<>()
+
+    private boolean enableStdErr
+
+    TeamcityServiceMessageEncoder() {
+        enableStdErr = Boolean.valueOf(System.getProperty("teamcity.enableStdErr", "true"))
+        System.out.println("TeamcityServiceMessageEncoder: teamcity.enableStdErr=${enableStdErr}")
+    }
+
+    @Override
+    byte[] encode(ILoggingEvent event) {
+        String originLog = layout.doLayout(event)
+
+        SuiteContext suiteContext = CURRENT_SUITE_CONTEXT.get()
+        if (suiteContext == null) {
+            return convertToBytes(originLog)
+        }
+
+        String serviceMessageLog
+        if (event.getLevel().levelInt == Level.ERROR.levelInt && enableStdErr) {
+            serviceMessageLog = TeamcityUtils.formatStdErr(suiteContext, originLog)
+        } else {
+            serviceMessageLog = TeamcityUtils.formatStdOut(suiteContext, originLog)
+        }
+        return convertToBytes(serviceMessageLog + "\n")
+    }
+
+    private byte[] convertToBytes(String s) {
+        if (getCharset() == null) {
+            return s.getBytes()
+        } else {
+            return s.getBytes(getCharset())
+        }
+    }
+}
@@ -0,0 +1,175 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.suite
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.doris.regression.Config
+import org.apache.doris.regression.suite.event.EventListener
+
+import java.lang.reflect.UndeclaredThrowableException
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Phaser
+import java.util.function.Function
+
+@Slf4j
+@CompileStatic
+class ScriptContext implements Closeable {
+    public final File file
+    public final Config config
+    public final File dataPath
+    public final File outputFile
+    public final String name
+    public final String flowName
+    public final String flowId
+    public final List<EventListener> eventListeners
+    public final ExecutorService suiteExecutors
+    public final ExecutorService actionExecutors
+    public final Closure<Boolean> suiteFilter
+    private long startTime
+    private long finishTime
+    private final Thread createContextThread
+    private final Phaser phaser = new Phaser(1) // register main script thread
+
+    ScriptContext(File file, ExecutorService suiteExecutors, ExecutorService actionExecutors, Config config,
+                  List<EventListener> eventListeners, Closure<Boolean> suiteFilter) {
+        this.file = file
+        this.config = config
+        this.name = this.flowId = new File(config.suitePath).relativePath(file)
+        this.flowName = '' // force set to empty string, for teamcity
+        this.eventListeners = eventListeners
+        this.suiteExecutors = suiteExecutors
+        this.actionExecutors = actionExecutors
+        this.suiteFilter = suiteFilter
+        this.createContextThread = Thread.currentThread()
+
+        def path = new File(config.suitePath).relativePath(file)
+        def outputRelativePath = path.substring(0, path.lastIndexOf(".")) + ".out"
+        this.outputFile = new File(new File(config.dataPath), outputRelativePath)
+        this.dataPath = this.outputFile.getParentFile().getCanonicalFile()
+    }
+
+    private final synchronized Suite newSuite(String suiteName, String group) {
+        if (Thread.currentThread() != createContextThread) {
+            throw new IllegalStateException("Can not create suite in another thread")
+        }
+        // if close scriptContext, the phaser will become to 1
+        if (phaser.getPhase() > 0) {
+            throw new IllegalStateException("Can not create suite after close scriptContext")
+        }
+
+        SuiteContext suiteContext = new SuiteContext(file, suiteName, group, this,
+                suiteExecutors, actionExecutors, config) {
+            @Override
+            void close() {
+                try {
+                    super.close()
+                } finally {
+                    // count down
+                    phaser.arriveAndDeregister()
+                }
+            }
+        }
+        Suite suite = new Suite(suiteName, group, suiteContext)
+        // count up, register suite thread
+        phaser.register()
+        return suite
+    }
+
+    synchronized final Suite createAndRunSuite(String suiteName, String group, Closure suiteBody) {
+        Suite suite = newSuite(suiteName, group)
+
+        suiteExecutors.submit {
+            suite.context.start {
+                log.info("Run ${suiteName} in ${file}".toString())
+
+                try {
+                    // delegate closure
+                    suiteBody.setResolveStrategy(Closure.DELEGATE_FIRST)
+                    suiteBody.setDelegate(suite)
+
+                    // execute body
+                    suiteBody.call(suite)
+
+                    // check
+                    suite.doLazyCheck()
+
+                    // success
+                    try {
+                        suite.successCallbacks.each { it(suite) }
+                    } catch (Throwable t) {
+                        throw new IllegalStateException("Run suite success callbacks failed", t)
+                    }
+
+                    log.info("Run ${suiteName} in ${file.absolutePath} succeed".toString())
+                } catch (Throwable t) {
+                    log.error("Run ${suiteName} in ${file.absolutePath} failed".toString(), t)
+                    try {
+                        // fail
+                        if (suite != null) {
+                            suite.failCallbacks.each { it(suite) }
+                        }
+                    } catch (Throwable ex) {
+                        log.error("Run suite fail callbacks failed", ex)
+                        t.addSuppressed(ex)
+                    }
+                    throw t
+                } finally {
+                    try {
+                        // finish
+                        if (suite != null) {
+                            suite.finishCallbacks.each { it(suite) }
+                        }
+                    } catch (Throwable t) {
+                        log.error("Run suite finish callbacks failed", t)
+                    }
+                }
+            }
+        }
+        return suite
+    }
+
+    public <T> T start(Function<ScriptContext, T> action) {
+        this.startTime = System.currentTimeMillis()
+        eventListeners.each {it.onScriptStarted(this) }
+        this.withCloseable { scriptContext ->
+            try {
+                action.apply(scriptContext)
+            } catch (Throwable t) {
+                if (t instanceof UndeclaredThrowableException) {
+                    t = ((UndeclaredThrowableException) t).undeclaredThrowable
+                }
+                log.error("Fatal: run SuiteScript in ${file} failed".toString(), t)
+                eventListeners.each { it.onScriptFailed(scriptContext, t) }
+                null
+            }
+        }
+    }
+
+    synchronized void close() {
+        if (phaser.getPhase() > 0) {
+            throw new IllegalStateException("Can not close scriptContext twice")
+        }
+        // wait for all suite finished, phase += 1
+        phaser.arriveAndAwaitAdvance()
+
+        this.finishTime = System.currentTimeMillis()
+        long elapsed = finishTime - startTime
+        eventListeners.each {it.onScriptFinished(this, elapsed) }
+    }
+}
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.suite
+
+import groovy.transform.CompileStatic
+
+@CompileStatic
+class ScriptInfo {
+    File file
+
+    ScriptInfo(File file) {
+        this.file = file
+    }
+}
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.suite
+
+interface ScriptSource {
+    SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell)
+    File getFile()
+}
+
+class GroovyFileSource implements ScriptSource {
+    private File file
+
+    GroovyFileSource(File file) {
+        this.file = file
+    }
+
+    @Override
+    SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) {
+        SuiteScript suiteScript = shell.parse(file) as SuiteScript
+        suiteScript.init(scriptContext)
+        return suiteScript
+    }
+
+    @Override
+    File getFile() {
+        return file
+    }
+}
+
+class SqlFileSource implements ScriptSource {
+    private File suiteRoot
+    private File file
+
+    SqlFileSource(File suiteRoot, File file) {
+        this.suiteRoot = suiteRoot
+        this.file = file
+    }
+
+    String getGroup() {
+        return SuiteScript.getDefaultGroups(suiteRoot, file)
+    }
+
+    @Override
+    SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) {
+        String suiteName = file.name.substring(0, file.name.lastIndexOf("."))
+        String groupName = getGroup()
+        boolean order = suiteName.endsWith("_order")
+        String tag = suiteName
+        String sql = file.text
+
+        SuiteScript script = new SuiteScript() {
+            @Override
+            Object run() {
+                suite(suiteName, groupName) {
+                    quickTest(tag, sql, order)
+                }
+            }
+        }
+        script.init(scriptContext)
+        return script
+    }
+
+    @Override
+    File getFile() {
+        return file
+    }
+}
@@ -41,11 +41,11 @@ import java.util.stream.LongStream

 import static org.apache.doris.regression.util.DataUtils.sortByToString

-abstract class Suite extends Script implements GroovyInterceptable {
-    SuiteContext context
-    String name
-    String group
-    final Logger logger = LoggerFactory.getLogger(getClass())
+class Suite implements GroovyInterceptable {
+    final SuiteContext context
+    final String name
+    final String group
+    final Logger logger = LoggerFactory.getLogger(this.class)

    final List<Closure> successCallbacks = new Vector<>()
    final List<Closure> failCallbacks = new Vector<>()
@@ -53,7 +53,7 @@ abstract class Suite extends Script implements GroovyInterceptable {
    final List<Throwable> lazyCheckExceptions = new Vector<>()
    final List<Future> lazyCheckFutures = new Vector<>()

-    void init(String name, String group, SuiteContext context) {
+    Suite(String name, String group, SuiteContext context) {
        this.name = name
        this.group = group
        this.context = context
@@ -139,17 +139,25 @@ abstract class Suite extends Script implements GroovyInterceptable {
    }

    public <T> ListenableFuture<T> thread(String threadName = null, Closure<T> actionSupplier) {
-        return MoreExecutors.listeningDecorator(context.executorService).submit((Callable<T>) {
+        return MoreExecutors.listeningDecorator(context.actionExecutors).submit((Callable<T>) {
+            long startTime = System.currentTimeMillis()
            def originThreadName = Thread.currentThread().name
            try {
                Thread.currentThread().setName(threadName == null ? originThreadName : threadName)
+                context.scriptContext.eventListeners.each { it.onThreadStarted(context) }
+
                return actionSupplier.call()
+            } catch (Throwable t) {
+                context.scriptContext.eventListeners.each { it.onThreadFailed(context, t) }
+                throw t
            } finally {
                try {
                    context.closeThreadLocal()
                } catch (Throwable t) {
                    logger.warn("Close thread local context failed", t)
                }
+                long finishTime = System.currentTimeMillis()
+                context.scriptContext.eventListeners.each { it.onThreadFinished(context, finishTime - startTime) }
                Thread.currentThread().setName(originThreadName)
            }
        })
@@ -175,7 +183,7 @@ abstract class Suite extends Script implements GroovyInterceptable {
    }

    List<List<Object>> sql(String sqlStr, boolean isOrder = false) {
-        logger.info("Execute sql: ${sqlStr}".toString())
+        logger.info("Execute ${isOrder ? "order_" : ""}sql: ${sqlStr}".toString())
        def result = JdbcUtils.executeToList(context.getConnection(), sqlStr)
        if (isOrder) {
            result = DataUtils.sortByToString(result)
@@ -249,16 +257,16 @@ abstract class Suite extends Script implements GroovyInterceptable {
    void runAction(SuiteAction action, Closure actionSupplier) {
        actionSupplier.setDelegate(action)
        actionSupplier.setResolveStrategy(Closure.DELEGATE_FIRST)
-        actionSupplier.call()
+        actionSupplier.call(action)
        action.run()
    }

-    void quickTest(String tag, String sql, boolean order = false) {
-        logger.info("Execute tag: ${tag}, sql: ${sql}".toString())
+    void quickTest(String tag, String sql, boolean isOrder = false) {
+        logger.info("Execute tag: ${tag}, ${isOrder ? "order_" : ""}sql: ${sql}".toString())

        if (context.config.generateOutputFile || context.config.forceGenerateOutputFile) {
            def result = JdbcUtils.executorToStringList(context.getConnection(), sql)
-            if (order) {
+            if (isOrder) {
                result = sortByToString(result)
            }
            Iterator<List<Object>> realResults = result.iterator()
@@ -267,36 +275,29 @@ abstract class Suite extends Script implements GroovyInterceptable {
            writer.write(realResults, tag)
        } else {
            if (!context.outputFile.exists()) {
-                String res = "Missing outputFile: ${context.outputFile.getAbsolutePath()}"
-                List excelContentList = [context.file.getName(), context.file, context.file, res]
-                context.recorder.reportDiffResult(excelContentList)
                throw new IllegalStateException("Missing outputFile: ${context.outputFile.getAbsolutePath()}")
            }

            if (!context.getOutputIterator().hasNextTagBlock(tag)) {
-                String res = "Missing output block for tag '${tag}': ${context.outputFile.getAbsolutePath()}"
-                List excelContentList = [context.file.getName(), tag, context.file, res]
-                context.recorder.reportDiffResult(excelContentList)
                throw new IllegalStateException("Missing output block for tag '${tag}': ${context.outputFile.getAbsolutePath()}")
            }

            OutputUtils.TagBlockIterator expectCsvResults = context.getOutputIterator().next()
            List<List<Object>> realResults = JdbcUtils.executorToStringList(context.getConnection(), sql)
-            if (order) {
+            if (isOrder) {
                realResults = sortByToString(realResults)
            }
            String errorMsg = null
            try {
-                errorMsg = OutputUtils.checkOutput(expectCsvResults, realResults.iterator(), "Check tag '${tag}' failed")
+                errorMsg = OutputUtils.checkOutput(expectCsvResults, realResults.iterator(),
+                    { row -> OutputUtils.toCsvString(row as List<Object>) },
+                    {row ->  OutputUtils.toCsvString(row) },
+                    "Check tag '${tag}' failed")
            } catch (Throwable t) {
-                List excelContentList = [context.file.getName(), tag, sql.trim(), t]
-                context.recorder.reportDiffResult(excelContentList)
-                throw new IllegalStateException("Check tag '${tag}' failed", t)
+                throw new IllegalStateException("Check tag '${tag}' failed, sql:\n${sql}", t)
            }
            if (errorMsg != null) {
-                List excelContentList = [context.file.getName(), tag, sql.trim(), errorMsg]
-                context.recorder.reportDiffResult(excelContentList)
-                throw new IllegalStateException(errorMsg)
+                throw new IllegalStateException("Check tag '${tag}' failed:\n${errorMsg}\n\nsql:\n${sql}")
            }
        }
    }
@@ -20,50 +20,88 @@ package org.apache.doris.regression.suite
 import groovy.transform.CompileStatic
 import org.apache.doris.regression.Config
 import org.apache.doris.regression.util.OutputUtils
-import org.apache.doris.regression.util.Recorder
 import groovy.util.logging.Slf4j

+import java.lang.reflect.UndeclaredThrowableException
 import java.sql.Connection
 import java.sql.DriverManager
 import java.util.concurrent.ExecutorService
+import java.util.function.Function

@Slf4j
@CompileStatic
 class SuiteContext implements Closeable {
    public final File file
-    private final Connection conn
+    public final String suiteName
+    public final String group
    public final ThreadLocal<Connection> threadLocalConn = new ThreadLocal<>()
    public final Config config
    public final File dataPath
    public final File outputFile
+    public final ScriptContext scriptContext
+    public final String flowName
+    public final String flowId
    public final ThreadLocal<OutputUtils.OutputBlocksIterator> threadLocalOutputIterator = new ThreadLocal<>()
-    public final ExecutorService executorService
-    public final Recorder recorder
-//    public final File tmpOutputPath
+    public final ExecutorService suiteExecutors
+    public final ExecutorService actionExecutors
    private volatile OutputUtils.OutputBlocksWriter outputBlocksWriter
+    private long startTime
+    private long finishTime
+    private volatile Throwable throwable

-    SuiteContext(File file, Connection conn, ExecutorService executorService, Config config, Recorder recorder) {
+    SuiteContext(File file, String suiteName, String group, ScriptContext scriptContext,
+                 ExecutorService suiteExecutors, ExecutorService actionExecutors, Config config) {
        this.file = file
-        this.conn = conn
+        this.suiteName = suiteName
+        this.group = group
        this.config = config
-        this.executorService = executorService
-        this.recorder = recorder
+        this.scriptContext = scriptContext
+
+        String packageName = getPackageName()
+        String className = getClassName()
+        this.flowName = "${packageName}.${className}.${suiteName}"
+        this.flowId = "${scriptContext.flowId}#${suiteName}"
+        this.suiteExecutors = suiteExecutors
+        this.actionExecutors = actionExecutors

        def path = new File(config.suitePath).relativePath(file)
        def outputRelativePath = path.substring(0, path.lastIndexOf(".")) + ".out"
        this.outputFile = new File(new File(config.dataPath), outputRelativePath)
        this.dataPath = this.outputFile.getParentFile().getCanonicalFile()
-//        def dataParentPath = new File(config.dataPath).parentFile.absolutePath
-//        def tmpOutputPath = "${dataParentPath}/tmp_output/${outputRelativePath}".toString()
-//        this.tmpOutputPath = new File(tmpOutputPath)
+    }
+
+    String getPackageName() {
+        String packageName = scriptContext.name
+        int dirSplitPos = packageName.lastIndexOf(File.separator)
+        if (dirSplitPos != -1) {
+            packageName = packageName.substring(0, dirSplitPos)
+        }
+        packageName = packageName.replace(File.separator, ".")
+        return packageName
+    }
+
+    String getClassName() {
+        String scriptFileName = scriptContext.file.name
+        int suffixPos = scriptFileName.lastIndexOf(".")
+        String className = scriptFileName
+        if (suffixPos != -1) {
+            className = scriptFileName.substring(0, suffixPos)
+        }
+        return className
+    }
+
+    // compatible to context.conn
+    Connection getConn() {
+        return getConnection()
    }

    Connection getConnection() {
        def threadConn = threadLocalConn.get()
-        if (threadConn != null) {
-            return threadConn
+        if (threadConn == null) {
+            threadConn = config.getConnection()
+            threadLocalConn.set(threadConn)
        }
-        return this.conn
+        return threadConn
    }

    public <T> T connect(String user, String password, String url, Closure<T> actionSupplier) {
@@ -122,8 +160,40 @@ class SuiteContext implements Closeable {
    void closeThreadLocal() {
        def outputIterator = threadLocalOutputIterator.get()
        if (outputIterator != null) {
-            outputIterator.close()
            threadLocalOutputIterator.remove()
+            try {
+                outputIterator.close()
+            } catch (Throwable t) {
+                log.warn("Close outputIterator failed", t)
+            }
+        }
+
+        Connection conn = threadLocalConn.get()
+        if (conn != null) {
+            threadLocalConn.remove()
+            try {
+                conn.close()
+            } catch (Throwable t) {
+                log.warn("Close connection failed", t)
+            }
+        }
+    }
+
+    public <T> T start(Function<SuiteContext, T> func) {
+        this.startTime = System.currentTimeMillis()
+        scriptContext.eventListeners.each { it.onSuiteStarted(this) }
+
+        this.withCloseable {suiteContext ->
+            try {
+                func.apply(suiteContext)
+            } catch (Throwable t) {
+                if (t instanceof UndeclaredThrowableException) {
+                    t = ((UndeclaredThrowableException) t).undeclaredThrowable
+                }
+                scriptContext.eventListeners.each { it.onSuiteFailed(this, t) }
+                throwable = t
+                null
+            }
        }
    }

@@ -135,10 +205,8 @@ class SuiteContext implements Closeable {
            outputBlocksWriter.close()
        }

-        try {
-            conn.close()
-        } catch (Throwable t) {
-            log.warn("Close connection failed", t)
-        }
+        this.finishTime = System.currentTimeMillis()
+        long elapsed = finishTime - startTime
+        scriptContext.eventListeners.each { it.onSuiteFinished(this, throwable == null, elapsed) }
    }
 }
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.

-package org.apache.doris.regression.util
+package org.apache.doris.regression.suite

 import groovy.transform.CompileStatic

@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.suite
+
+import groovy.transform.CompileStatic
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+@CompileStatic
+abstract class SuiteScript extends Script {
+    public ScriptContext context
+    public final Logger logger = LoggerFactory.getLogger(getClass())
+
+    void init(ScriptContext scriptContext) {
+        this.context = scriptContext
+    }
+
+    void suite(String suiteName, String group = getDefaultGroups(new File(context.config.suitePath), context.file), Closure suiteBody) {
+        if (!context.suiteFilter.call(suiteName, group)) {
+            return
+        }
+
+        try {
+            context.createAndRunSuite(suiteName, group, suiteBody)
+        } catch (Throwable t) {
+            logger.warn("Unexcept exception when run ${suiteName} in ${context.file.absolutePath} failed", t)
+        }
+    }
+
+    static String getDefaultGroups(File suiteRoot, File scriptFile) {
+        String path = suiteRoot.relativePath(scriptFile.parentFile)
+        List<String> groups = ["default"]
+
+        String parentGroup = ""
+
+        path.split(File.separator)
+            .collect {it.trim()}
+            .findAll {it != "." && it != ".." && !it.isEmpty()}
+            .each {
+                String currentGroup = parentGroup + it
+                groups.add(currentGroup)
+                parentGroup = currentGroup + "/"
+            }
+        return groups.join(",")
+    }
+}
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.suite.event
+
+import groovy.transform.CompileStatic
+import org.apache.doris.regression.suite.ScriptContext
+import org.apache.doris.regression.suite.SuiteContext
+
+@CompileStatic
+interface EventListener {
+    void onScriptStarted(ScriptContext scriptContext)
+    void onScriptFailed(ScriptContext scriptContext, Throwable t)
+    void onScriptFinished(ScriptContext scriptContext, long elapsed)
+    void onSuiteStarted(SuiteContext suiteContext)
+    void onSuiteFailed(SuiteContext suiteContext, Throwable t)
+    void onSuiteFinished(SuiteContext suiteContext, boolean success, long elapsed)
+    void onThreadStarted(SuiteContext suiteContext)
+    void onThreadFailed(SuiteContext suiteContext, Throwable t)
+    void onThreadFinished(SuiteContext suiteContext, long elapsed)
+}
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.suite.event
+
+import groovy.transform.CompileStatic
+import org.apache.doris.regression.suite.ScriptContext
+import org.apache.doris.regression.suite.ScriptInfo
+import org.apache.doris.regression.suite.SuiteContext
+import org.apache.doris.regression.suite.SuiteInfo
+import org.apache.doris.regression.util.Recorder
+
+@CompileStatic
+class RecorderEventListener implements EventListener {
+    private final Recorder recorder
+
+    RecorderEventListener(Recorder recorder) {
+        this.recorder = recorder
+    }
+
+    @Override
+    void onScriptStarted(ScriptContext scriptContext) {
+
+    }
+
+    @Override
+    void onScriptFailed(ScriptContext scriptContext, Throwable t) {
+        recorder.onFatal(new ScriptInfo(scriptContext.file))
+    }
+
+    @Override
+    void onScriptFinished(ScriptContext scriptContext, long elapsed) {
+
+    }
+
+    @Override
+    void onSuiteStarted(SuiteContext suiteContext) {
+
+    }
+
+    @Override
+    void onSuiteFailed(SuiteContext suiteContext, Throwable t) {
+        recorder.onFailure(new SuiteInfo(suiteContext.scriptContext.file, suiteContext.group, suiteContext.suiteName))
+    }
+
+    @Override
+    void onSuiteFinished(SuiteContext suiteContext, boolean success, long elapsed) {
+        if (success) {
+            recorder.onSuccess(new SuiteInfo(suiteContext.scriptContext.file, suiteContext.group, suiteContext.suiteName))
+        }
+    }
+
+    @Override
+    void onThreadStarted(SuiteContext suiteContext) {
+
+    }
+
+    @Override
+    void onThreadFailed(SuiteContext suiteContext, Throwable t) {
+
+    }
+
+    @Override
+    void onThreadFinished(SuiteContext suiteContext, long elapsed) {
+
+    }
+}
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.suite.event
+
+import groovy.transform.CompileStatic
+import org.apache.doris.regression.suite.ScriptContext
+import org.apache.doris.regression.suite.SuiteContext
+
+import java.util.function.Consumer
+
+@CompileStatic
+class StackEventListeners implements EventListener {
+    private final Stack<EventListener> listeners = new Stack<>()
+
+    synchronized void addListener(EventListener eventListener) {
+        this.listeners.push(eventListener)
+    }
+
+    synchronized void foreach(Consumer<EventListener> func) {
+        for (int i = listeners.size() - 1; i >= 0; i--) {
+            func.accept(listeners.get(i))
+        }
+    }
+
+    @Override
+    synchronized void onScriptStarted(ScriptContext scriptContext) {
+        foreach {
+            it.onScriptStarted(scriptContext)
+        }
+    }
+
+    @Override
+    synchronized void onScriptFailed(ScriptContext scriptContext, Throwable t) {
+        foreach {
+            it.onScriptFailed(scriptContext, t)
+        }
+    }
+
+    @Override
+    synchronized void onScriptFinished(ScriptContext scriptContext, long elapsed) {
+        foreach {
+            it.onScriptFinished(scriptContext, elapsed)
+        }
+    }
+
+    @Override
+    synchronized void onSuiteStarted(SuiteContext suiteContext) {
+        foreach {
+            it.onSuiteStarted(suiteContext)
+        }
+    }
+
+    @Override
+    synchronized void onSuiteFailed(SuiteContext suiteContext, Throwable t) {
+        foreach {
+            it.onSuiteFailed(suiteContext, t)
+        }
+    }
+
+    @Override
+    synchronized void onSuiteFinished(SuiteContext suiteContext, boolean success, long elapsed) {
+        foreach {
+            it.onSuiteFinished(suiteContext, success, elapsed)
+        }
+    }
+
+    @Override
+    synchronized void onThreadStarted(SuiteContext suiteContext) {
+        foreach {
+            it.onThreadStarted(suiteContext)
+        }
+    }
+
+    @Override
+    synchronized void onThreadFailed(SuiteContext suiteContext, Throwable t) {
+        foreach {
+            it.onThreadFailed(suiteContext, t)
+        }
+    }
+
+    @Override
+    synchronized void onThreadFinished(SuiteContext suiteContext, long elapsed) {
+        foreach {
+            it.onThreadFinished(suiteContext, elapsed)
+        }
+    }
+}
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.suite.event
+
+import com.google.common.base.Throwables
+import groovy.transform.CompileStatic
+import org.apache.doris.regression.logger.TeamcityServiceMessageEncoder
+import org.apache.doris.regression.suite.ScriptContext
+import org.apache.doris.regression.suite.SuiteContext
+import org.apache.doris.regression.util.LoggerUtils
+import org.apache.doris.regression.util.TeamcityUtils
+
+@CompileStatic
+class TeamcityEventListener implements EventListener {
+    @Override
+    void onScriptStarted(ScriptContext scriptContext) {
+        // do nothing
+    }
+
+    @Override
+    void onScriptFailed(ScriptContext scriptContext, Throwable t) {
+        // do nothing
+    }
+
+    @Override
+    void onScriptFinished(ScriptContext scriptContext, long elapsed) {
+        // do nothing
+    }
+
+    @Override
+    void onSuiteStarted(SuiteContext suiteContext) {
+        TeamcityServiceMessageEncoder.CURRENT_SUITE_CONTEXT.set(suiteContext)
+        TeamcityUtils.testStarted(suiteContext)
+    }
+
+    @Override
+    void onSuiteFailed(SuiteContext suiteContext, Throwable t) {
+        def (Integer errorLine, String errorInfo) = LoggerUtils.getErrorInfo(t, suiteContext.scriptContext.file)
+        String errorMsg = errorInfo == null
+                ? "Exception in ${suiteContext.scriptContext.name}:"
+                : "Exception in ${suiteContext.scriptContext.name}(line ${errorLine}):\n\n${errorInfo}\n\nException:"
+        def stackTrace = Throwables.getStackTraceAsString(t)
+        TeamcityUtils.testFailed(suiteContext, errorMsg, stackTrace)
+    }
+
+    @Override
+    void onSuiteFinished(SuiteContext suiteContext, boolean success, long elapsed) {
+        TeamcityServiceMessageEncoder.CURRENT_SUITE_CONTEXT.remove()
+        TeamcityUtils.testFinished(suiteContext, elapsed)
+    }
+
+    @Override
+    void onThreadStarted(SuiteContext suiteContext) {
+        TeamcityServiceMessageEncoder.CURRENT_SUITE_CONTEXT.set(suiteContext)
+    }
+
+    @Override
+    void onThreadFailed(SuiteContext suiteContext, Throwable t) {
+        // do nothing
+    }
+
+    @Override
+    void onThreadFinished(SuiteContext suiteContext, long elapsed) {
+        TeamcityServiceMessageEncoder.CURRENT_SUITE_CONTEXT.remove()
+    }
+}
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+class LoggerUtils {
+    static Tuple2<Integer, String> getErrorInfo(Throwable t, File file) {
+        if (file.name.endsWith(".groovy")) {
+            int lineNumber = -1
+            for (def st : t.getStackTrace()) {
+                if (Objects.equals(st.fileName, file.name)) {
+                    lineNumber = st.getLineNumber()
+                    break
+                }
+            }
+            if (lineNumber == -1) {
+                return new Tuple2<Integer, String>(null, null)
+            }
+
+            List<String> lines = file.text.split("\n").toList()
+            String errorPrefixText = lines.subList(Math.max(0, lineNumber - 10), lineNumber).join("\n")
+            String errorSuffixText = lines.subList(lineNumber, Math.min(lines.size(), lineNumber + 10)).join("\n")
+            String errorText = "${errorPrefixText}\n^^^^^^^^^^^^^^^^^^^^^^^^^^ERROR LINE^^^^^^^^^^^^^^^^^^^^^^^^^^\n${errorSuffixText}".toString()
+            return new Tuple2<Integer, String>(lineNumber, errorText)
+        } else {
+            return new Tuple2<Integer, String>(null, null)
+        }
+    }
+}
@@ -24,8 +24,17 @@ import org.apache.commons.csv.CSVPrinter
 import org.apache.commons.csv.CSVRecord
 import org.apache.commons.io.LineIterator

+import java.util.function.Function
+
@CompileStatic
 class OutputUtils {
+    static String columnToCsvString(Object column) {
+        StringWriter writer = new StringWriter()
+        def printer = new CSVPrinter(new PrintWriter(writer), CSVFormat.MYSQL)
+        printer.print(column)
+        return writer.toString()
+    }
+
    static String toCsvString(List<Object> row) {
        StringWriter writer = new StringWriter()
        def printer = new CSVPrinter(new PrintWriter(writer), CSVFormat.MYSQL)
@@ -35,23 +44,27 @@ class OutputUtils {
        return writer.toString()
    }

-    static String checkOutput(Iterator<List<String>> expect, Iterator<List<Object>> real, String info) {
+    static <T1, T2> String checkOutput(Iterator<T1> expect, Iterator<T2> real,
+                                       Function<T1, String> transform1, Function<T2, String> transform2,
+                                       String info) {
+        int line = 1
        while (true) {
            if (expect.hasNext() && !real.hasNext()) {
-                return "${info}, result mismatch, real line is empty, but expect is ${expect.next()}"
+                return "${info}, line ${line} mismatch, real line is empty, but expect is ${transform1(expect.next())}"
            }
            if (!expect.hasNext() && real.hasNext()) {
-                return "${info}, result mismatch, expect line is empty, but real is ${toCsvString(real.next())}"
+                return "${info}, line ${line} mismatch, expect line is empty, but real is ${transform2(real.next())}"
            }
            if (!expect.hasNext() && !real.hasNext()) {
                break
            }

-            def expectCsvString = toCsvString(expect.next() as List<Object>)
-            def realCsvString = toCsvString(real.next())
+            def expectCsvString = transform1.apply(expect.next())
+            def realCsvString = transform2.apply(real.next())
            if (!expectCsvString.equals(realCsvString)) {
-                return "${info}, result mismatch.\nExpect line is: ${expectCsvString}\nBut real is   : ${realCsvString}"
+                return "${info}, line ${line} mismatch.\nExpect line is: ${expectCsvString}\nBut real is   : ${realCsvString}"
            }
+            line++
        }
    }

@@ -18,11 +18,14 @@
 package org.apache.doris.regression.util

 import groovy.transform.CompileStatic
+import org.apache.doris.regression.suite.ScriptInfo
+import org.apache.doris.regression.suite.SuiteInfo

@CompileStatic
 class Recorder {
    public final List<SuiteInfo> successList = new Vector<>()
    public final List<SuiteInfo> failureList = new Vector<>()
+    public final List<ScriptInfo> fatalScriptList = new Vector<>()

    void onSuccess(SuiteInfo suiteInfo) {
        successList.add(suiteInfo)
@@ -32,7 +35,7 @@ class Recorder {
        failureList.add(suiteInfo)
    }

-    void reportDiffResult(List res) {
-        // TODO
+    void onFatal(ScriptInfo scriptInfo) {
+        fatalScriptList.add(scriptInfo)
    }
 }
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+import groovy.transform.CompileStatic
+import org.apache.doris.regression.suite.ScriptContext
+import org.apache.doris.regression.suite.SuiteContext
+import org.apache.tools.ant.util.DateUtils
+
+@CompileStatic
+class TeamcityUtils {
+    static String formatNow() {
+        return DateUtils.format(System.currentTimeMillis(), "yyyy-MM-dd'T'HH:mm:ss.SSSZ")
+    }
+
+    static String formatStdOut(SuiteContext suiteContext, String msg) {
+        String timestamp = formatNow()
+        return "##teamcity[testStdOut name='${suiteContext.flowName}' out='${escape(msg)}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']"
+    }
+
+    static String formatStdErr(SuiteContext suiteContext, String msg) {
+        String timestamp = formatNow()
+        return "##teamcity[testStdErr name='${suiteContext.flowName}' out='${escape(msg)}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']"
+    }
+
+    static void testStarted(SuiteContext suiteContext) {
+        String timestamp = formatNow()
+        println("##teamcity[flowStarted flowId='${suiteContext.flowId}' timestamp='${timestamp}']")
+        println("##teamcity[testStarted name='${suiteContext.flowName}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']")
+    }
+
+    static void testFailed(SuiteContext suiteContext, String msg, String details) {
+        String timestamp = formatNow()
+        println("##teamcity[testFailed name='${suiteContext.flowName}' message='${escape(msg)}' flowId='${suiteContext.flowId}' details='${escape(details)}' timestamp='${timestamp}']")
+    }
+
+    static void testFinished(SuiteContext suiteContext, long elapsed) {
+        String timestamp = formatNow()
+        println("##teamcity[testFinished name='${suiteContext.flowName}' flowId='${suiteContext.flowId}' duration='${elapsed}' timestamp='${timestamp}']")
+        println("##teamcity[flowFinished flowId='${suiteContext.flowId}' timestamp='${timestamp}']")
+    }
+
+    static String escape(String str) {
+        StringBuilder sb = new StringBuilder()
+        char[] chars = str.toCharArray()
+        for (int i = 0; i < chars.length; ++i) {
+            char c = chars[i]
+
+            switch (c) {
+                case '|': sb.append("||"); break
+                case '\'': sb.append("|'"); break
+                case '[': sb.append("|["); break
+                case ']': sb.append("|]"); break
+                case '\n': sb.append("|n"); break
+                case '\r': sb.append("|r"); break
+                case '\\':
+                    if (i + 5 < chars.length && chars[i + 1] == 'u' && isHex(chars[i + 2]) && isHex(chars[i + 3]) && isHex(chars[i + 4]) && isHex(chars[i + 5])) {
+                        sb.append("|0x")
+                        ++i
+                        break
+                    }
+                default: sb.append(c)
+            }
+        }
+        return sb.toString()
+    }
+
+    static boolean isHex(char c) {
+        return (('0' as char) <= c && c <= ('9' as char)) || (('A' as char) <= c && c <= ('F' as char)) || (('a' as char) <= c && c <= ('f' as char))
+    }
+}
@@ -22,14 +22,24 @@ def suiteContext = context(
    filetypes: ["groovy"]
 )

+def suiteScriptClassName = "org.apache.doris.regression.suite.SuiteScript"
 def suiteClassName = "org.apache.doris.regression.suite.Suite"

+// bind suite
+contributor([suiteContext]) {
+    if (!enclosingCall("suite")) {
+        delegatesTo(findClass(suiteScriptClassName))
+    }
+}
+
 def bindAction = { actionName, actionClassName ->
    def closureBody = context(scope: closureScope(isArg: false))
    contributor([closureBody]) {
-        if (enclosingCall(actionName)) {
-            def actionClass = findClass(actionClassName)
-            delegatesTo(actionClass)
+        if (enclosingCall("suite")) {
+            if (enclosingCall(actionName)) {
+                def actionClass = findClass(actionClassName)
+                delegatesTo(actionClass)
+            }
        }
    }
 }
@@ -40,44 +50,48 @@ bindAction("streamLoad", "org.apache.doris.regression.action.StreamLoadAction")

 // bind qt_xxx and order_qt_xxx methods
 contributor([suiteContext]) {
-    def place = getPlace()
-    if (place == null || !place.getClass().getName().contains("GrReferenceExpressionImpl")) {
-        return
-    }
-    def invokeMethodName = place.getQualifiedReferenceName()
-    if (invokeMethodName == null) {
-        return
-    }
-    if (invokeMethodName.startsWith("qt_") || invokeMethodName.startsWith("order_qt_")) {
-        def suiteClass = findClass(suiteClassName)
-        def quickTestMethods = suiteClass.findMethodsByName("quickTest")
-        method(name: invokeMethodName, bindsTo: quickTestMethods[0])
+    if (enclosingCall("suite")) {
+        def place = getPlace()
+        if (place == null || !place.getClass().getName().contains("GrReferenceExpressionImpl")) {
+            return
+        }
+        def invokeMethodName = place.getQualifiedReferenceName()
+        if (invokeMethodName == null) {
+            return
+        }
+        if (invokeMethodName.startsWith("qt_") || invokeMethodName.startsWith("order_qt_")) {
+            def suiteClass = findClass(suiteClassName)
+            def quickTestMethods = suiteClass.findMethodsByName("quickTest")
+            method(name: invokeMethodName, bindsTo: quickTestMethods[0])
+        }
    }
 }

 contributor([suiteContext]) {
-    // bind assertXxx
-    def assertionsClass = findClass("org.junit.jupiter.api.Assertions")
-    delegatesTo(assertionsClass)
+    if (enclosingCall("suite")) {
+        // bind assertXxx
+        def assertionsClass = findClass("org.junit.jupiter.api.Assertions")
+        delegatesTo(assertionsClass)

-    if (enclosingCall("check") ||
-            (!enclosingCall("test") &&
-            !enclosingCall("explain") &&
-            !enclosingCall("streamLoad"))) {
-        // bind other suite method and field
-        def suiteClass = findClass(suiteClassName)
-        delegatesTo(suiteClass)
+        if (enclosingCall("check") ||
+                (!enclosingCall("test") &&
+                        !enclosingCall("explain") &&
+                        !enclosingCall("streamLoad"))) {
+            // bind other suite method and field
+            def suiteClass = findClass(suiteClassName)
+            delegatesTo(suiteClass)

-        // bind try_xxx
-        suiteClass.methods.each { m ->
-            if (m.isConstructor()) {
-                return
+            // bind try_xxx
+            suiteClass.methods.each { m ->
+                if (m.isConstructor()) {
+                    return
+                }
+                def parameters = m.getParameterList().getParameters().collectEntries { p ->
+                    [p.name, p.getType().getPresentableText()]
+                }
+                def returnType = m.returnType.getPresentableText()
+                method(name: "try_${m.name}", bindsTo: m, params: parameters, type: returnType)
            }
-            def parameters = m.getParameterList().getParameters().collectEntries { p ->
-                [p.name, p.getType().getPresentableText()]
-            }
-            def returnType = m.returnType.getPresentableText()
-            method(name: "try_${m.name}", bindsTo: m, params: parameters, type: returnType)
        }
    }
 }
@@ -19,83 +19,85 @@
 // /testing/trino-product-tests/src/main/resources/sql-tests/testcases/aggregate
 // and modified by Doris.

-def tableName = "datetype"
+suite("aggregate") {
+    def tableName = "datetype"

-sql """ DROP TABLE IF EXISTS ${tableName} """
-sql """
-    CREATE TABLE IF NOT EXISTS ${tableName} (
-        c_bigint bigint,
-        c_double double,
-        c_string string,
-        c_date date,
-        c_timestamp datetime,
-        c_boolean boolean,
-        c_short_decimal decimal(5,2),
-        c_long_decimal decimal(27,9)
-    )
-    DUPLICATE KEY(c_bigint)
-    DISTRIBUTED BY HASH(c_bigint) BUCKETS 1
-    PROPERTIES (
-      "replication_num" = "1"
-    )
-"""
+    sql """ DROP TABLE IF EXISTS ${tableName} """
+    sql """
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                c_bigint bigint,
+                c_double double,
+                c_string string,
+                c_date date,
+                c_timestamp datetime,
+                c_boolean boolean,
+                c_short_decimal decimal(5,2),
+                c_long_decimal decimal(27,9)
+            )
+            DUPLICATE KEY(c_bigint)
+            DISTRIBUTED BY HASH(c_bigint) BUCKETS 1
+            PROPERTIES (
+              "replication_num" = "1"
+            )
+        """

-streamLoad {
-    // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy
-    // db 'regression_test'
-    table tableName
+    streamLoad {
+        // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy
+        // db 'regression_test'
+        table tableName

-    // default label is UUID:
-    // set 'label' UUID.randomUUID().toString()
+        // default label is UUID:
+        // set 'label' UUID.randomUUID().toString()

-    // default column_separator is specify in doris fe config, usually is '\t'.
-    // this line change to ','
-    set 'column_separator', '|'
+        // default column_separator is specify in doris fe config, usually is '\t'.
+        // this line change to ','
+        set 'column_separator', '|'

-    // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
-    // also, you can stream load a http stream, e.g. http://xxx/some.csv
-    file 'datetype.csv'
+        // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
+        // also, you can stream load a http stream, e.g. http://xxx/some.csv
+        file 'datetype.csv'

-    time 10000 // limit inflight 10s
+        time 10000 // limit inflight 10s

-    // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
+        // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows

-    // if declared a check callback, the default check condition will ignore.
-    // So you must check all condition
-    check { result, exception, startTime, endTime ->
-        if (exception != null) {
-            throw exception
+        // if declared a check callback, the default check condition will ignore.
+        // So you must check all condition
+        check { result, exception, startTime, endTime ->
+            if (exception != null) {
+                throw exception
+            }
+            log.info("Stream load result: ${result}".toString())
+            def json = parseJson(result)
+            assertEquals("success", json.Status.toLowerCase())
+            assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+            assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
        }
-        log.info("Stream load result: ${result}".toString())
-        def json = parseJson(result)
-        assertEquals("success", json.Status.toLowerCase())
-        assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
-        assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
    }
-}

-qt_aggregate """ select max(upper(c_string)), min(upper(c_string)) from ${tableName} """
-qt_aggregate """ select avg(c_bigint), avg(c_double) from ${tableName} """
-qt_aggregate """ select avg(distinct c_bigint), avg(distinct c_double) from ${tableName} """
-qt_aggregate """ select count(c_bigint),count(c_double),count(c_string),count(c_date),count(c_timestamp),count(c_boolean) from ${tableName} """
-qt_aggregate """ select count(distinct c_bigint),count(distinct c_double),count(distinct c_string),count(distinct c_date),count(distinct c_timestamp),count(distinct c_boolean) from ${tableName} """
-qt_aggregate """ select max(c_bigint), max(c_double),max(c_string), max(c_date), max(c_timestamp) from ${tableName} """
-qt_aggregate """ select min(c_bigint), min(c_double), min(c_string), min(c_date), min(c_timestamp) from ${tableName} """
-qt_aggregate """ select count(c_string), max(c_double), avg(c_bigint) from ${tableName} """
-qt_aggregate """ select stddev_pop(c_bigint), stddev_pop(c_double) from ${tableName} """
-qt_aggregate """ select stddev_pop(distinct c_bigint), stddev_pop(c_double) from ${tableName} """
-qt_aggregate """ select stddev_pop(c_bigint), stddev_pop(distinct c_double) from ${tableName} """
-qt_aggregate """ select stddev_samp(c_bigint), stddev_samp(c_double) from ${tableName} """
-qt_aggregate """ select stddev_samp(distinct c_bigint), stddev_samp(c_double) from ${tableName} """
-qt_aggregate """ select stddev_samp(c_bigint), stddev_samp(distinct c_double) from ${tableName} """
-qt_aggregate """ select sum(c_bigint), sum(c_double) from ${tableName} """
-qt_aggregate """ select sum(distinct c_bigint), sum(distinct c_double) from ${tableName} """
-qt_aggregate """ select var_pop(c_bigint), var_pop(c_double) from ${tableName} """
-qt_aggregate """ select var_pop(distinct c_bigint), var_pop(c_double) from ${tableName} """
-qt_aggregate """ select var_pop(c_bigint), var_pop(distinct c_double) from ${tableName} """
-qt_aggregate """ select var_samp(c_bigint), var_samp(c_double) from ${tableName} """
-qt_aggregate """ select var_samp(distinct c_bigint), var_samp(c_double) from ${tableName} """
-qt_aggregate """ select var_samp(c_bigint), var_samp(distinct c_double) from ${tableName} """
-qt_aggregate """ select variance(c_bigint), variance(c_double) from ${tableName}  """
-qt_aggregate """ select variance(distinct c_bigint), variance(c_double) from ${tableName}  """
-qt_aggregate """ select variance(c_bigint), variance(distinct c_double) from ${tableName}  """
+    qt_aggregate """ select max(upper(c_string)), min(upper(c_string)) from ${tableName} """
+    qt_aggregate """ select avg(c_bigint), avg(c_double) from ${tableName} """
+    qt_aggregate """ select avg(distinct c_bigint), avg(distinct c_double) from ${tableName} """
+    qt_aggregate """ select count(c_bigint),count(c_double),count(c_string),count(c_date),count(c_timestamp),count(c_boolean) from ${tableName} """
+    qt_aggregate """ select count(distinct c_bigint),count(distinct c_double),count(distinct c_string),count(distinct c_date),count(distinct c_timestamp),count(distinct c_boolean) from ${tableName} """
+    qt_aggregate """ select max(c_bigint), max(c_double),max(c_string), max(c_date), max(c_timestamp) from ${tableName} """
+    qt_aggregate """ select min(c_bigint), min(c_double), min(c_string), min(c_date), min(c_timestamp) from ${tableName} """
+    qt_aggregate """ select count(c_string), max(c_double), avg(c_bigint) from ${tableName} """
+    qt_aggregate """ select stddev_pop(c_bigint), stddev_pop(c_double) from ${tableName} """
+    qt_aggregate """ select stddev_pop(distinct c_bigint), stddev_pop(c_double) from ${tableName} """
+    qt_aggregate """ select stddev_pop(c_bigint), stddev_pop(distinct c_double) from ${tableName} """
+    qt_aggregate """ select stddev_samp(c_bigint), stddev_samp(c_double) from ${tableName} """
+    qt_aggregate """ select stddev_samp(distinct c_bigint), stddev_samp(c_double) from ${tableName} """
+    qt_aggregate """ select stddev_samp(c_bigint), stddev_samp(distinct c_double) from ${tableName} """
+    qt_aggregate """ select sum(c_bigint), sum(c_double) from ${tableName} """
+    qt_aggregate """ select sum(distinct c_bigint), sum(distinct c_double) from ${tableName} """
+    qt_aggregate """ select var_pop(c_bigint), var_pop(c_double) from ${tableName} """
+    qt_aggregate """ select var_pop(distinct c_bigint), var_pop(c_double) from ${tableName} """
+    qt_aggregate """ select var_pop(c_bigint), var_pop(distinct c_double) from ${tableName} """
+    qt_aggregate """ select var_samp(c_bigint), var_samp(c_double) from ${tableName} """
+    qt_aggregate """ select var_samp(distinct c_bigint), var_samp(c_double) from ${tableName} """
+    qt_aggregate """ select var_samp(c_bigint), var_samp(distinct c_double) from ${tableName} """
+    qt_aggregate """ select variance(c_bigint), variance(c_double) from ${tableName}  """
+    qt_aggregate """ select variance(distinct c_bigint), variance(c_double) from ${tableName}  """
+    qt_aggregate """ select variance(c_bigint), variance(distinct c_double) from ${tableName}  """
+}
@@ -1 +1,3 @@
-qt_select1 'select 100, "test", date("2021-01-02")'
+suite("test_select_constant") {
+    qt_select1 'select 100, "test", date("2021-01-02")'
+}
@@ -1,16 +1,19 @@
-def result1 = connect(user = 'admin', password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
-    // execute sql with admin user
-    sql 'select 99 + 1'
-}
+suite("connect_action", "demo") {
+    logger.info("ok")
+    def result1 = connect(user = 'admin', password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        // execute sql with admin user
+        sql 'select 99 + 1'
+    }

-// if not specify <user, password, url>, it will be set to context.config.jdbc<User, Password, Url>
-//
-// user: 'root'
-// password: context.config.jdbcPassword
-// url: context.config.jdbcUrl
-def result2 = connect('root') {
-    // execute sql with root user
-    sql 'select 50 + 50'
-}
+    // if not specify <user, password, url>, it will be set to context.config.jdbc<User, Password, Url>
+    //
+    // user: 'root'
+    // password: context.config.jdbcPassword
+    // url: context.config.jdbcUrl
+    def result2 = connect('root') {
+        // execute sql with root user
+        sql 'select 50 + 50'
+    }

-assertEquals(result1, result2)
+    assertEquals(result1, result2)
+}
@@ -1,40 +1,42 @@
-def createTable = { tableName ->
-    sql """
-        create table ${tableName}
-        (id int)
-        distributed by hash(id)
-        properties
-        (
-          "replication_num"="1"
-        )
-        """
+suite("event_action", "demo") {
+    def createTable = { tableName ->
+        sql """
+            create table ${tableName}
+            (id int)
+            distributed by hash(id)
+            properties
+            (
+              "replication_num"="1"
+            )
+            """
+    }
+
+    def tableName = "test_events_table1"
+    createTable(tableName)
+
+    // lazy drop table when execute this suite finished
+    onFinish {
+        try_sql "drop table if exists ${tableName}"
+    }
+
+
+
+    // all event: success, fail, finish
+    // and you can listen event multiple times
+
+    onSuccess {
+        try_sql "drop table if exists ${tableName}"
+    }
+
+    onSuccess {
+        try_sql "drop table if exists ${tableName}_not_exist"
+    }
+
+    onFail {
+        try_sql "drop table if exists ${tableName}"
+    }
+
+    onFail {
+        try_sql "drop table if exists ${tableName}_not_exist"
+    }
 }
-
-def tableName = "test_events_table1"
-createTable(tableName)
-
-// lazy drop table when execute this suite finished
-onFinish {
-    try_sql "drop table if exists ${tableName}"
-}
-
-
-
-// all event: success, fail, finish
-// and you can listen event multiple times
-
-onSuccess {
-    try_sql "drop table if exists ${tableName}"
-}
-
-onSuccess {
-    try_sql "drop table if exists ${tableName}_not_exist"
-}
-
-onFail {
-    try_sql "drop table if exists ${tableName}"
-}
-
-onFail {
-    try_sql "drop table if exists ${tableName}_not_exist"
-}
@@ -1,31 +1,33 @@
-explain {
-    sql("select 100")
+suite("explain_action", "demo") {
+    explain {
+        sql("select 100")

-    // contains("OUTPUT EXPRS:<slot 0> 100\n") && contains("PARTITION: UNPARTITIONED\n")
-    contains "OUTPUT EXPRS:<slot 0> 100\n"
-    contains "PARTITION: UNPARTITIONED\n"
-}
+        // contains("OUTPUT EXPRS:<slot 0> 100\n") && contains("PARTITION: UNPARTITIONED\n")
+        contains "OUTPUT EXPRS:<slot 0> 100\n"
+        contains "PARTITION: UNPARTITIONED\n"
+    }

-explain {
-    sql("select 100")
+    explain {
+        sql("select 100")

-    // contains(" 100\n") && !contains("abcdefg") && !("1234567")
-    contains " 100\n"
-    notContains "abcdefg"
-    notContains "1234567"
-}
+        // contains(" 100\n") && !contains("abcdefg") && !("1234567")
+        contains " 100\n"
+        notContains "abcdefg"
+        notContains "1234567"
+    }

-explain {
-    sql("select 100")
-    // simple callback
-    check { explainStr -> explainStr.contains("abcdefg") || explainStr.contains(" 100\n") }
-}
+    explain {
+        sql("select 100")
+        // simple callback
+        check { explainStr -> explainStr.contains("abcdefg") || explainStr.contains(" 100\n") }
+    }

-explain {
-    sql("a b c d e")
-    // callback with exception and time
-    check { explainStr, exception, startTime, endTime ->
-        // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
-        assertTrue(exception != null)
+    explain {
+        sql("a b c d e")
+        // callback with exception and time
+        check { explainStr, exception, startTime, endTime ->
+            // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
+            assertTrue(exception != null)
+        }
    }
 }
@@ -1,33 +1,36 @@
-/***** 1. lazy check exceptions *****/
+suite("lazyCheck_action_exceptions", "demo") {
+    /***** 1. lazy check exceptions *****/

-// will not throw exception immediately
-def result = lazyCheck {
-    sql "a b c d e d" // syntax error
+    // will not throw exception immediately
+    def result = lazyCheck {
+        sql "a b c d e d" // syntax error
+    }
+    assertTrue(result == null)
+
+    result = lazyCheck {
+        sql "select 100"
+    }
+    assertEquals(result[0][0], 100)
+
+    logger.info("You will see this log")
+
+    // if you not clear the lazyCheckExceptions, and then,
+    // after this suite execute finished, the syntax error in the lazyCheck action will be thrown.
+    lazyCheckExceptions.clear()
 }
-assertTrue(result == null)

-result = lazyCheck {
-    sql "select 100"
-}
-assertEquals(result[0][0], 100)
+suite("lazyCheck_action_futures", "demo") {
+    /***** 2. lazy check futures *****/

-logger.info("You will see this log")
+    // start new thread and lazy check future
+    def futureResult = lazyCheckThread {
+        sql "a b c d e d f"
+    }
+    assertTrue(futureResult instanceof java.util.concurrent.Future)

-// if you not clear the lazyCheckExceptions, and then,
-// after this suite execute finished, the syntax error in the lazyCheck action will be thrown.
-lazyCheckExceptions.clear()
+    logger.info("You will see this log too")

-
-/***** 2. lazy check futures *****/
-
-// start new thread and lazy check future
-def futureResult = lazyCheckThread {
-    sql "a b c d e d"
-}
-assertTrue(futureResult instanceof java.util.concurrent.Future)
-
-logger.info("You will see this log too")
-
-// if you not clear the lazyCheckFutures, and then,
-// after this suite execute finished, the syntax error in the lazyCheckThread action will be thrown.
-lazyCheckFutures.clear()
+    // if you not clear the lazyCheckFutures, and then,
+    // after this suite execute finished, the syntax error in the lazyCheckThread action will be thrown.
+    lazyCheckFutures.clear()
+}
@@ -1,19 +1,20 @@
-/**
- * qt_xxx sql equals to quickTest(xxx, sql) witch xxx is tag.
- * the result will be compare to the relate file: ${DORIS_HOME}/regression_test/data/qt_action.out.
- *
- * if you want to generate .out tsv file for real execute result. you can run with -genOut or -forceGenOut option.
- * e.g
- *   ${DORIS_HOME}/run-regression-test.sh --run qt_action -genOut
- *   ${DORIS_HOME}/run-regression-test.sh --run qt_action -forceGenOut
- */
-qt_select "select 1, 'beijing' union all select 2, 'shanghai'"
+suite("qt_action", "demo") {
+    /**
+     * qt_xxx sql equals to quickTest(xxx, sql) witch xxx is tag.
+     * the result will be compare to the relate file: ${DORIS_HOME}/regression_test/data/qt_action.out.
+     *
+     * if you want to generate .out tsv file for real execute result. you can run with -genOut or -forceGenOut option.
+     * e.g
+     *   ${DORIS_HOME}/run-regression-test.sh --run qt_action -genOut
+     *   ${DORIS_HOME}/run-regression-test.sh --run qt_action -forceGenOut
+     */
+    qt_select "select 1, 'beijing' union all select 2, 'shanghai'"

-qt_select2 "select 2"
+    qt_select2 "select 2"

-// order result by string dict then compare to .out file.
-// order_qt_xxx sql equals to quickTest(xxx, sql, true).
-order_qt_union_all  """
+    // order result by string dict then compare to .out file.
+    // order_qt_xxx sql equals to quickTest(xxx, sql, true).
+    order_qt_union_all  """
                select 2
                union all
                select 1
@@ -23,4 +24,5 @@ order_qt_union_all  """
                select 15
                union all
                select 3
-                """
+                """
+}
@@ -1,6 +1,7 @@
-// 3 rows and 1 column
-def rows = [3, 1, 10]
-order_qt_select_union_all1 """
+suite("select_union_all_action", "demo") {
+    // 3 rows and 1 column
+    def rows = [3, 1, 10]
+    order_qt_select_union_all1 """
            select c1
            from
            (
@@ -8,12 +9,13 @@ order_qt_select_union_all1 """
            ) a
            """

-// 3 rows and 2 columns
-rows = [[1, "123"], [2, null], [0, "abc"]]
-order_qt_select_union_all2 """
+    // 3 rows and 2 columns
+    rows = [[1, "123"], [2, null], [0, "abc"]]
+    order_qt_select_union_all2 """
             select c1, c2
             from
             (
                ${selectUnionAll(rows)}
             ) b
-             """
+             """
+}
@@ -1,27 +1,28 @@
-// execute sql and ignore result
-sql "show databases"
+suite("sql_action", "demo") {
+    // execute sql and ignore result
+    sql "show databases"

-// execute sql and get result, outer List denote rows, inner List denote columns in a single row
-List<List<Object>> tables = sql "show tables"
+    // execute sql and get result, outer List denote rows, inner List denote columns in a single row
+    List<List<Object>> tables = sql "show tables"

-// assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
-assertTrue(tables.size() >= 0) // test rowCount >= 0
+    // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
+    assertTrue(tables.size() >= 0) // test rowCount >= 0

-// syntax error
-try {
-    sql "a b c d e"
-    throw new IllegalStateException("Should be syntax error")
-} catch (java.sql.SQLException t) {
-    assertTrue(true)
-}
+    // syntax error
+    try {
+        sql "a b c d e"
+        throw new IllegalStateException("Should be syntax error")
+    } catch (java.sql.SQLException t) {
+        assertTrue(true)
+    }

-def testTable = "test_sql_action1"
+    def testTable = "test_sql_action1"

-try {
-    sql "DROP TABLE IF EXISTS ${testTable}"
+    try {
+        sql "DROP TABLE IF EXISTS ${testTable}"

-    // multi-line sql
-    def result1 = sql """
+        // multi-line sql
+        def result1 = sql """
                        CREATE TABLE IF NOT EXISTS ${testTable} (
                            id int
                        )
@@ -31,40 +32,40 @@ try {
                        ) 
                        """

-    // DDL/DML return 1 row and 1 column, the only value is update row count
-    assertTrue(result1.size() == 1)
-    assertTrue(result1[0].size() == 1)
-    assertTrue(result1[0][0] == 0, "Create table should update 0 rows")
+        // DDL/DML return 1 row and 1 column, the only value is update row count
+        assertTrue(result1.size() == 1)
+        assertTrue(result1[0].size() == 1)
+        assertTrue(result1[0][0] == 0, "Create table should update 0 rows")

-    def result2 = sql "INSERT INTO test_sql_action1 values(1), (2), (3)"
-    assertTrue(result2.size() == 1)
-    assertTrue(result2[0].size() == 1)
-    assertTrue(result2[0][0] == 3, "Insert should update 3 rows")
-} finally {
-    /**
-     * try_xxx(args) means:
-     *
-     * try {
-     *    return xxx(args)
-     * } catch (Throwable t) {
-     *     // do nothing
-     *     return null
-     * }
-     */
-    try_sql("DROP TABLE IF EXISTS ${testTable}")
+        def result2 = sql "INSERT INTO test_sql_action1 values(1), (2), (3)"
+        assertTrue(result2.size() == 1)
+        assertTrue(result2[0].size() == 1)
+        assertTrue(result2[0][0] == 3, "Insert should update 3 rows")
+    } finally {
+        /**
+         * try_xxx(args) means:
+         *
+         * try {
+         *    return xxx(args)
+         * } catch (Throwable t) {
+         *     // do nothing
+         *     return null
+         * }
+         */
+        try_sql("DROP TABLE IF EXISTS ${testTable}")

-    // you can see the error sql will not throw exception and return
-    try {
-        def errorSqlResult = try_sql("a b c d e f g")
-        assertTrue(errorSqlResult == null)
-    } catch (Throwable t) {
-        assertTrue(false, "Never catch exception")
+        // you can see the error sql will not throw exception and return
+        try {
+            def errorSqlResult = try_sql("a b c d e f g")
+            assertTrue(errorSqlResult == null)
+        } catch (Throwable t) {
+            assertTrue(false, "Never catch exception")
+        }
    }
-}

-// order_sql(sqlStr) equals to sql(sqlStr, isOrder=true)
-// sort result by string dict
-def list = order_sql """
+    // order_sql(sqlStr) equals to sql(sqlStr, isOrder=true)
+    // sort result by string dict
+    def list = order_sql """
                select 2
                union all
                select 1
@@ -76,8 +77,9 @@ def list = order_sql """
                select 3
                """

-assertEquals(null, list[0][0])
-assertEquals(1, list[1][0])
-assertEquals(15, list[2][0])
-assertEquals(2, list[3][0])
-assertEquals(3, list[4][0])
+    assertEquals(null, list[0][0])
+    assertEquals(1, list[1][0])
+    assertEquals(15, list[2][0])
+    assertEquals(2, list[3][0])
+    assertEquals(3, list[4][0])
+}
@@ -1,61 +1,64 @@
-def tableName = "test_streamload_action1"
+suite("streamLoad_action", "demo") {

-sql """
-    CREATE TABLE IF NOT EXISTS ${tableName} (
-        id int,
-        name varchar(255)
-    )
-    DISTRIBUTED BY HASH(id) BUCKETS 1
-    PROPERTIES (
-      "replication_num" = "1"
-    ) 
-"""
+    def tableName = "test_streamload_action1"

-streamLoad {
-    // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy
-    // db 'regression_test'
-    table tableName
+    sql """
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                id int,
+                name varchar(255)
+            )
+            DISTRIBUTED BY HASH(id) BUCKETS 1
+            PROPERTIES (
+              "replication_num" = "1"
+            ) 
+        """

-    // default label is UUID:
-    // set 'label' UUID.randomUUID().toString()
+    streamLoad {
+        // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy
+        // db 'regression_test'
+        table tableName

-    // default column_separator is specify in doris fe config, usually is '\t'.
-    // this line change to ','
-    set 'column_separator', ','
+        // default label is UUID:
+        // set 'label' UUID.randomUUID().toString()

-    // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
-    // also, you can stream load a http stream, e.g. http://xxx/some.csv
-    file 'streamload_input.csv'
+        // default column_separator is specify in doris fe config, usually is '\t'.
+        // this line change to ','
+        set 'column_separator', ','

-    time 10000 // limit inflight 10s
+        // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
+        // also, you can stream load a http stream, e.g. http://xxx/some.csv
+        file 'streamload_input.csv'

-    // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
-}
+        time 10000 // limit inflight 10s

-
-// stream load 100 rows
-def rowCount = 100
-// range: [0, rowCount)
-// or rangeClosed: [0, rowCount]
-def rowIt = range(0, rowCount)
-        .mapToObj({i -> [i, "a_" + i]}) // change Long to List<Long, String>
-        .iterator()
-
-streamLoad {
-    table tableName
-    // also, you can upload a memory iterator
-    inputIterator rowIt
-
-    // if declared a check callback, the default check condition will ignore.
-    // So you must check all condition
-    check { result, exception, startTime, endTime ->
-        if (exception != null) {
-            throw exception
-        }
-        log.info("Stream load result: ${result}".toString())
-        def json = parseJson(result)
-        assertEquals("success", json.Status.toLowerCase())
-        assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
-        assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+        // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
    }
-}
+
+
+    // stream load 100 rows
+    def rowCount = 100
+    // range: [0, rowCount)
+    // or rangeClosed: [0, rowCount]
+    def rowIt = range(0, rowCount)
+            .mapToObj({i -> [i, "a_" + i]}) // change Long to List<Long, String>
+            .iterator()
+
+    streamLoad {
+        table tableName
+        // also, you can upload a memory iterator
+        inputIterator rowIt
+
+        // if declared a check callback, the default check condition will ignore.
+        // So you must check all condition
+        check { result, exception, startTime, endTime ->
+            if (exception != null) {
+                throw exception
+            }
+            log.info("Stream load result: ${result}".toString())
+            def json = parseJson(result)
+            assertEquals("success", json.Status.toLowerCase())
+            assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+            assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+        }
+    }
+}
@@ -1,11 +1,12 @@
-test {
-    sql "abcdefg"
-    // check exception message contains
-    exception "errCode = 2, detailMessage = Syntax error"
-}
+suite("test_action", "demo") {
+    test {
+        sql "abcdefg"
+        // check exception message contains
+        exception "errCode = 2, detailMessage = Syntax error"
+    }

-test {
-    sql """
+    test {
+        sql """
            select *
            from (
                select 1 id
@@ -14,33 +15,33 @@ test {
            ) a
            order by id"""

-    // multi check condition
+        // multi check condition

-    // check return 2 rows
-    rowNum 2
-    // execute time must <= 5000 millisecond
-    time 5000
-    // check result, must be 2 rows and 1 column, the first row is 1, second is 2
-    result(
-        [[1], [2]]
-    )
-}
-
-test {
-    sql "a b c d e f g"
-
-    // other check will not work because already declared a check callback
-    exception "aaaaaaaaa"
-
-    // callback
-    check { result, exception, startTime, endTime ->
-        // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
-        assertTrue(exception != null)
+        // check return 2 rows
+        rowNum 2
+        // execute time must <= 5000 millisecond
+        time 5000
+        // check result, must be 2 rows and 1 column, the first row is 1, second is 2
+        result(
+            [[1], [2]]
+        )
    }
-}

-test {
-    sql  """
+    test {
+        sql "a b c d e f g"
+
+        // other check will not work because already declared a check callback
+        exception "aaaaaaaaa"
+
+        // callback
+        check { result, exception, startTime, endTime ->
+            // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically
+            assertTrue(exception != null)
+        }
+    }
+
+    test {
+        sql  """
                select 2
                union all
                select 1
@@ -52,14 +53,33 @@ test {
                select 3
                """

-    check { result, ex, startTime, endTime ->
-        // same as order_sql(sqlStr)
-        result = sortRows(result)
+        check { result, ex, startTime, endTime ->
+            // same as order_sql(sqlStr)
+            result = sortRows(result)

-        assertEquals(null, result[0][0])
-        assertEquals(1, result[1][0])
-        assertEquals(15, result[2][0])
-        assertEquals(2, result[3][0])
-        assertEquals(3, result[4][0])
+            assertEquals(null, result[0][0])
+            assertEquals(1, result[1][0])
+            assertEquals(15, result[2][0])
+            assertEquals(2, result[3][0])
+            assertEquals(3, result[4][0])
+        }
    }
-}
+
+    // execute sql and order query result, then compare to iterator
+    def selectValues = [1, 2, 3, 4]
+    test {
+        order true
+        sql selectUnionAll(selectValues)
+        resultIterator(selectValues.iterator())
+    }
+
+    // compare to data/demo/test_action.csv
+    test {
+        order true
+        sql selectUnionAll(selectValues)
+
+        // you can set to http://xxx or https://xxx
+        // and compare to http response body
+        resultFile "test_action.csv"
+    }
+}
@@ -0,0 +1,5 @@
+select 200
+union all
+select 100
+union all
+select 300
@@ -0,0 +1,5 @@
+select 200
+union all
+select 100
+union all
+select 300
@@ -1,48 +1,50 @@
-def (_, elapsedMillis) = timer {
-    /**
-     * the default max thread num is 10, you can specify by 'actionParallel' param.
-     * e.g. ./run-regression-test.sh --run someSuite -actionParallel 10
-     */
-    def future1 = thread("threadName1") {
-        sleep(200)
-        sql"select 1"
+suite("thread_action", "demo") {
+    def (_, elapsedMillis) = timer {
+        /**
+         * the default max thread num is 10, you can specify by 'actionParallel' param.
+         * e.g. ./run-regression-test.sh --run someSuite -actionParallel 10
+         */
+        def future1 = thread("threadName1") {
+            sleep(200)
+            sql"select 1"
+        }
+
+        // create new thread but not specify name
+        def future2 = thread {
+            sleep(200)
+            sql "select 2"
+        }
+
+        def future3 = thread("threadName3") {
+            sleep(200)
+            sql "select 3"
+        }
+
+        def future4 = thread {
+            sleep(200)
+            sql "select 4"
+        }
+
+        // equals to combineFutures([future1, future2, future3, future4]), which [] is a Iterable<ListenableFuture>
+        def combineFuture = combineFutures(future1, future2, future3, future4)
+        // or you can use lazyCheckThread action(see lazyCheck_action.groovy), and not have to check exception from futures.
+        List<List<List<Object>>> result = combineFuture.get()
+        assertEquals(result[0][0][0], 1)
+        assertEquals(result[1][0][0], 2)
+        assertEquals(result[2][0][0], 3)
+        assertEquals(result[3][0][0], 4)
+    }
+    assertTrue(elapsedMillis < 600)
+
+
+    // you can use qt action in thread action, and you **MUST** specify different tag,
+    // testing framework can compare different qt result in different order.
+    lazyCheckThread {
+        sleep(100)
+        qt_diffrent_tag1 "select 100"
    }

-    // create new thread but not specify name
-    def future2 = thread {
-        sleep(200)
-        sql "select 2"
+    lazyCheckThread("lazyCheckThread2") {
+        qt_diffrent_tag2 "select 100"
    }
-
-    def future3 = thread("threadName3") {
-        sleep(200)
-        sql "select 3"
-    }
-
-    def future4 = thread {
-        sleep(200)
-        sql "select 4"
-    }
-
-    // equals to combineFutures([future1, future2, future3, future4]), which [] is a Iterable<ListenableFuture>
-    def combineFuture = combineFutures(future1, future2, future3, future4)
-    // or you can use lazyCheckThread action(see lazyCheck_action.groovy), and not have to check exception from futures.
-    List<List<List<Object>>> result = combineFuture.get()
-    assertEquals(result[0][0][0], 1)
-    assertEquals(result[1][0][0], 2)
-    assertEquals(result[2][0][0], 3)
-    assertEquals(result[3][0][0], 4)
 }
-assertTrue(elapsedMillis < 600)
-
-
-// you can use qt action in thread action, and you **MUST** specify different tag,
-// testing framework can compare different qt result in different order.
-lazyCheckThread {
-    sleep(100)
-    qt_diffrent_tag1 "select 100"
-}
-
-lazyCheckThread("lazyCheckThread2") {
-    qt_diffrent_tag2 "select 100"
-}
@@ -1,7 +1,13 @@
-def (sumResult, elapsedMillis) = timer {
-    long sum = 0
-    (1..10000).each {sum += it}
-    sum // return value
-}
+suite("timer_action", "demo") {
+    def (sumResult, elapsedMillis) = timer {
+        long sum = 0
+        (1..10000).each {sum += it}
+        sum // return value
+    }

-logger.info("sum: ${sumResult}, elapsed: ${elapsedMillis} ms")
+    // you should convert GString to String because the log of GString not contains the correct file name,
+    // e.g. (NativeMethodAccessorImpl.java:-2) - sum: 50005000, elapsed: 47 ms
+    // so, invoke toString and then we can get the log:
+    // (timer_action.groovy:10) - sum: 50005000, elapsed: 47 ms
+    logger.info("sum: ${sumResult}, elapsed: ${elapsedMillis} ms".toString())
+}
@@ -20,12 +20,14 @@
 // /testing/trino-product-tests/src/main/resources/sql-tests/testcases
 // and modified by Doris.

-def tables=["empty"]
+suite("load") {
+    def tables=["empty"]

-for (String table in tables) {
-    sql """ DROP TABLE IF EXISTS $table """
-}
+    for (String table in tables) {
+        sql """ DROP TABLE IF EXISTS $table """
+    }

-for (String table in tables) {
-    sql new File("""${context.file.parent}/ddl/${table}.sql""").text
+    for (String table in tables) {
+        sql new File("""${context.file.parent}/ddl/${table}.sql""").text
+    }
 }
@@ -20,15 +20,17 @@
 // /testing/trino-product-tests/src/main/resources/sql-tests/testcases
 // and modified by Doris.

-def tables=["test_join"]
+suite("load") {
+    def tables=["test_join"]

-for (String table in tables) {
-    sql """ DROP TABLE IF EXISTS $table """
+    for (String table in tables) {
+        sql """ DROP TABLE IF EXISTS $table """
+    }
+
+    for (String table in tables) {
+        sql new File("""${context.file.parent}/ddl/${table}.sql""").text
+    }
+
+    sql """ insert into test_join select 1 """
+    sql """ insert into test_join select 2 """
 }
-
-for (String table in tables) {
-    sql new File("""${context.file.parent}/ddl/${table}.sql""").text
-}
-
-sql """ insert into test_join select 1 """
-sql """ insert into test_join select 2 """
@@ -1,7 +1,8 @@
-def tableName = "test_streamload_performance1"
+suite("test_streamload_perfomance", "performance") {
+    def tableName = "test_streamload_performance1"

-try {
-    sql """
+    try {
+        sql """
        CREATE TABLE IF NOT EXISTS ${tableName} (
            id int,
            name varchar(255)
@@ -12,16 +13,17 @@ try {
        ) 
        """

-    def rowCount = 10000
-    def rowIt = java.util.stream.LongStream.range(0, rowCount)
-            .mapToObj({i -> [i, "a_" + i]})
-            .iterator()
+        def rowCount = 10000
+        def rowIt = java.util.stream.LongStream.range(0, rowCount)
+                .mapToObj({i -> [i, "a_" + i]})
+                .iterator()

-    streamLoad {
-        table tableName
-        time 5000
-        inputIterator rowIt
+        streamLoad {
+            table tableName
+            time 5000
+            inputIterator rowIt
+        }
+    } finally {
+        try_sql "DROP TABLE IF EXISTS ${tableName}"
    }
-} finally {
-    try_sql "DROP TABLE IF EXISTS ${tableName}"
-}
+}
@@ -15,14 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.

-def tables=["bitmap_basic_agg","hll_basic_agg"]
+suite("basic_agg_test") {
+    def tables=["bitmap_basic_agg","hll_basic_agg"]

-for (String table in tables) {
-    sql """drop table if exists ${table};"""
-    sql new File("""regression-test/common/table/${table}.sql""").text
-    sql new File("""regression-test/common/load/${table}.sql""").text
-}
+    for (String table in tables) {
+        sql """drop table if exists ${table};"""
+        sql new File("""regression-test/common/table/${table}.sql""").text
+        sql new File("""regression-test/common/load/${table}.sql""").text
+    }

-qt_sql_bitmap """select * from bitmap_basic_agg;"""
+    qt_sql_bitmap """select * from bitmap_basic_agg;"""

-qt_sql_hll """select * from hll_basic_agg;"""
+    qt_sql_hll """select * from hll_basic_agg;"""
+}
@@ -21,6 +21,7 @@
 # Usage: $0 <shell_options> <framework_options>
 #  Optional shell_options:
 #     --clean      clean output of regression test
+#     --teamcity   print teamcity service messages
 #     --run        run regression test. build framework if necessary
 #
 #  Optional framework_options
@@ -49,6 +50,7 @@ usage() {
 Usage: $0 <shell_options> <framework_options>
  Optional shell_options:
     --clean    clean output of regression test framework
+     --teamcity print teamcity service messages
     --run      run regression test. build framework if necessary

  Optional framework_options:
@@ -68,9 +70,11 @@ Usage: $0 <shell_options> <framework_options>
    $0 --run -s test_select            run a suite which named as test_select
    $0 --run test_select -genOut       generate output file for test_select if not exist
    $0 --run -g default                run all suite in the group which named as default
+    $0 --run -d demo,correctness/tmp   run all suite in the directories which named as demo and correctness/tmp
    $0 --clean                         clean output of regression test framework
    $0 --clean --run test_select       clean output and build regression test framework and run a suite which named as test_select
    $0 --run -h                        print framework options
+    $0 --teamcity --run test_select    print teamcity service messages and build regression test framework and run test_select

 Log path: \${DORIS_HOME}/output/regression-test/log
 Default config file: \${DORIS_HOME}/regression-test/conf/regression-conf.groovy
@@ -80,20 +84,24 @@ Default config file: \${DORIS_HOME}/regression-test/conf/regression-conf.groovy

 CLEAN=
 WRONG_CMD=
+TEAMCITY=
 RUN=
 if [ $# == 0 ] ; then
    #default
    CLEAN=0
    WRONG_CMD=0
+    TEAMCITY=0
    RUN=1
 else
    CLEAN=0
    RUN=0
+    TEAMCITY=0
    WRONG_CMD=0
    while true; do
        case "$1" in
-            --clean) CLEAN=1 ; shift ;;
-            --run) RUN=1 ; shift ;;
+            --clean)      CLEAN=1 ; shift ;;
+            --teamcity)   TEAMCITY=1 ; shift ;;
+            --run)        RUN=1 ; shift ;;
            *)
                if [ ${RUN} -eq 0 ] && [ ${CLEAN} -eq 0 ]; then
                    WRONG_CMD=1
@@ -171,10 +179,15 @@ fi

 echo "===== Run Regression Test ====="

+JAVA_OPTS=${JAVA_OPTS}
+if [ ${TEAMCITY} -eq 1 ]; then
+  JAVA_OPTS="$JAVA_OPTS -DstdoutAppenderType=teamcity"
+fi
+
 $JAVA -DDORIS_HOME=$DORIS_HOME \
      -DLOG_PATH=$LOG_OUTPUT_FILE \
      -Dlogback.configurationFile=${LOG_CONFIG_FILE} \
-      -Xmx2048m \
+      ${JAVA_OPTS} \
      -jar ${RUN_JAR} \
      -cf ${CONFIG_FILE} \
      ${REGRESSION_OPTIONS_PREFIX} "$@"