Feat: Add Apache Doris engine adapter with OLAP and multi-catalog lakehouse support

.github/scripts/install-prerequisites.sh

@@ -28,6 +28,8 @@ elif [ "$ENGINE" == "fabric" ]; then
     rm packages-microsoft-prod.deb
 
     ENGINE_DEPENDENCIES="msodbcsql18"
+elif [ "$ENGINE" == "doris" ]; then
+    sudo sysctl -w vm.max_map_count=2000000
 fi
 
 ALL_DEPENDENCIES="$COMMON_DEPENDENCIES $ENGINE_DEPENDENCIES"

.github/scripts/wait-for-db.sh

@@ -24,8 +24,15 @@ function_exists() {
 
 probe_port() {
     HOSTNAME=${DOCKER_HOSTNAME:-localhost}
-    echo "Probing '$HOSTNAME' on port $1"
-    while ! nc -z $HOSTNAME $1; do
+    local port="$1"
+    local deadline="$2"
+    local timeout_secs="$3"
+    echo "Probing '$HOSTNAME' on port $port"
+    while ! nc -z "$HOSTNAME" "$port"; do
+        if [ -n "$deadline" ] && [ "$SECONDS" -ge "$deadline" ]; then
+            echo "ERROR: Timed out after ${timeout_secs:-0}s waiting for '$HOSTNAME' on port $port" >&2
+            return 1
+        fi
         sleep 1
     done
 }
@@ -104,6 +111,115 @@ db2_ready() {
     done
 }
 
+doris_ready() {
+    if ! command -v timeout > /dev/null 2>&1; then
+        echo "ERROR: 'timeout' is required by wait-for-db.sh (GNU coreutils; on macOS install coreutils and add it to PATH)" >&2
+        exit 1
+    fi
+    local timeout_secs="${DORIS_STARTUP_TIMEOUT_SECS:-300}"
+    local deadline=$((SECONDS + timeout_secs))
+    local remaining=0
+    local backend_registered=0
+    local reg_out=""
+
+    if ! probe_port 9030 "$deadline" "$timeout_secs"; then
+        echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris FE port 9030 to open" >&2
+        exit 1
+    fi
+
+    echo "Waiting for Doris FE to accept SQL connections..."
+    while true; do
+        remaining=$((deadline - SECONDS))
+        if [ "$remaining" -le 0 ]; then
+            echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris FE to become alive" >&2
+            exit 1
+        fi
+        if timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "SHOW FRONTENDS\G" 2>/dev/null | grep -q "Alive: true"; then
+            echo "Doris FE is alive"
+            break
+        fi
+        if [ "$SECONDS" -ge "$deadline" ]; then
+            echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris FE to become alive" >&2
+            exit 1
+        fi
+        sleep 2
+    done
+
+    echo "Checking for 1 alive Doris backends..."
+    while true; do
+        if [ "$backend_registered" -eq 0 ]; then
+            remaining=$((deadline - SECONDS))
+            if [ "$remaining" -le 0 ]; then
+                echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend registration: $reg_out" >&2
+                exit 1
+            fi
+            if reg_out=$(timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '172.30.80.3:9050';" 2>&1); then
+                backend_registered=1
+            elif echo "$reg_out" | grep -Eqi "already exists|already registered"; then
+                backend_registered=1
+            else
+                echo "WARN: Failed to register Doris backend (will retry): $reg_out" >&2
+            fi
+        fi
+
+        echo "Checking Doris backends..."
+        remaining=$((deadline - SECONDS))
+        if [ "$remaining" -le 0 ]; then
+            if [ "$backend_registered" -eq 0 ] && [ -n "$reg_out" ]; then
+                echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend registration: $reg_out" >&2
+            else
+                echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend to become alive" >&2
+            fi
+            exit 1
+        fi
+        ALIVE_BACKENDS=$(timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "show backends \G" 2>/dev/null | grep -c "^ *Alive: true *$" || true)
+
+        # fallback value if failed to get number
+        if ! [[ "$ALIVE_BACKENDS" =~ ^[0-9]+$ ]]; then
+            echo "WARN: Unable to parse number of alive backends, got: '$ALIVE_BACKENDS'"
+            ALIVE_BACKENDS=0
+        fi
+
+        echo "Found $ALIVE_BACKENDS alive backends"
+
+        if [ "$ALIVE_BACKENDS" -ge 1 ]; then
+            echo "Doris has 1 or more alive backends"
+            break
+        fi
+
+        if [ "$SECONDS" -ge "$deadline" ]; then
+            if [ "$backend_registered" -eq 0 ] && [ -n "$reg_out" ]; then
+                echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend registration: $reg_out" >&2
+            else
+                echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend to become alive" >&2
+            fi
+            exit 1
+        fi
+
+        echo "Waiting for more backends to become alive..."
+        sleep 3
+    done
+
+    # set default replication num to 1 (there is only one be in the docker compose file)
+    cfg_out=""
+    while true; do
+        remaining=$((deadline - SECONDS))
+        if [ "$remaining" -le 0 ]; then
+            echo "ERROR: Timed out after ${timeout_secs}s configuring Doris frontend: $cfg_out" >&2
+            exit 1
+        fi
+        if cfg_out=$(timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ADMIN SET FRONTEND CONFIG ('force_olap_table_replication_num' = '1');" 2>&1); then
+            break
+        fi
+        if [ "$SECONDS" -ge "$deadline" ]; then
+            echo "ERROR: Timed out after ${timeout_secs}s configuring Doris frontend: $cfg_out" >&2
+            exit 1
+        fi
+        echo "WARN: Failed to configure Doris frontend (will retry): $cfg_out" >&2
+        sleep 2
+    done
+}
+
 echo "Waiting for $ENGINE to be ready..."
 
 READINESS_FUNC="${ENGINE}_ready"

.github/workflows/pr.yaml

@@ -274,6 +274,7 @@ jobs:
             clickhouse-datalake,
             risingwave,
             starrocks,
+            doris,
           ]
     env:
       PYTEST_XDIST_AUTO_NUM_WORKERS: 2

.readthedocs.yaml

@@ -6,7 +6,7 @@ build:
     python: "3.10"
   jobs:
     pre_build:
-      - pip install -e ".[athena,azuresql,bigframes,bigquery,clickhouse,databricks,dbt,dlt,gcppostgres,github,llm,mssql,mysql,mwaa,postgres,redshift,slack,snowflake,starrocks,trino,web,risingwave]"
+      - pip install -e ".[athena,azuresql,bigframes,bigquery,clickhouse,databricks,dbt,dlt,doris,gcppostgres,github,llm,mssql,mysql,mwaa,postgres,redshift,slack,snowflake,starrocks,trino,web,risingwave]"
       - make api-docs
 
 mkdocs:

Makefile

@@ -184,6 +184,7 @@ engine-%-install:
 	$(PIP) install -e ".[dev,web,slack,lsp,${*}]" ./examples/custom_materializations
 
 engine-docker-%-up:
+	@if [ "${*}" = "doris" ]; then $(MAKE) doris-host-setup; fi
 	docker compose -f ./tests/core/engine_adapter/integration/docker/compose.${*}.yaml up -d
 	./.github/scripts/wait-for-db.sh ${*}
 
@@ -242,6 +243,20 @@ starrocks-test: engine-starrocks-up
 db2-test: engine-db2-up
 	pytest -n auto -m "db2" --reruns 3 --junitxml=test-results/junit-db2.xml
 
+doris-host-setup:
+	@if [ "$$(uname -s)" = "Linux" ]; then \
+		current=$$(sysctl -n vm.max_map_count 2>/dev/null || echo 0); \
+		if [ "$$current" -lt 2000000 ]; then \
+			sudo -n sysctl -w vm.max_map_count=2000000 >/dev/null 2>&1 || { \
+				echo "ERROR: vm.max_map_count is $$current (< 2000000); run 'sudo sysctl -w vm.max_map_count=2000000' before starting Doris."; \
+				exit 1; \
+			}; \
+		fi; \
+	fi
+
+doris-test: doris-host-setup engine-doris-up
+	pytest -n auto -m "doris" --reruns 3 --junitxml=test-results/junit-doris.xml
+
 #################
 # Cloud Engines #
 #################

docs/concepts/models/model_kinds.md

@@ -444,6 +444,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_TIME_RANGE` kind a
 | Redshift   | DELETE by time range, then INSERT         |
 | Postgres   | DELETE by time range, then INSERT         |
 | DuckDB     | DELETE by time range, then INSERT         |
+| Doris      | DELETE by time range, then INSERT (internal OLAP & Doris 4.1+ Iceberg v2; not supported on Hive — non-atomic: concurrent readers may observe a gap and failed inserts leave the deleted interval missing; see [Doris catalog matrix & recovery](../../integrations/engines/doris.md#supported-model-kinds-catalog-matrix)) |
 
 ## INCREMENTAL_BY_UNIQUE_KEY
 
@@ -669,6 +670,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_UNIQUE_KEY` kind a
 | Redshift   | MERGE ON unique key                 |
 | Postgres   | MERGE ON unique key                 |
 | DuckDB     | DELETE ON matched + INSERT new rows |
+| Doris      | DELETE ON matched + INSERT new rows (internal OLAP & Doris 4.1+ Iceberg v2; not supported on Hive — non-atomic: concurrent readers may observe a gap and failed inserts leave deleted keys missing; see [Doris catalog matrix & recovery](../../integrations/engines/doris.md#supported-model-kinds-catalog-matrix); or single-statement native MERGE when `when_matched` / `merge_filter` is set on Doris 4.1+ internal OLAP `UNIQUE KEY` tables — external catalogs and `DUPLICATE KEY` / `AGGREGATE KEY` tables are not supported for native MERGE; direct INSERT when a sequence column is configured) |
 
 ## FULL
 Models of the `FULL` kind cause the dataset associated with a model to be fully refreshed (rewritten) upon each model evaluation.
@@ -761,6 +763,7 @@ Depending on the target engine, models of the `FULL` kind are materialized using
 | Redshift   | DROP TABLE, CREATE TABLE, INSERT |
 | Postgres   | DROP TABLE, CREATE TABLE, INSERT |
 | DuckDB     | CREATE OR REPLACE TABLE          |
+| Doris      | CREATE TABLE ... AS SELECT (new OLAP), TRUNCATE / DELETE + INSERT (existing OLAP & Doris 4.1+ Iceberg v2 — non-atomic: concurrent readers may observe a gap and failed inserts leave the table empty or partially populated; see [Doris catalog matrix & recovery](../../integrations/engines/doris.md#supported-model-kinds-catalog-matrix)), INSERT OVERWRITE (unpartitioned Hive on Doris 2.1.6+) |
 
 ## VIEW
 The model kinds described so far cause the output of a model query to be materialized and stored in a physical table.
@@ -837,6 +840,7 @@ MODEL (
 
 * BigQuery
 * Databricks
+* Doris
 * Snowflake
 
 During the evaluation of a model of this kind, the view will be replaced or recreated only if the model's query rendered during evaluation does not match the query used during the previous view creation for this model, or if the target view does not exist. Thus, views are recreated only when necessary in order to realize all the benefits provided by materialized views.
@@ -1629,6 +1633,7 @@ Depending on the target engine, models of the `INCREMENTAL_BY_PARTITION` kind ar
 | Redshift   | DELETE by partitioning key, then INSERT |
 | Postgres   | DELETE by partitioning key, then INSERT |
 | DuckDB     | DELETE by partitioning key, then INSERT |
+| Doris      | DELETE by partitioning key, then INSERT (internal OLAP & Doris 4.1+ Iceberg v2; not supported on Hive — non-atomic: concurrent readers may observe a gap and failed inserts leave deleted partitions missing; see [Doris catalog matrix & recovery](../../integrations/engines/doris.md#supported-model-kinds-catalog-matrix)) |
 
 ## INCREMENTAL_UNMANAGED
 

docs/concepts/models/overview.md

@@ -181,6 +181,7 @@ This table lists each engine's support for `TABLE` and `VIEW` object comments:
 | BigQuery      | Y                | Y               |
 | ClickHouse    | Y                | Y               |
 | Databricks    | Y                | Y               |
+| Doris         | Y                | Y               |
 | DuckDB <=0.9  | N                | N               |
 | DuckDB >=0.10 | Y                | Y               |
 | MySQL         | Y                | Y               |

docs/guides/configuration.md

@@ -933,6 +933,7 @@ These pages describe the connection configuration options for each execution eng
 * [BigQuery](../integrations/engines/bigquery.md)
 * [ClickHouse](../integrations/engines/clickhouse.md)
 * [Databricks](../integrations/engines/databricks.md)
+* [Doris](../integrations/engines/doris.md)
 * [DuckDB](../integrations/engines/duckdb.md)
 * [Fabric](../integrations/engines/fabric.md)
 * [MotherDuck](../integrations/engines/motherduck.md)
@@ -975,6 +976,7 @@ Other state engines with fast and reliable database transactions (less tested th
 Unsupported state engines, even for development:
 
 * [ClickHouse](../integrations/engines/clickhouse.md)
+* [Doris](../integrations/engines/doris.md)
 * [Spark](../integrations/engines/spark.md)
 * [StarRocks](../integrations/engines/starrocks.md)
 * [Trino](../integrations/engines/trino.md)

docs/guides/connections.md

@@ -33,7 +33,7 @@ gateways:
       database: state.db
 ```
 
-NOTE: Spark and Trino engines may not be used for the state connection.
+NOTE: Spark, Trino, and Doris engines may not be used for the state connection; configure a separate `state_connection` (for example, DuckDB for local development or Postgres/MySQL in production) when using these engines.
 
 ## Default connection
 
@@ -85,6 +85,7 @@ default_gateway: local_db
 * [ClickHouse](../integrations/engines/clickhouse.md)
 * [Databricks](../integrations/engines/databricks.md)
 * [Db2](../integrations/engines/db2.md)
+* [Doris](../integrations/engines/doris.md)
 * [DuckDB](../integrations/engines/duckdb.md)
 * [Fabric](../integrations/engines/fabric.md)
 * [MotherDuck](../integrations/engines/motherduck.md)

docs/installation.md

@@ -54,6 +54,7 @@ Other extras are required to use specific SQL engines, like Bigquery or Postgres
     | Bigquery      | `pip install "sqlmesh[bigquery]"`    |
     | ClickHouse    | `pip install "sqlmesh[clickhouse]"`  |
     | Databricks    | `pip install "sqlmesh[databricks]"`  |
+    | Doris         | `pip install "sqlmesh[doris]"`       |
     | Fabric        | `pip install "sqlmesh[fabric]"`      |
     | GCP Postgres  | `pip install "sqlmesh[gcppostgres]"` |
     | MS SQL Server | `pip install "sqlmesh[mssql]"`       |