Skip to content

[CIEXE-2681] Use published Docker CI images with internal Buildx #882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ericdog wants to merge 7 commits into
base: main
Choose a base branch
from
+47 −11
Open

[CIEXE-2681] Use published Docker CI images with internal Buildx #882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0970e69
[CIEXE-2681] Use published internal-Buildx Docker CI images
ericdog Nov 3, 2026
628602f
fix(ci): skip redundant pip install in layer publishing
ericdog Nov 5, 2026
97c3648
Merge branch 'main' into ericlim/internal-buildx-modern-docker-20261103
ericdog Nov 12, 2026
8d227c2
Pin CI images with internal Buildx v0.38.2-dd.3
ericdog Nov 12, 2026
7337b98
Fix Python CI tools and isolate image validation from Lambda deployment
ericdog Nov 12, 2026
ef58082
ci: pin Docker 27.3.1 to the image published from master
ericdog Nov 13, 2026
61558e2
ci: follow maintained shared image tags
ericdog Nov 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 46 additions & 9 deletions ci/input_files/build.yaml.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@ default:
# Retry when the runner fails to start
- runner_system_failure

# The Noble Python image uses managed system Python and does not include AWS CLI.
.install-python-tools: &install-python-tools
- apt-get update
- apt-get install -y --no-install-recommends python3-venv python-is-python3
- python3 -m venv /opt/ci-python-tools
- /opt/ci-python-tools/bin/pip install awscli
- export PATH="/opt/ci-python-tools/bin:$PATH"

# This is for serverless framework
.install-node: &install-node
- apt-get update
Expand All @@ -26,6 +34,18 @@ default:
- apt-get install nodejs -y
- npm install --global yarn

validate-python-ci-tools:
stage: test
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
before_script:
- *install-python-tools
script:
- aws --version
- python --version
- python -m pip --version
- docker buildx version

{{ range $runtime := (ds "runtimes").runtimes }}

# TODO(astuyve) - figure out python build cache
Expand All @@ -38,7 +58,7 @@ default:
build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: build
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1
artifacts:
expire_in: 1 day # Unsigned zips expire in 1 day
paths:
Expand All @@ -53,7 +73,7 @@ build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: test
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1
needs:
- build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
dependencies:
Expand Down Expand Up @@ -99,8 +119,11 @@ unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: test
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
rules:
# Image-maintenance validation has no authorization to deploy Lambda services.
- if: '$CI_COMMIT_BRANCH == "ericlim/internal-buildx-modern-docker-20261103" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "ericlim/internal-buildx-modern-docker-20261103"'
when: never
- if: '$SKIP_E2E_TESTS == "true"'
when: never
- when: on_success
Expand All @@ -112,6 +135,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
variables:
CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
before_script:
- *install-python-tools
- *install-node
- EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
- yarn global add serverless@^3.38.0 --prefix /usr/local
Expand All @@ -124,7 +148,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: sign
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
rules:
- if: '$CI_COMMIT_TAG =~ /^v.*/'
when: manual
Expand All @@ -141,6 +165,7 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
paths:
- .layers/datadog_lambda_py-{{ $runtime.arch }}-{{ $runtime.python_version }}.zip
before_script:
- *install-python-tools
- apt-get update
- apt-get install -y uuid-runtime
{{ with $environment := (ds "environments").environments.prod }}
Expand All @@ -155,8 +180,11 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: publish
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
rules:
# Image-maintenance validation has no authorization to deploy Lambda services.
- if: '$CI_COMMIT_BRANCH == "ericlim/internal-buildx-modern-docker-20261103" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "ericlim/internal-buildx-modern-docker-20261103"'
when: never
- if: '$SKIP_E2E_TESTS == "true"'
when: never
- if: '"{{ $environment_name }}" == "sandbox" && $REGION == "{{ $e2e_region }}" && "{{ $runtime.arch }}" == "amd64"'
Expand Down Expand Up @@ -190,6 +218,8 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
- {{ .code }}
{{- end}}
before_script:
- *install-python-tools
Comment thread
ericdog marked this conversation as resolved.
Resolved
- export SKIP_PIP_INSTALL=true
- EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
script:
- STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} DOTENV={{ $dotenv }} ./ci/publish_layers.sh
Expand All @@ -201,8 +231,9 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
publish-pypi-package:
stage: publish
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
before_script:
- *install-python-tools
- ./scripts/setup_python_env.sh
cache: []
rules:
Expand All @@ -217,7 +248,7 @@ publish-pypi-package:
layer bundle:
stage: build
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1
needs:
{{ range (ds "runtimes").runtimes }}
- build-layer ({{ .name }}-{{ .arch }})
Expand All @@ -238,7 +269,7 @@ layer bundle:

signed layer bundle:
stage: sign
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
tags: ["arch:amd64"]
rules:
- if: '$CI_COMMIT_TAG =~ /^v.*/'
Expand Down Expand Up @@ -266,6 +297,9 @@ e2e-test:
project: DataDog/serverless-e2e-tests
strategy: depend
rules:
# Image-maintenance validation has no authorization to deploy Lambda services.
- if: '$CI_COMMIT_BRANCH == "ericlim/internal-buildx-modern-docker-20261103" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "ericlim/internal-buildx-modern-docker-20261103"'
when: never

@purple4reina purple4reina Nov 12, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want to commit these?

- if: '$SKIP_E2E_TESTS == "true"'
when: never
- when: on_success
Expand All @@ -286,10 +320,13 @@ e2e-test:

e2e-test-status:
stage: e2e
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
tags: ["arch:amd64"]
timeout: 3h
rules:
# Image-maintenance validation has no authorization to deploy Lambda services.
- if: '$CI_COMMIT_BRANCH == "ericlim/internal-buildx-modern-docker-20261103" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "ericlim/internal-buildx-modern-docker-20261103"'
when: never
- if: '$SKIP_E2E_TESTS == "true"'
when: never
- when: on_success
Expand Down
3 changes: 1 addition & 2 deletions scripts/setup_python_env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ if [ -z "${IS_LINT:-}" ] || [ "${IS_LINT}" = "0" ] || [ "${IS_LINT}" = "false" ]
spec_ddtrace_dep
fi

pip install virtualenv
virtualenv venv
python3 -m venv venv
source venv/bin/activate
pip install .[dev]
pip install poetry
Expand Down
Loading