diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl index cf89c5ea..3f35b13f 100644 --- a/ci/input_files/build.yaml.tpl +++ b/ci/input_files/build.yaml.tpl @@ -14,6 +14,14 @@ default: # Retry when the runner fails to start - runner_system_failure +# The Noble Python image uses managed system Python and does not include AWS CLI. +.install-python-tools: &install-python-tools + - apt-get update + - apt-get install -y --no-install-recommends python3-venv python-is-python3 + - python3 -m venv /opt/ci-python-tools + - /opt/ci-python-tools/bin/pip install awscli + - export PATH="/opt/ci-python-tools/bin:$PATH" + # This is for serverless framework .install-node: &install-node - apt-get update @@ -26,6 +34,18 @@ default: - apt-get install nodejs -y - npm install --global yarn +validate-python-ci-tools: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:29.4.0-noble-py3 + before_script: + - *install-python-tools + script: + - aws --version + - python --version + - python -m pip --version + - docker buildx version + {{ range $runtime := (ds "runtimes").runtimes }} # TODO(astuyve) - figure out python build cache @@ -38,7 +58,7 @@ default: build-layer ({{ $runtime.name }}-{{ $runtime.arch }}): stage: build tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 + image: registry.ddbuild.io/images/docker:27.3.1 artifacts: expire_in: 1 day # Unsigned zips expire in 1 day paths: @@ -53,7 +73,7 @@ build-layer ({{ $runtime.name }}-{{ $runtime.arch }}): check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}): stage: test tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 + image: registry.ddbuild.io/images/docker:27.3.1 needs: - build-layer ({{ $runtime.name }}-{{ $runtime.arch }}) dependencies: @@ -99,8 +119,11 @@ unit-test ({{ $runtime.name }}-{{ $runtime.arch }}): integration-test ({{ $runtime.name }}-{{ $runtime.arch }}): stage: test tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 + image: registry.ddbuild.io/images/docker:29.4.0-noble-py3 rules: + # Image-maintenance validation has no authorization to deploy Lambda services. + - if: '$CI_COMMIT_BRANCH == "ericlim/internal-buildx-modern-docker-20261103" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "ericlim/internal-buildx-modern-docker-20261103"' + when: never - if: '$SKIP_E2E_TESTS == "true"' when: never - when: on_success @@ -112,6 +135,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}): variables: CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" before_script: + - *install-python-tools - *install-node - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - yarn global add serverless@^3.38.0 --prefix /usr/local @@ -124,7 +148,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}): sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}): stage: sign tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 + image: registry.ddbuild.io/images/docker:29.4.0-noble-py3 rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' when: manual @@ -141,6 +165,7 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}): paths: - .layers/datadog_lambda_py-{{ $runtime.arch }}-{{ $runtime.python_version }}.zip before_script: + - *install-python-tools - apt-get update - apt-get install -y uuid-runtime {{ with $environment := (ds "environments").environments.prod }} @@ -155,8 +180,11 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}): publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}): stage: publish tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 + image: registry.ddbuild.io/images/docker:29.4.0-noble-py3 rules: + # Image-maintenance validation has no authorization to deploy Lambda services. + - if: '$CI_COMMIT_BRANCH == "ericlim/internal-buildx-modern-docker-20261103" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "ericlim/internal-buildx-modern-docker-20261103"' + when: never - if: '$SKIP_E2E_TESTS == "true"' when: never - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "{{ $e2e_region }}" && "{{ $runtime.arch }}" == "amd64"' @@ -190,6 +218,8 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}): - {{ .code }} {{- end}} before_script: + - *install-python-tools + - export SKIP_PIP_INSTALL=true - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh script: - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} DOTENV={{ $dotenv }} ./ci/publish_layers.sh @@ -201,8 +231,9 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}): publish-pypi-package: stage: publish tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 + image: registry.ddbuild.io/images/docker:29.4.0-noble-py3 before_script: + - *install-python-tools - ./scripts/setup_python_env.sh cache: [] rules: @@ -217,7 +248,7 @@ publish-pypi-package: layer bundle: stage: build tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 + image: registry.ddbuild.io/images/docker:27.3.1 needs: {{ range (ds "runtimes").runtimes }} - build-layer ({{ .name }}-{{ .arch }}) @@ -238,7 +269,7 @@ layer bundle: signed layer bundle: stage: sign - image: registry.ddbuild.io/images/docker:20.10-py3 + image: registry.ddbuild.io/images/docker:29.4.0-noble-py3 tags: ["arch:amd64"] rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' @@ -266,6 +297,9 @@ e2e-test: project: DataDog/serverless-e2e-tests strategy: depend rules: + # Image-maintenance validation has no authorization to deploy Lambda services. + - if: '$CI_COMMIT_BRANCH == "ericlim/internal-buildx-modern-docker-20261103" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "ericlim/internal-buildx-modern-docker-20261103"' + when: never - if: '$SKIP_E2E_TESTS == "true"' when: never - when: on_success @@ -286,10 +320,13 @@ e2e-test: e2e-test-status: stage: e2e - image: registry.ddbuild.io/images/docker:20.10-py3 + image: registry.ddbuild.io/images/docker:29.4.0-noble-py3 tags: ["arch:amd64"] timeout: 3h rules: + # Image-maintenance validation has no authorization to deploy Lambda services. + - if: '$CI_COMMIT_BRANCH == "ericlim/internal-buildx-modern-docker-20261103" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "ericlim/internal-buildx-modern-docker-20261103"' + when: never - if: '$SKIP_E2E_TESTS == "true"' when: never - when: on_success diff --git a/scripts/setup_python_env.sh b/scripts/setup_python_env.sh index 42dfa710..fdf871cc 100755 --- a/scripts/setup_python_env.sh +++ b/scripts/setup_python_env.sh @@ -65,8 +65,7 @@ if [ -z "${IS_LINT:-}" ] || [ "${IS_LINT}" = "0" ] || [ "${IS_LINT}" = "false" ] spec_ddtrace_dep fi -pip install virtualenv -virtualenv venv +python3 -m venv venv source venv/bin/activate pip install .[dev] pip install poetry