From fc8053af152488cd61d8f0069287dce66390506c Mon Sep 17 00:00:00 2001 From: David Wolfe Date: Fri, 21 Aug 2026 09:46:31 -0700 Subject: [PATCH] DUPLO-42774 Enable the Amazon Linux 2023 Packer builders There was no Duplo Docker Native image for Amazon Linux 2023. The Packer sources for it were added commented out in 2024-02 and had never produced an image. - Enable the `amazonlinux-2023` and `amazonlinux-2023-arm64` sources, and add the matching build, OS-update, install, and cleanup provisioners. - Add `AgentAmazonLinux2023/Setup.sh`, which installs Docker and the agent from `dnf` into a Python 3 venv. It enables `NetworkAgent.service` without starting it, since starting the agent during the bake disrupts iptables and severs the session. - Install the agent dependencies in one `pip` call. Sequential installs let `botocore` downgrade `urllib3` to 1.26.x on the `boto3` call alone. - Pin the AL2023 source AMI to `al2023-ami-2023.*`. The wider `al2023-ami-*` also matches the ECS-optimized and minimal variants, and `most_recent` could select one of those. - Merge `BuiltInNativeImages.json` by `Name` rather than replacing every `Docker-Duplo*` row. A build scoped with `only_builders` was wiping the Amazon Linux 2, Ubuntu, and GovCloud rows it had not rebuilt. - Fail `gen-native-images.sh` on a builder name with no SSH username mapping. It previously reused the previous row's username. - Run the GCP credential step only when `only_builders` names a `googlecompute` builder, and treat an empty value the same as `all`. `all` passes `-except=googlecompute.*`, and an empty value reached Packer unfiltered and built every source. - Add `amazon-ebs.amazonlinux-2-arm64` to the Amazon Linux OS-update step, which had listed only the x86_64 builder. - Pin every third-party action to a commit SHA, bump `actions/checkout` to v7, `actions/upload-artifact` to v7, and `actions/download-artifact` to v8, and move the commercial AWS role step to OIDC. - Add `AgentAmazonLinux2023/README.md` and a CHANGELOG entry, and correct the README claim that `all` builds every image. --- .github/workflows/build-image.yaml | 46 +++--- .github/workflows/lint.yaml | 5 +- AgentAmazonLinux2023/README.md | 46 ++++++ AgentAmazonLinux2023/Setup.sh | 174 ++++++++++++++++++++++ CHANGELOG.md | 15 ++ README.md | 6 +- packer/aws.pkr.hcl | 232 +++++++++++++++-------------- packer/gen-native-images.sh | 41 +++-- packer/main.pkr.hcl | 27 +++- 9 files changed, 426 insertions(+), 166 deletions(-) create mode 100644 AgentAmazonLinux2023/README.md create mode 100755 AgentAmazonLinux2023/Setup.sh diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 389f36e..6bdeefd 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -31,22 +31,20 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - # GCP credentials + # GCP credentials - only when a GCP builder is in scope - name: Packer GCP Service Account - uses: google-github-actions/auth@v2 + if: contains(github.event.inputs.only_builders, 'googlecompute') + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 with: workload_identity_provider: 'projects/17033121890/locations/global/workloadIdentityPools/duplo-githubactions/providers/duplo-githubactions' service_account: 'packer@msp-duplocloud-01.iam.gserviceaccount.com' # AWS credentials - name: Packer AWS Role - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 with: - aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} - aws-session-token: ${{ env.AWS_SESSION_TOKEN }} aws-region: us-west-2 role-to-assume: arn:aws:iam::227120241369:role/packer-builder role-session-name: github-duplocloud-linuxagent @@ -71,10 +69,10 @@ jobs: # Validate the template. packer validate -syntax-only ./packer - # Parse build options. - if [ "$ONLY_BUILDERS" = "all" ]; then + # Parse build options. An empty input would otherwise reach packer unfiltered and build the GCP sources too. + if [ -z "$ONLY_BUILDERS" ] || [ "$ONLY_BUILDERS" = "all" ]; then ONLY_BUILDERS="-except=googlecompute.ubuntu-20,googlecompute.ubuntu-22" - elif [ -n "$ONLY_BUILDERS" ]; then + else ONLY_BUILDERS="-only=$ONLY_BUILDERS" fi @@ -105,7 +103,7 @@ jobs: # Upload the image manifest - name: Attach Manifest - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: packer-manifest.json path: packer-manifest.json @@ -117,17 +115,17 @@ jobs: duplo_token: "${{ secrets.GOVCLOUD_DUPLO_TOKEN }}" steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 # AWS credentials - name: Tenant AWS JIT - uses: duplocloud/ghactions-aws-jit@master + uses: duplocloud/ghactions-aws-jit@8add5087285847c4f96923be523cee313aa4009d # master, newer than the v0.3.3 tag with: tenant: github # AWS credentials - name: Packer AWS Role - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 with: aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} @@ -157,10 +155,10 @@ jobs: # Validate the template. packer validate -syntax-only ./packer - # Parse build options. - if [ "$ONLY_BUILDERS" = "all" ]; then + # Parse build options. An empty input would otherwise reach packer unfiltered and build the GCP sources too. + if [ -z "$ONLY_BUILDERS" ] || [ "$ONLY_BUILDERS" = "all" ]; then ONLY_BUILDERS="-except=googlecompute.ubuntu-20,googlecompute.ubuntu-22" - elif [ -n "$ONLY_BUILDERS" ]; then + else ONLY_BUILDERS="-only=$ONLY_BUILDERS" fi @@ -179,7 +177,7 @@ jobs: done # Build the images. - rm -f *packer-manifest.json # always be clean + rm -f ./*packer-manifest.json # always be clean packer build $ONLY_BUILDERS \ -color=false -on-error=cleanup -parallel-builds=10 -timestamp-ui \ -var-file=packer/duplo-gov.json \ @@ -193,7 +191,7 @@ jobs: # Upload the image manifest - name: Attach Manifest - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: govcloud-packer-manifest.json path: govcloud-packer-manifest.json @@ -208,9 +206,9 @@ jobs: steps: # Get the code for the image JSON generation, and the code for Duplo master. - name: Checkout duplo-infra - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Checkout duplo (backend) - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: repository: duplocloud-internal/duplo ref: master # always start from master @@ -219,12 +217,12 @@ jobs: # Download the image manifest - name: Download Manifest (Commercial) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: packer-manifest.json path: packer - name: Download Manifest (Govcloud) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: govcloud-packer-manifest.json path: packer @@ -238,7 +236,7 @@ jobs: # Create a PR - name: Create Pull Request - uses: peter-evans/create-pull-request@v4 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: title: '[duplo-bot] Update Duplo Docker AMI(s)' branch: auto-update/duplo-docker-amis diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 922f9fd..9a99130 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -6,11 +6,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 # Validate templates - name: Validate Template - uses: hashicorp/packer-github-actions@master + # Last commit of the command-based action. v2+ is a different, setup-only action with no command/target inputs. + uses: hashicorp/packer-github-actions@8c999b2ff7c48cce2adc5fe43ad7fd8a80e6a8b5 # master with: command: validate arguments: -syntax-only diff --git a/AgentAmazonLinux2023/README.md b/AgentAmazonLinux2023/README.md new file mode 100644 index 0000000..12ff68e --- /dev/null +++ b/AgentAmazonLinux2023/README.md @@ -0,0 +1,46 @@ +### Amazon Linux 2023 base image + +Use the plain AL2023 base image, published by `amazon` and named `al2023-ami-2023.*-kernel-6.1-x86_64` (or `-arm64`). The ECS-optimized and minimal variants also match the looser `al2023-ami-*` pattern and are not supported here. + +The agent script and the systemd unit are downloaded from `AgentAmazonLinux2/`. Only the install steps differ on AL2023, so there is no copy of `NetworkAgentV2.py` or `NetworkAgent.service` in this directory. + +### Amazon Linux 2023 user data + +``` +#!/bin/bash + +curl -H "Accept: application/vnd.github.v3.raw" -O -L https://api.github.com/repos/duplocloud/linuxagent/contents/AgentAmazonLinux2023/Setup.sh +chmod +x ./Setup.sh +sudo bash ./Setup.sh +# + +``` + +### Amazon Linux 2023 user data, base64 + +``` +IyEvYmluL2Jhc2gKCmN1cmwgLUggIkFjY2VwdDogYXBwbGljYXRpb24vdm5kLmdpdGh1Yi52My5yYXciIC1PIC1MIGh0dHBzOi8vYXBpLmdpdGh1Yi5jb20vcmVwb3MvZHVwbG9jbG91ZC9saW51eGFnZW50L2NvbnRlbnRzL0FnZW50QW1hem9uTGludXgyMDIzL1NldHVwLnNoCmNobW9kICt4IC4vU2V0dXAuc2gKc3VkbyBiYXNoIC4vU2V0dXAuc2gKIwo= +``` + +### Amazon Linux 2023 manual + +``` +curl -H "Accept: application/vnd.github.v3.raw" -O -L https://api.github.com/repos/duplocloud/linuxagent/contents/AgentAmazonLinux2023/Setup.sh +chmod +x ./Setup.sh +sudo bash ./Setup.sh + +# now, reboot the system +sudo reboot + +# after reboot +sudo systemctl daemon-reload +sudo systemctl status NetworkAgent +sudo systemctl stop NetworkAgent +sudo systemctl start NetworkAgent +sudo systemctl status NetworkAgent + +tail -f /var/log/NetworkAgent.log + +#vi /lib/systemd/system/NetworkAgent.service +#vi /usr/local/src/AgentV2/NetworkAgentV2.py +``` diff --git a/AgentAmazonLinux2023/Setup.sh b/AgentAmazonLinux2023/Setup.sh new file mode 100755 index 0000000..b644fd1 --- /dev/null +++ b/AgentAmazonLinux2023/Setup.sh @@ -0,0 +1,174 @@ +#!/bin/bash +set -eu + +AGENT='NetworkAgentV2' +DAEMON_DEFAULT_FILE="/etc/default/$AGENT" +DAEMON_DIR='/usr/local/src/AgentV2' +PYTHON_PATH="$DAEMON_DIR/flask/bin" +DAEMON="$DAEMON_DIR/NetworkAgentV2.py" +# Agent script and systemd unit are shared with Amazon Linux 2. Only the install steps differ on AL2023. +DOWNLOAD_URL="https://api.github.com/repos/duplocloud/linuxagent/contents/AgentAmazonLinux2" + +DOCKER_OVERRIDE_DIR="/etc/systemd/system/docker.service.d" +DOCKER_OVERRIDE_FILE="$DOCKER_OVERRIDE_DIR/api.conf" + +# GitHub redirects raw content to a signed CDN URL, so -L is required. These pin both hops to https. +CURL_HTTPS_OPTS=(--proto '=https' --proto-redir '=https' -fsSL) + +if [[ -z "${DOWNLOAD_REF:-}" ]]; then + DOWNLOAD_REF='' +else + DOWNLOAD_REF="?ref=${DOWNLOAD_REF}" +fi + +# AL2023 has no amazon-linux-extras. Docker comes from the default repo. +install_dependencies () { + echo "AL2023: installing Docker and base packages" + sudo dnf update -q -y + sudo dnf install -q -y \ + docker \ + amazon-ecr-credential-helper \ + git wget net-tools vim \ + gcc \ + python3 python3-pip python3-devel \ + iptables-devel kernel-headers + + sudo usermod -a -G docker ec2-user + mkdir -p ~/.docker + echo '{ "credsStore": "ecr-login" }' > ~/.docker/config.json + + sudo mkdir -p "$DOCKER_OVERRIDE_DIR" + sudo tee "$DOCKER_OVERRIDE_FILE" > /dev/null </dev/null || true + sudo systemctl mask ecs 2>/dev/null || true + sudo dnf -q -y remove ecs-init || true + sudo rm -rf /var/lib/ecs /etc/ecs /var/log/ecs + for image in amazon/amazon-ecs-agent amazon/amazon-ecs-pause; do + sudo docker image rm -f "$image:latest" 2>/dev/null || true + sudo docker image rm -f "$image:0.1.0" 2>/dev/null || true + done + sudo docker rm -f ecs-agent 2>/dev/null || true +} + +agent_install () { + echo "AL2023: installing NetworkAgent in Python 3 venv" + + if [[ -f "$DAEMON_DEFAULT_FILE" ]]; then + sudo rm $DAEMON_DEFAULT_FILE + sudo touch $DAEMON_DEFAULT_FILE + fi + + echo "DAEMON=$DAEMON" | sudo tee --append $DAEMON_DEFAULT_FILE > /dev/null + echo "DAEMON_DIR=$DAEMON_DIR" | sudo tee --append $DAEMON_DEFAULT_FILE > /dev/null + echo "PYTHON_PATH=$PYTHON_PATH" | sudo tee --append $DAEMON_DEFAULT_FILE > /dev/null + cat $DAEMON_DEFAULT_FILE + + cd "$DAEMON_DIR" + python3 -m venv flask + flask/bin/pip install --upgrade pip + # Install all agent dependencies in a single pip invocation so the + # resolver picks a globally-consistent set. Sequential pip installs + # silently downgrade urllib3 from 2.x to 1.26.x because botocore's + # transitive constraint is only seen on the boto3 install call. + flask/bin/pip --trusted-host pypi.python.org install \ + flask \ + requests \ + python-pytun \ + python-iptables \ + docker \ + boto3 + + cd "$DAEMON_DIR" + curl "${CURL_HTTPS_OPTS[@]}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o NetworkAgentV2.py "$DOWNLOAD_URL/NetworkAgentV2.py$DOWNLOAD_REF" + chmod a+x NetworkAgentV2.py + ls -alt "$DAEMON_DIR" + + cd /lib/systemd/system + sudo curl "${CURL_HTTPS_OPTS[@]}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o NetworkAgent.service "$DOWNLOAD_URL/NetworkAgent.service$DOWNLOAD_REF" + ls -alt NetworkAgent.service + + sudo systemctl daemon-reload + sudo systemctl enable NetworkAgent.service + # Do not start NetworkAgent.service here. Starting it during the + # Packer bake disrupts iptables/networking and severs the SSM/SSH + # session, which surfaces as `Bad exit status: -1`. The unit is + # enabled and will start automatically on first boot. +} + +get_os_type () { + if [[ -f /etc/os-release ]]; then + . /etc/os-release + OS=$ID + VER=$VERSION_ID + else + OS=$(uname -s) + VER=$(uname -r) + fi +} + +get_os_type +echo "Detected OS=$OS VER=$VER" +if [[ "$OS" != "amzn" ]] || [[ "$VER" != "2023" ]]; then + echo "WARNING: this script targets Amazon Linux 2023; detected $OS $VER" +fi + +echo "--------------------------OS=$OS VER=$VER--------------------------" +echo "Step 1: install Docker and base packages" +echo "--------------------------OS=$OS VER=$VER--------------------------" +install_dependencies + +echo "===========================" +echo " Docker installed " +echo "===========================" + +echo "--------------------------OS=$OS VER=$VER--------------------------" +echo "Step 1b: remove Amazon ECS agent if present" +echo "--------------------------OS=$OS VER=$VER--------------------------" +remove_ecs_agent + +echo "--------------------------OS=$OS VER=$VER--------------------------" +echo "Step 2: ensure agent directory exists" +echo "--------------------------OS=$OS VER=$VER--------------------------" +sudo mkdir -p "$DAEMON_DIR" +sudo chown -R "${USER:-ec2-user}" "$DAEMON_DIR" +ls -alt "$DAEMON_DIR" + +echo "--------------------------OS=$OS VER=$VER--------------------------" +echo "Step 3: stop any running $AGENT" +echo "--------------------------OS=$OS VER=$VER--------------------------" +if pgrep -f "$DAEMON" > /dev/null; then + echo "$AGENT is running, killing" + sudo pkill -9 -f "$DAEMON" || true +else + echo "$AGENT is not running" +fi + +echo "--------------------------OS=$OS VER=$VER--------------------------" +echo "Step 4: install NetworkAgent" +echo "--------------------------OS=$OS VER=$VER--------------------------" +cd "$DAEMON_DIR" +sudo rm -rf NetworkAgentV2.py flask +agent_install + +echo "AL2023 setup complete" +exit 0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 17e7fa8..866e763 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +## 2026-08-19 + +### Added +- Enabled the Amazon Linux 2023 x86_64 and arm64 Packer builders, along with `AgentAmazonLinux2023/Setup.sh` and a README for it. These sources were added commented-out in 2024-02, so the Amazon Linux 2023 builders claimed by the 2024-02-07 entry below have never run until now. +- Pinned the Amazon Linux 2023 `source_ami_filter` to `al2023-ami-2023.*`, which excludes the ECS-optimized and minimal AMI variants that the looser `al2023-ami-*` pattern also matches. +- Added a per-builder SSH username check to `gen-native-images.sh`. An unmapped builder name previously reused the previous row's username instead of failing. + +### Changed +- Merged the generated native-image rows by `Name` instead of replacing every `Docker-Duplo*` row. A build scoped with `only_builders` now leaves the Amazon Linux 2, Ubuntu, and GovCloud rows intact. +- Pinned all third-party GitHub Actions to commit SHAs, and bumped `actions/checkout` to v7, `actions/upload-artifact` to v7, and `actions/download-artifact` to v8. +- Switched the commercial `Packer AWS Role` step to OIDC alone, dropping the static credential inputs that were resolving to empty strings. +- Skipped the GCP credential step unless `only_builders` names a `googlecompute` builder. It had been authenticating on the `all` path, which excludes those builders. +- Treated an empty `only_builders` the same as `all`. A cleared input previously reached packer with no filter, building every source including the GCP ones. +- Added `amazon-ebs.amazonlinux-2-arm64` to the Amazon Linux OS-update step, which had listed only the x86_64 builder. + ## 2024-08-13 ### Changed diff --git a/README.md b/README.md index 76cd3bd..ce44b3a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ **Full instructions TBD** -Run the `Image: Agent - duplocloud-docker` action to build both AWS and GCP images. +Run the `Image: Agent - duplocloud-docker` action to build the AWS images. - `image_version`: set this to `release-MONTH-YEAR` to build a release (no underscores allowed) - `only_builders`: set this to `all` for release - you can also change this build only certain images (packer `-only` syntax) @@ -21,7 +21,7 @@ Run the `Image: Agent - duplocloud-docker` action to build both AWS and GCP imag #### Parameter: only_builders - If you change the `only_builders` to a comma-delimited list (packer `-only` syntax) - you will only build those images -- If you change the `only_builders` to `all` - you will build all images +- If you change the `only_builders` to `all` - you will build every AWS image. `all` excludes the GCP builders, which build only when named explicitly. # Manual Installation Steps @@ -31,4 +31,6 @@ Run the `Image: Agent - duplocloud-docker` action to build both AWS and GCP imag [Ubuntu 20.04 and Ubuntu 22.04](docs/README_UBUNTU_20_04_AND_22_04.md) ### OS : Amazon linux 2 [Amazon Linux 2](docs/README_AMAZON_LINUX_2.md) +### OS : Amazon Linux 2023 +[Amazon Linux 2023](AgentAmazonLinux2023/README.md) diff --git a/packer/aws.pkr.hcl b/packer/aws.pkr.hcl index a6ee3d7..8313683 100644 --- a/packer/aws.pkr.hcl +++ b/packer/aws.pkr.hcl @@ -380,118 +380,120 @@ source "amazon-ebs" "amazonlinux-2-arm64" { } } -// source "amazon-ebs" "amazonlinux-2023" { -// ami_name = "${local.image_family}-al2023-${local.image_version}" -// ami_description = "${local.image_description} (al2023)" -// instance_type = var.aws_instance_type -// region = var.aws_region -// vpc_id = var.aws_vpc_id -// subnet_id = var.aws_subnet_id -// security_group_id = var.aws_security_group_id -// iam_instance_profile = var.aws_iam_instance_profile -// associate_public_ip_address = true - -// temporary_key_pair_type = var.temporary_key_pair_type -// ssh_username = "ec2-user" -// ssh_interface = "session_manager" - -// source_ami_filter { -// filters = { -// name = "al2023-ami-*-kernel-6.1-x86_64" -// root-device-type = "ebs" -// virtualization-type = "hvm" -// } -// most_recent = true -// owners = ["amazon"] -// } - -// # Build a public AMI -// encrypt_boot = false -// ami_groups = local.is_public ? ["all"] : [] -// ami_regions = [for region in local.ami_regions: region if region != var.aws_region] - -// # Customize the volumes -// launch_block_device_mappings { -// device_name = "/dev/xvda" -// encrypted = false -// volume_size = 35 -// volume_type = "gp3" -// delete_on_termination = true -// } - -// # Source instance tags. -// run_tags = { -// Name = "Packer Builder: ${local.image_family}-al2023-${local.image_version}" -// Creator = "Packer" -// } -// run_volume_tags = { -// Creator = "Packer" -// } - -// # Target AMI tags. -// tags = { -// Name = "${local.image_family}-al2023-${local.image_version}" -// Creator = "Packer" -// } -// snapshot_tags = { -// Creator = "Packer" -// } -// } - -// source "amazon-ebs" "amazonlinux-2023-arm64" { -// ami_name = "${local.image_family}-al2023-arm64-${local.image_version}" -// ami_description = "${local.image_description} arm64 (al2023)" -// instance_type = var.aws_instance_type_arm64 -// region = var.aws_region -// vpc_id = var.aws_vpc_id -// subnet_id = var.aws_subnet_id -// security_group_id = var.aws_security_group_id -// iam_instance_profile = var.aws_iam_instance_profile -// associate_public_ip_address = true - -// temporary_key_pair_type = var.temporary_key_pair_type -// ssh_username = "ec2-user" -// ssh_interface = "session_manager" - -// source_ami_filter { -// filters = { -// name = "al2023-ami-*-kernel-6.1-arm64" -// root-device-type = "ebs" -// virtualization-type = "hvm" -// } -// most_recent = true -// owners = ["amazon"] -// } - -// # Build a public AMI -// encrypt_boot = false -// ami_groups = local.is_public ? ["all"] : [] -// ami_regions = [for region in local.ami_regions: region if region != var.aws_region] - -// # Customize the volumes -// launch_block_device_mappings { -// device_name = "/dev/xvda" -// encrypted = false -// volume_size = 35 -// volume_type = "gp3" -// delete_on_termination = true -// } - -// # Source instance tags. -// run_tags = { -// Name = "Packer Builder: ${local.image_family}-al2023-arm64-${local.image_version}" -// Creator = "Packer" -// } -// run_volume_tags = { -// Creator = "Packer" -// } - -// # Target AMI tags. -// tags = { -// Name = "${local.image_family}-al2023-arm64-${local.image_version}" -// Creator = "Packer" -// } -// snapshot_tags = { -// Creator = "Packer" -// } -// } +source "amazon-ebs" "amazonlinux-2023" { + ami_name = "${local.image_family}-amazonlinux2023-${local.image_version}" + ami_description = "${local.image_description} (amazonlinux2023)" + instance_type = var.aws_instance_type + region = var.aws_region + vpc_id = var.aws_vpc_id + subnet_id = var.aws_subnet_id + security_group_id = var.aws_security_group_id + iam_instance_profile = var.aws_iam_instance_profile + associate_public_ip_address = true + + temporary_key_pair_type = var.temporary_key_pair_type + ssh_username = "ec2-user" + ssh_interface = "session_manager" + + # Pin to the plain AL2023 base. "al2023-ami-*" also matches the ecs and minimal variants. + source_ami_filter { + filters = { + name = "al2023-ami-2023.*-kernel-6.1-x86_64" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["amazon"] + } + + # Build a public AMI + encrypt_boot = false + ami_groups = local.is_public ? ["all"] : [] + ami_regions = [for region in local.ami_regions: region if region != var.aws_region] + + # Customize the volumes + launch_block_device_mappings { + device_name = "/dev/xvda" + encrypted = false + volume_size = 35 + volume_type = "gp3" + delete_on_termination = true + } + + # Source instance tags. + run_tags = { + Name = "Packer Builder: ${local.image_family}-amazonlinux2023-${local.image_version}" + Creator = "Packer" + } + run_volume_tags = { + Creator = "Packer" + } + + # Target AMI tags. + tags = { + Name = "${local.image_family}-amazonlinux2023-${local.image_version}" + Creator = "Packer" + } + snapshot_tags = { + Creator = "Packer" + } +} + +source "amazon-ebs" "amazonlinux-2023-arm64" { + ami_name = "${local.image_family}-amazonlinux2023-arm64-${local.image_version}" + ami_description = "${local.image_description} arm64 (amazonlinux2023)" + instance_type = var.aws_instance_type_arm64 + region = var.aws_region + vpc_id = var.aws_vpc_id + subnet_id = var.aws_subnet_id + security_group_id = var.aws_security_group_id + iam_instance_profile = var.aws_iam_instance_profile + associate_public_ip_address = true + + temporary_key_pair_type = var.temporary_key_pair_type + ssh_username = "ec2-user" + ssh_interface = "session_manager" + + # See the x86_64 source above for why this pins "al2023-ami-2023.*". + source_ami_filter { + filters = { + name = "al2023-ami-2023.*-kernel-6.1-arm64" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["amazon"] + } + + # Build a public AMI + encrypt_boot = false + ami_groups = local.is_public ? ["all"] : [] + ami_regions = [for region in local.ami_regions: region if region != var.aws_region] + + # Customize the volumes + launch_block_device_mappings { + device_name = "/dev/xvda" + encrypted = false + volume_size = 35 + volume_type = "gp3" + delete_on_termination = true + } + + # Source instance tags. + run_tags = { + Name = "Packer Builder: ${local.image_family}-amazonlinux2023-arm64-${local.image_version}" + Creator = "Packer" + } + run_volume_tags = { + Creator = "Packer" + } + + # Target AMI tags. + tags = { + Name = "${local.image_family}-amazonlinux2023-arm64-${local.image_version}" + Creator = "Packer" + } + snapshot_tags = { + Creator = "Packer" + } +} diff --git a/packer/gen-native-images.sh b/packer/gen-native-images.sh index f4d4268..5365790 100755 --- a/packer/gen-native-images.sh +++ b/packer/gen-native-images.sh @@ -4,25 +4,24 @@ # # Utility functions. -out() { echo "$0:" "$@" ; } -err() { echo "$0:" "$@" 1>&2 ; } -die() { err "$@" ; exit 1 ; } +out() { echo "$0:" "$@" ; return 0 ; } +err() { echo "$0:" "$@" 1>&2 ; return 0 ; } # Sanity checks. if file "${BASH_SOURCE[0]}" | grep -q CRLF -then die 'STOP! +then err 'STOP! Your Windows system has converted LF to CRLF. The script will not function properly. Please re-checkout the files after running: git config core.autocrlf false -' +' ; exit 1 fi if ! command -v jq >/dev/null 2>&1 -then die "JQ must be installed" +then err "JQ must be installed" ; exit 1 fi -if ! [ "$(git config core.autocrlf)" == "false" ] -then die "git config core.autocrlf must be set to 'false' for this repo" +if [[ "$(git config core.autocrlf)" != "false" ]] +then err "git config core.autocrlf must be set to 'false' for this repo" ; exit 1 fi # Step 1 - Get all image IDs from packer and use them to generate a JSON snippet @@ -39,7 +38,7 @@ do ')" arch="amd64" - [ "${name/arm64/}" != $name ] && arch="arm64" + [[ "${name/arm64/}" != "$name" ]] && arch="arm64" nicename="${nicename/-arm64/ (arm64)}" echo " @@ -81,11 +80,16 @@ $nicename images:" ubuntu*) user=ubuntu ;; + *) + # Without this the previous iteration's $user would silently carry over. + err "no SSH username mapping for builder name '$name'" + exit 1 + ;; esac echo "$niceregion" "$image" - [ -n "$json" ] && json="${json}," + [[ -n "$json" ]] && json="${json}," json="${json} { \"Name\": \"Docker-Duplo-${niceregion}-${nicename}\", @@ -110,12 +114,19 @@ out "NativeImages JSON: snippet done" snippet="$(pwd -P)/snippet-NativeImages.json" (cd "$DUPLO_SOURCE" && - # Join the default Duplo docker image ... - # ... with the remaining Duplo docker images - # ... and then all other images - jq 'input + (. | map(select(.Name | startswith("Docker-Duplo") | not)))' \ + # Merge by Name so a scoped Packer build (for example only_builders=AL2023) only replaces the rows it + # rebuilt. The previous join dropped every "Docker-Duplo*" row, wiping AL2 / Ubuntu / GovCloud entries. + # IN($newNames[]) rather than `$newNames | index(.Name)`, since piping into index rebinds `.` and would + # read .Name off the array itself. + jq 'input as $snippet + | ($snippet | map(.Name)) as $newNames + | $snippet + + (. | map(select( + ((.Name | startswith("Docker-Duplo")) | not) + or ((.Name | IN($newNames[])) | not) + )))' \ config/V1/BuiltInNativeImages.json "$snippet" > temp.json && - + # Replace the existing JSON mv temp.json config/V1/BuiltInNativeImages.json ) diff --git a/packer/main.pkr.hcl b/packer/main.pkr.hcl index 07e0ef2..5e37056 100644 --- a/packer/main.pkr.hcl +++ b/packer/main.pkr.hcl @@ -26,16 +26,21 @@ build { "sources.amazon-ebs.ubuntu-22-arm64", "sources.amazon-ebs.amazonlinux-2", "sources.amazon-ebs.amazonlinux-2-arm64", -// "sources.amazon-ebs.amazonlinux-2023", -// "sources.amazon-ebs.amazonlinux-2023-arm64", + "sources.amazon-ebs.amazonlinux-2023", + "sources.amazon-ebs.amazonlinux-2023-arm64", "sources.googlecompute.ubuntu-20", "sources.googlecompute.ubuntu-22" ] - // OS updates - Amazon Linux + // OS updates - Amazon Linux (yum on AL2023 is a wrapper for dnf) provisioner "shell" { inline = [ "sleep 10", "sudo yum update -y" ] - only = [ "amazon-ebs.amazonlinux-2" ] + only = [ + "amazon-ebs.amazonlinux-2", + "amazon-ebs.amazonlinux-2-arm64", + "amazon-ebs.amazonlinux-2023", + "amazon-ebs.amazonlinux-2023-arm64" + ] } // OS updates - Ubuntu @@ -59,10 +64,16 @@ build { environment_vars = [ "DOWNLOAD_REF=${var.agent_git_ref}" ] - only = [ - "amazon-ebs.amazonlinux-2", "amazon-ebs.amazonlinux-2-arm64", - // "amazon-ebs.amazonlinux-2023", "amazon-ebs.amazonlinux-2023-arm64" + only = [ "amazon-ebs.amazonlinux-2", "amazon-ebs.amazonlinux-2-arm64" ] + } + + // Install - Amazon Linux 2023 + provisioner "shell" { + script = "${path.root}/../AgentAmazonLinux2023/Setup.sh" + environment_vars = [ + "DOWNLOAD_REF=${var.agent_git_ref}" ] + only = [ "amazon-ebs.amazonlinux-2023", "amazon-ebs.amazonlinux-2023-arm64" ] } // Install - Ubuntu 20 @@ -92,7 +103,7 @@ build { ] only = [ "amazon-ebs.amazonlinux-2", "amazon-ebs.amazonlinux-2-arm64", - // "amazon-ebs.amazonlinux-2023", "amazon-ebs.amazonlinux-2023-arm64" + "amazon-ebs.amazonlinux-2023", "amazon-ebs.amazonlinux-2023-arm64" ] }