backport generic improvements from glamsterdam-devnets and bal-devnets

.github/actions/setup/action.yaml

@@ -3,6 +3,12 @@ description: Install deps
 runs:
   using: composite
   steps:
+    # asdf builds python from source; the runner image lacks these headers,
+    # which makes the build silently drop bz2/readline/lzma and later break pip.
+    - name: Install Python build deps
+      shell: bash
+      run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libbz2-dev libreadline-dev liblzma-dev
+
     - name: Install asdf
       uses: asdf-vm/actions/setup@b7bcd026f18772e44fe1026d729e1611cc435d47 # v4.0.1
 

ansible/ansible.cfg

@@ -12,7 +12,7 @@ host_key_checking    = False
 vars_plugins_enabled = host_group_vars,community.sops.sops
 vars_plugin_stage    = inventory
 strategy             = mitogen_free
-strategy_plugins     = vendor/mitogen-0.3.43/ansible_mitogen/plugins/strategy
+strategy_plugins     = vendor/mitogen-0.3.49/ansible_mitogen/plugins/strategy
 
 # Persist facts locally so that they can be used within multiple runs
 fact_caching            = jsonfile

ansible/install_dependencies.sh

@@ -4,7 +4,7 @@ ansible-galaxy install -r requirements.yaml --force
 
 # Install Mitogen for Ansible performance optimization
 # Following official installation instructions from https://mitogen.networkgenomics.com/ansible_detailed.html
-MITOGEN_VERSION="0.3.43"
+MITOGEN_VERSION="0.3.49"
 
 MITOGEN_DIR="vendor/mitogen-${MITOGEN_VERSION}"
 MITOGEN_URL="https://files.pythonhosted.org/packages/source/m/mitogen/mitogen-${MITOGEN_VERSION}.tar.gz"

ansible/inventories/devnet-0/group_vars/buildoor.yaml

@@ -24,22 +24,37 @@ buildoor_wallet_privkey: "{{ secret_buildoor_wallet_privkey }}"
 buildoor_builder_mnemonic: "{{ secret_buildoor_builder_mnemonic }}"
 buildoor_builder_mnemonic_index: "{{ builder_index }}"
 
+# Lets the overview label builders by the validator range they serve.
+buildoor_container_command_extra_args:
+  - --validator-ranges-url=https://config.{{ network_subdomain }}/api/v1/nodes/validator-ranges
+
 buildoor_container_env:
   VIRTUAL_HOST: "api-{{ server_fqdn }}"
   VIRTUAL_PORT: "{{ buildoor_api_port | string }}"
   LETSENCRYPT_HOST: "api-{{ server_fqdn }}"
 
-# Re-apply the prysm.yaml merge so we don't lose the bootnode args while adding --prepare-all-payloads
+# A builder has to see every attestation subnet to value a payload correctly,
+# so each CL also gets its subscribe-all-subnets flag here.
+#
+# These re-apply each client's base args because buildoor wins on group
+# precedence — assigning the flag alone would drop the client's own args
+# (bootnodes included).
 prysm_container_command_extra_args: >-
   {{ prysm_container_command_extra_simple_args
      + prysm_container_command_extra_bootnode_args
-     + ['--prepare-all-payloads'] }}
+     + ['--prepare-all-payloads', '--subscribe-all-subnets'] }}
 lighthouse_container_command_extra_args: >-
   {{ lighthouse_container_command_extra_simple_args
-     + ['--always-prepare-payload'] }}
+     + ['--always-prepare-payload', '--subscribe-all-subnets', '--import-all-attestations'] }}
 lodestar_container_command_extra_args: >-
   {{ lodestar_container_command_extra_simple_args
-     + ['--emitPayloadAttributes'] }}
+     + ['--emitPayloadAttributes', '--subscribeAllSubnets'] }}
+teku_container_command_extra_args: >-
+  {{ teku_container_command_extra_simple_args
+     + ['--Xfork-choice-updated-always-send-payload-attributes=true', '--p2p-subscribe-all-subnets-enabled'] }}
+grandine_container_command_extra_args: >-
+  {{ grandine_container_command_extra_simple_args
+     + ['--features=AlwaysPrepareExecutionPayload', '--subscribe-all-subnets'] }}
 
 ethereum_node_docker_watchtower_containers_list:
   - execution

ansible/inventories/devnet-0/group_vars/grandine.yaml

@@ -38,7 +38,11 @@ grandine_container_volumes:
   - "{{ grandine_datadir }}:/data"
   - "{{ grandine_auth_jwt_path }}:/execution-auth.jwt:ro"
   - "{{ eth_testnet_config_dir }}:/network-config:ro"
-grandine_container_command_extra_args:
+# Split so groups with higher priority (e.g. buildoor) can re-merge the base
+# args instead of replacing them wholesale.
+grandine_container_command_extra_args: >-
+  {{ grandine_container_command_extra_simple_args }}
+grandine_container_command_extra_simple_args:
   - --configuration-directory=/network-config/
   - --boot-nodes={{ ethereum_cl_bootnodes | join(',') }}
   - --graffiti={{ ansible_hostname }}

ansible/inventories/devnet-0/group_vars/prysm.yaml

@@ -49,6 +49,9 @@ prysm_container_command_extra_simple_args:
   - --min-sync-peers=1
   - --verbosity=debug
   - --subscribe-all-subnets
+  # Devnet peers are few and often share a /24, which prysm's colocation limit
+  # would otherwise treat as a sybil cluster and refuse to peer with.
+  - --p2p-colocation-whitelist=0.0.0.0/0,::/0
 prysm_container_command_extra_bootnode_args: >-
   {{ ethereum_cl_bootnodes | map('regex_replace', '^', '--bootstrap-node=') | list }}
 

ansible/inventories/devnet-0/group_vars/teku.yaml

@@ -31,7 +31,11 @@ teku_container_volumes:
   - "{{ teku_datadir }}:/data"
   - "{{ teku_auth_jwt_path }}:/execution-auth.jwt:ro"
   - "{{ eth_testnet_config_dir }}:/network-config:ro"
-teku_container_command_extra_args:
+# Split so groups with higher priority (e.g. buildoor) can re-merge the base
+# args instead of replacing them wholesale.
+teku_container_command_extra_args: >-
+  {{ teku_container_command_extra_simple_args }}
+teku_container_command_extra_simple_args:
   - --network=/network-config/config.yaml
   - --genesis-state=/network-config/genesis.ssz
   - --p2p-discovery-bootnodes={{ ethereum_cl_bootnodes | join(',') }}

ansible/playbook.yaml

@@ -102,8 +102,8 @@
       tags: [eth_inventory_web, ethereum_inventory_web]
       when: inventory_hostname == primary_bootnode
       vars:
-        eth_inventory_web_container_networks: "{{ docker_networks_shared }}" #noqa var-naming[no-role-prefix]
-        eth_inventory_web_container_env: #noqa var-naming[no-role-prefix]
+        eth_inventory_web_container_networks: "{{ docker_networks_shared }}" # noqa: var-naming[no-role-prefix]
+        eth_inventory_web_container_env: # noqa: var-naming[no-role-prefix]
           VIRTUAL_HOST: "{{ primary_bootnode }}.{{ network_server_subdomain }}"
           VIRTUAL_PORT: "80"
           VIRTUAL_PATH: "/meta/api"

scripts/run.zsh

@@ -14,6 +14,61 @@ bn_endpoint="${BEACON_ENDPOINT:-https://$sops_name:$sops_password@$beacon_prefix
 rpc_endpoint="${RPC_ENDPOINT:-https://$sops_name:$sops_password@$rpc_prefix$node.$srv.$prefix-$network.$domain}"
 bootnode_endpoint="${BOOTNODE_ENDPOINT:-https://bootnode-1.$prefix-$network.$domain}"
 
+# Verify every external tool run.zsh depends on is installed.
+# Pass 1 to print every tool; pass 0 (or empty) to print only missing tools.
+check_deps() {
+  local verbose="$1"
+  local tools=(sops yq curl jq awk bc python3 docker cast ethdo ethereal eth2-val-tools)
+  local os
+  case "$(uname -s)" in
+    Darwin) os="macos" ;;
+    Linux)  os="linux" ;;
+    *)      os="other" ;;
+  esac
+  typeset -A hints
+  if [[ "$os" == "macos" ]]; then
+    hints[sops]="brew install sops"
+    hints[yq]="brew install yq"
+    hints[curl]="preinstalled on macOS"
+    hints[jq]="brew install jq"
+    hints[awk]="preinstalled on macOS"
+    hints[bc]="preinstalled on macOS"
+    hints[python3]="preinstalled on macOS (or: brew install python)"
+    hints[docker]="https://docs.docker.com/desktop/install/mac-install/"
+  else
+    hints[sops]="apt install sops  # or download from https://github.com/getsops/sops/releases"
+    hints[yq]="apt install yq  # or download from https://github.com/mikefarah/yq/releases"
+    hints[curl]="apt install curl"
+    hints[jq]="apt install jq"
+    hints[awk]="apt install gawk"
+    hints[bc]="apt install bc"
+    hints[python3]="apt install python3"
+    hints[docker]="https://docs.docker.com/engine/install/"
+  fi
+  hints[cast]="curl -L https://foundry.paradigm.xyz | bash && foundryup"
+  hints[ethdo]="go install github.com/wealdtech/ethdo@latest"
+  hints[ethereal]="go install github.com/wealdtech/ethereal/v2@latest"
+  hints[eth2-val-tools]="go install github.com/protolambda/eth2-val-tools@latest"
+
+  local missing=0
+  [[ "$verbose" == "1" ]] && { echo "Checking dependencies for run.zsh..."; echo "" }
+  for tool in "${tools[@]}"; do
+    if command -v "$tool" >/dev/null 2>&1; then
+      [[ "$verbose" == "1" ]] && printf "  \033[32m✓\033[0m %s\n" "$tool"
+    else
+      printf "  \033[31m✗\033[0m %s — install: %s\n" "$tool" "${hints[$tool]}" >&2
+      missing=$((missing + 1))
+    fi
+  done
+  if [[ $missing -eq 0 ]]; then
+    [[ "$verbose" == "1" ]] && { echo ""; echo "All dependencies present." }
+    return 0
+  fi
+  echo "" >&2
+  echo "$missing tool(s) missing. Install them and retry." >&2
+  return 1
+}
+
 # Helper function to display available options
 print_usage() {
   echo "Usage:"
@@ -52,6 +107,8 @@ print_usage() {
   echo "  set_withdrawal_addr s e address   Set the withdrawal credentials for validator index start (mandatory) to end (optional) and Ethereum address"
   echo "  full_withdrawal s e               Withdraw from the network from validator index start to end - mandatory argument"
   echo "  sync_mapping [src]                Extend validator_names.yaml with post-genesis mnemonic deposits found on the beacon node (src default: main-mnemonic)"
+  echo "  send_funds address amount         Send ETH to an address - amount in ETH"
+  echo "  check_deps                        Verify every external tool this script needs is installed"
   echo "  help                              Print this help message"
   echo ""
   echo " To use an alternative endpoint run the script by setting the environment variable:"
@@ -75,6 +132,10 @@ fi
 # Loop through each argument
 for arg in "${command[@]}"; do
   case $arg in
+    "check_deps")
+      check_deps 1
+      exit $?
+      ;;
     "genesis")
       # Get the genesis block of the network
       genesis=$(curl -s $bn_endpoint/eth/v1/beacon/genesis | jq .data)
@@ -501,6 +562,47 @@ for arg in "${command[@]}"; do
         fi
       fi
       ;;
+    "send_funds")
+      # Send ETH to an address
+      if [[ $# -ne 3 ]]; then
+        echo "Usage: ${0} send_funds <address> <amount_in_eth>"
+        echo "  Example: ${0} send_funds 0x742d35Cc6634C0532925a3b844Bc9e7595f2bD18 10"
+        exit 1
+      fi
+      to_address=${command[2]}
+      eth_amount=${command[3]}
+      if [[ ! $to_address =~ ^0x[a-fA-F0-9]{40}$ ]]; then
+        echo "Invalid address format. Must be a valid Ethereum address (0x + 40 hex chars)."
+        exit 1
+      fi
+      if ! [[ "$eth_amount" =~ ^[0-9]+(\.[0-9]+)?$ ]] || (( $(echo "$eth_amount <= 0" | bc -l) )); then
+        echo "Invalid amount. Must be a positive number."
+        exit 1
+      fi
+      deposit_path="m/44'/60'/0'/0/7"
+      privatekey=$(ethereal hd keys --path="$deposit_path" --seed="$sops_mnemonic" | awk '/Private key/{print $NF}')
+      publickey=$(ethereal hd keys --path="$deposit_path" --seed="$sops_mnemonic" | awk '/Ethereum address/{print $NF}')
+      balance=$(curl -s --header 'Content-Type: application/json' --data-raw '{"jsonrpc":"2.0","method":"eth_getBalance","params":["'$publickey'","latest"],"id":0}' $rpc_endpoint | jq -r '.result' | python -c "import sys; print(int(sys.stdin.read(), 16) / 1e18)")
+      echo "Sending $eth_amount ETH to $to_address"
+      echo "  From: $publickey (balance: $balance ETH)"
+      echo "  To: $to_address"
+      echo "  Amount: $eth_amount ETH"
+      echo ""
+      echo "Continue? (y/n)"
+      read -r response
+      if [[ $response == "y" ]]; then
+        cast send \
+          --private-key "$privatekey" \
+          --rpc-url "$rpc_endpoint" \
+          --value "${eth_amount}ether" \
+          --gas-limit 21000 \
+          "$to_address"
+        echo "Transaction sent!"
+      else
+        echo "Cancelled."
+      fi
+      exit
+      ;;
     "topup")
       # Top-up one or more validators with additional ETH (Pectra upgrade feature)
       if [[ $# -ne 3 ]]; then

terraform/devnet-0/digitalocean.tf

@@ -40,6 +40,8 @@ variable "digitalocean_regions" {
 //                                        LOCALS
 ////////////////////////////////////////////////////////////////////////////////////////
 locals {
+  digitalocean_has_servers = length(local.digitalocean_nodes) > 0
+
   digitalocean_vpcs = {
     for region in var.digitalocean_regions : region => {
       name     = "${var.ethereum_network}-${region}"
@@ -139,8 +141,10 @@ data "digitalocean_ssh_key" "main" {
   name = var.digitalocean_ssh_key_name
 }
 
+// Skip VPC creation entirely on Hetzner-only devnets; an empty VPC per region
+// is still billable surface and blocks project teardown.
 resource "digitalocean_vpc" "main" {
-  for_each = local.digitalocean_vpcs
+  for_each = local.digitalocean_has_servers ? local.digitalocean_vpcs : {}
 
   name     = each.value["name"]
   region   = each.value["region"]