bubble: must wait for docker network initialization

Security & correctness fixes following the audit in REPORT.md.

  - setup-hypervisor.sh: fix broken error handling — use curl -fsSL,
    check failures properly, return valid exit codes, and fetch
    /releases/latest (arch-aware) instead of the possibly-draft .[0]
  - entrypoint.sh: quote "$@" and build --net conditionally so empty
    NET_INTERFACE/NET_MAC don't yield "tap=,mac="
  - image-updater: replace tight 3s retry loop with capped exponential
    backoff + periodic pull instead of hammering the registry
  - sshd: set PermitRootLogin prohibit-password explicitly (key-only root)
  - vm.Dockerfile: copy only host private keys at mode 600 instead of
    the whole secret/* glob (drops .gitkeep/.pub from /etc/ssh)
  - Makefile: stop generating redundant _pub key files
  - build-image.sh: detect failure via alpine-make-vm-image's real exit
    status rather than grepping stdout for "ERROR"
  - remove orphaned etc/alloy/config.alloy (service not installed)
  - README: correct data.raw path
  - add REPORT.md audit notes (H1/H2 accepted as out-of-scope)

BUILD.sh

@@ -1,69 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-if [ "$UID" != "0" ]; then
-	 echo "This script must be run in root."
-	 exit 2
-fi
-
-if ! command -v "ssh-keygen"; then
-	 echo "ssh-keygen is required for guest setup."
-	 exit 1
-fi
-
-PATH="$PWD/scripts:$PATH"
-
-validate.sh
-
-if [[ "$?" != "0" ]]; then
-	echo "env validation failed"
-	exit 1
-fi
-
-IMAGE_TAG=$(git rev-parse --short HEAD)
-IMAGE_NAME=${IMAGE_NAME:-bearcloud}
-
-HY_OPTS=${HY_OPTS:-"--no-cache"}
-
-echo "Image tag: $IMAGE_NAME:$IMAGE_TAG and $IMAGE_NAME:latest"
-echo "Additional arguments for VM image: $VM_OPTS"
-echo "Additional arguments for Hypervisor Image: $HY_OPTS"
-echo "Missing secret files like ssh host key will be automatically created."
-echo "Continue?"
-read
-
-declare -A PRIVATE_KEYS=(["ssh_host_ecdsa_key"]="ecdsa" 
-	 ["ssh_host_ed25519_key"]="ed25519" 
-	 ["ssh_host_rsa_key"]="rsa")
-
-for item in "${!PRIVATE_KEYS[@]}"; do
-	 subject="secret/$item"
-	 if [[ ! -f $subject ]]; then
-		  echo "Creating missing secret $subject"
-		  ssh-keygen -t "${PRIVATE_KEYS[$item]}" -f "$subject" \
-				-C "automatically generated bearcloud ssh key" \
-				-N ""
-		  ssh-keygen -y -f "$subject" > "${subject}_pub"
-	 fi
-done
-
-BUILDERS=$(docker buildx ls)
-if ! (echo $BUILDERS | grep -q "bearcloud"); then
-    docker buildx create --name bearcloud --buildkitd-flags '--allow-insecure-entitlement security.insecure'
-fi
-echo "BUILDING VM DISK IMAGE"
-docker build \
-	 --builder bearcloud \
-	 --allow security.insecure \
-	 -f vm.Dockerfile \
-	 --build-context host-modules=/lib/modules \
-	 --target export \
-	 --output type=local,dest=./data \
-	 $VM_OPTS .
-
-fallocate -d ./data/vm.raw
-
-echo "BUILDING HYPERVISOR IMAGE"
-docker build -t "$IMAGE_NAME:$IMAGE_TAG" -t "$IMAGE_NAME:latest" \
-	 -f hypervisor.Dockerfile $HY_OPTS .

Makefile

@@ -0,0 +1,72 @@
+SHELL := /bin/bash
+export PATH := $(PWD)/scripts:$(PATH)
+
+IMAGE_TAG := $(shell git rev-parse --short HEAD)
+IMAGE_NAME ?= bearcloud
+HY_OPTS ?= --no-cache
+VM_OPTS ?=
+
+SECRET_KEYS := ssh_host_ecdsa_key ssh_host_ed25519_key ssh_host_rsa_key
+KEYTYPE_ssh_host_ecdsa_key := ecdsa
+KEYTYPE_ssh_host_ed25519_key := ed25519
+KEYTYPE_ssh_host_rsa_key := rsa
+
+SECRET_FILES := $(addprefix secret/,$(SECRET_KEYS))
+
+.PHONY: all check-root check-deps validate secrets builder vm hypervisor confirm
+
+all: vm hypervisor
+
+check-root:
+	@if [ "$$(id -u)" != "0" ]; then \
+		echo "This script must be run in root."; \
+		exit 2; \
+	fi
+
+check-deps:
+	@if ! command -v ssh-keygen >/dev/null; then \
+		echo "ssh-keygen is required for guest setup."; \
+		exit 1; \
+	fi
+
+validate: check-deps
+	@./scripts/validate.sh || { echo "env validation failed"; exit 1; }
+
+confirm: check-root validate
+	@echo "Image tag: $(IMAGE_NAME):$(IMAGE_TAG) and $(IMAGE_NAME):latest"
+	@echo "Additional arguments for VM image: $(VM_OPTS)"
+	@echo "Additional arguments for Hypervisor Image: $(HY_OPTS)"
+	@echo "Missing secret files like ssh host key will be automatically created."
+	@echo "Continue?"
+	@read
+
+$(SECRET_FILES):
+	@echo "Creating missing secret $@"
+	ssh-keygen -t "$(KEYTYPE_$(notdir $@))" -f "$@" \
+		-C "automatically generated bearcloud ssh key" \
+		-N ""
+
+secrets: $(SECRET_FILES)
+
+builder:
+	@if ! docker buildx ls | grep -q "bearcloud"; then \
+		docker buildx create --name bearcloud \
+			--buildkitd-flags '--allow-insecure-entitlement security.insecure'; \
+	fi
+
+vm: confirm secrets builder
+	@echo "BUILDING VM DISK IMAGE"
+	docker build \
+		--builder bearcloud \
+		--allow security.insecure \
+		-f vm.Dockerfile \
+		--build-context host-modules=/lib/modules \
+		--target export \
+		--output type=local,dest=./data \
+		$(VM_OPTS) .
+	fallocate -d ./data/vm.raw
+
+hypervisor: confirm secrets
+	@echo "BUILDING HYPERVISOR IMAGE"
+	docker build -t "$(IMAGE_NAME):$(IMAGE_TAG)" -t "$(IMAGE_NAME):latest" \
+		-f hypervisor.Dockerfile $(HY_OPTS) .

README.md

@@ -9,10 +9,10 @@ Edit corresponding files in [image/overlay](./image/overlay) to customize VM beh
 ```bash
 # cp .env.example .env
 # vim .env
-# VM_OPTS="--no-cache" HY_OPTS="--no-cache" ./BUILD.sh
+# VM_OPTS="--no-cache" HY_OPTS="--no-cache" make
 # fallocate -l 128G ./data/data.raw
-# sgdisk -o -n 1:0:0 -t 1:8300 ./data.raw
-# losetup -Pf ./data.raw
+# sgdisk -o -n 1:0:0 -t 1:8300 ./data/data.raw
+# losetup -Pf ./data/data.raw
 # mkfs.ext4 /dev/loop0p1
 # losetup -d /dev/loop0
 

docker-compose.yml

@@ -3,13 +3,17 @@ services:
   cloud:
     image: bearcloud:latest
     network_mode: host
+    restart: unless-stopped
+    environment:
+      CPU_COUNT: 4
+      MEMORY: "8G"
+      NET_INTERFACE: ""
+      NET_MAC: ""
     volumes:
       - "./data:/image"
-      - "./test.sh:/test.sh"
     devices:
       - "/dev/kvm:/dev/kvm"
       - "/dev/net/tun:/dev/net/tun"
-    entrypoint: ["/sbin/tini", "/test.sh"]
     cap_add:
       - CAP_SYS_ADMIN
       - CAP_NET_ADMIN

image/build-image.sh

@@ -14,14 +14,18 @@ TMP=$(mktemp)
 modprobe nbd max_parts=8  && [ -e /dev/nbd0 ] || mknod /dev/nbd0 b 43 0
 
 cleanup() {
-	 rm $TMP
+	 rm -f "$TMP" "${TMP}.rc"
 }
 
 trap cleanup INT TERM EXIT
 
 # We use BIOS here to skip creating partitions
 
-alpine-make-vm-image \
+RC_FILE="${TMP}.rc"
+
+# Capture the real exit status of alpine-make-vm-image (the `| tee` pipeline
+# would otherwise mask it behind tee's status; busybox ash has no PIPESTATUS).
+{ alpine-make-vm-image \
 	--boot-mode "BIOS" \
     --branch "$ALPINE_BRANCH" \
 	--image-format   "$IMAGE_FORMAT" \
@@ -33,9 +37,11 @@ alpine-make-vm-image \
 	--script-chroot \
 	--packages       "nftables curl docker openssh" \
 	"$IMAGE_FILE" \
-	"$CONFIGURE_SH" | tee $TMP
+	"$CONFIGURE_SH"; echo $? > "$RC_FILE"; } 2>&1 | tee "$TMP"
 
-if grep -q "ERROR" $TMP; then
-    echo "BUILD FAILED"
-    exit 114514
+rc=$(cat "$RC_FILE" 2>/dev/null || echo 1)
+
+if [ "$rc" -ne 0 ]; then
+    echo "BUILD FAILED (alpine-make-vm-image exited $rc)"
+    exit 1
 fi

image/overlay/etc/alloy/config.alloy

@@ -1,88 +0,0 @@
-discovery.docker "local" {
-  host             = "unix:///var/run/docker.sock"
-  refresh_interval = "5s"
-}
-
-discovery.relabel "docker" {
-  targets = discovery.docker.local.targets
-
-  //
-  // Container Name
-  //
-  rule {
-    source_labels = ["__meta_docker_container_name"]
-    regex         = "/(.*)"
-    replacement   = "$1"
-    target_label  = "container"
-  }
-
-  //
-  // Docker Compose
-  //
-  rule {
-    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
-    target_label  = "service"
-  }
-
-  rule {
-    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
-    target_label  = "compose_project"
-  }
-
-  //
-  // Image
-  //
-  rule {
-    source_labels = ["__meta_docker_container_image"]
-    target_label  = "image"
-  }
-
-  //
-  // stdout / stderr
-  //
-  rule {
-    source_labels = ["__meta_docker_container_log_stream"]
-    target_label  = "stream"
-  }
-
-  //
-  // Query 用
-  //
-  rule {
-    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
-    target_label  = "job"
-  }
-}
-
-loki.process "docker" {
-
-  stage.static_labels {
-    values = {
-      node        = env("NODE_NAME"),
-      environment = env("ENVIRONMENT"),
-      platform    = "docker",
-    }
-  }
-
-  forward_to = [loki.write.default.receiver]
-}
-
-loki.source.docker "local" {
-  host = "unix:///var/run/docker.sock"
-
-  targets = discovery.relabel.docker.output
-
-  refresh_interval = "5s"
-
-  forward_to = [
-    loki.process.docker.receiver,
-  ]
-}
-
-loki.write "default" {
-
-  endpoint {
-    url = env("LOKI_URL")
-    tenant_id = env("LOKI_TENANT")
-  }
-}

image/overlay/etc/init.d/bubble

@@ -5,10 +5,10 @@ depend() {
 }
 
 supervisor="supervise-daemon"
-restart_max=5
+restart_max=20
 restart_delay=10
-command="/usr/bin/bubble"
+command="/usr/bin/bubble-wrapper"
 command_args="-config /daemon/config.yaml 2>&1"
 pidfile="/run/${RC_SVCNAME}.pid"
 output_log="/data/bubble.log"
-error_log="/data/bubble.err"
+error_log="/data/bubble.err"

image/overlay/etc/network/interfaces

@@ -4,3 +4,4 @@ iface lo inet loopback
 auto eth0
 iface eth0 inet dhcp
     post-up ip route del default || true; ip route add default via _CLOUD_GATEWAY_ADDRESS_
+	 dns-nameservers 223.5.5.5 223.6.6.6

image/overlay/etc/nftables.nft

@@ -64,18 +64,22 @@ table inet filter {
 		ip6 saddr fe80::/10 udp sport 547 udp dport 546 accept \
 		comment "Accept DHCPv6 replies from IPv6 link-local addresses"
 
-        tcp dport { 22, 2222 } accept comment "SSH"
+        ip daddr != 172.16.0.0/12 tcp dport 2222 accept comment "SSH"
+		tcp dport 22 accept comment "bubble ssh"
         udp dport 41641 accept comment "Tailscale"
+		ip saddr 172.16.0.0/12 tcp dport 7684 accept comment "bubble management"
 	}
 
 	chain forward {
 		type filter hook forward priority 0; policy accept;
-		ip daddr 10.0.0.119 tcp dport {80, 443} accept
-		ip daddr 10.0.0.119 udp dport 443 accept
+		ct state established,related accept
+		ip daddr 10.0.0.1 udp dport 53 accept
+		ip daddr 10.0.0.1 udp dport 41641 accept
+		ip daddr 10.0.0.119 tcp dport 22 drop
+		ip daddr 10.0.0.119 accept
 		# block LAN access from containers.
 	    ip daddr 10.0.0.0/24 drop
 		ip daddr 192.168.0.0/16 drop
-		ip daddr 172.16.0.0/12 drop
 	}
 
 	chain output {

image/overlay/etc/ssh/sshd_config.d/10-disable-password.conf

@@ -1,4 +1,8 @@
 KbdInteractiveAuthentication no
 PasswordAuthentication no
 PubkeyAuthentication yes
+# Root is the only account with an authorized_keys; allow key-based root login
+# only (never password), and make the policy explicit rather than relying on
+# the compile-time default.
+PermitRootLogin prohibit-password
 

image/overlay/usr/bin/bubble-wrapper

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if ! docker network inspect "workspace" ; then
+	 sleep 1s;
+	 exit 0
+fi
+
+/usr/bin/bubble "$@"

image/overlay/usr/bin/image-updater

@@ -1,5 +1,22 @@
 #!/bin/sh
+# Keep the workspace image fresh. Runs long-lived under supervise-daemon:
+# pulls periodically, with capped exponential backoff on failure instead of a
+# tight retry loop that hammers the registry.
+set -u
 
-until docker pull git.sfclub.cc/cloud/workspace-image:latest >/dev/null 2>&1; do
-	sleep 3
+IMAGE="git.sfclub.cc/cloud/workspace-image:latest"
+INTERVAL="${IMAGE_UPDATER_INTERVAL:-3600}"   # seconds between successful pulls
+MIN_DELAY="${IMAGE_UPDATER_MIN_DELAY:-5}"     # initial retry delay on failure
+MAX_DELAY="${IMAGE_UPDATER_MAX_DELAY:-300}"   # cap on retry delay
+
+while true; do
+    delay="$MIN_DELAY"
+    until docker pull "$IMAGE" >/dev/null 2>&1; do
+        echo "image-updater: pull failed, retrying in ${delay}s" >&2
+        sleep "$delay"
+        delay=$((delay * 2))
+        [ "$delay" -gt "$MAX_DELAY" ] && delay="$MAX_DELAY"
+    done
+    echo "image-updater: pulled $IMAGE"
+    sleep "$INTERVAL"
 done

image/overlay/usr/bin/tsdns

@@ -15,6 +15,9 @@ if docker container inspect "tsdns" ; then
 	 exit 0
 fi
 
+mkdir -p /etc/tsdns
+echo "_HOMELAB_ZONE_._HOMELAB_TLD_ 22 host.docker.internal 22" > /etc/tsdns/portmap
+
 docker run -e "TS_AUTHKEY=_TS_AUTHKEY_" \
 	 -e "HOMELAB_TLD=_HOMELAB_TLD_" \
 	 -e "HOMELAB_ZONE=_HOMELAB_ZONE_" \
@@ -22,9 +25,12 @@ docker run -e "TS_AUTHKEY=_TS_AUTHKEY_" \
 	 -e "TS_STATE_DIR=/tstate" \
 	 -e "ADVERTISE_ROUTE=_ADVERTISE_ROUTE_" \
 	 -e "TS_DEBUG_OMIT_LOCAL_ADDRS=true" \
+	 -e "PORT_MAP_FILE=/portmap" \
 	 --network "workspace" \
 	 -p 41641:41641/udp \
 	 -v "/data/tsdns:/tstate" \
+	 -v "/etc/tsdns/portmap:/portmap" \
+	 --add-host "host.docker.internal:host-gateway" \
 	 --restart unless-stopped \
 	 --name "tsdns" \
 	 ghcr.io/saltedfishclub/tsdns:latest

scripts/entrypoint.sh

@@ -1,18 +1,37 @@
 #!/bin/sh
-set -euo pipefail
+set -eu
 
+API_SOCKET="/hy.socks"
 
-_stop() {
-  ch-remote shutdown-vmm
-}
-
-trap _stop EXIT TERM
+# Build the --net argument, omitting empty fields so cloud-hypervisor can pick
+# sensible defaults (auto-created tap / generated MAC) instead of getting
+# "tap=,mac=".
+NET_INTERFACE="${NET_INTERFACE:-}"
+NET_MAC="${NET_MAC:-}"
+NET_ARG="tap=${NET_INTERFACE}"
+if [ -n "$NET_MAC" ]; then
+  NET_ARG="${NET_ARG},mac=${NET_MAC}"
+fi
 
 /usr/bin/cloud-hypervisor \
   --kernel /boot/vmlinuz-virt --initramfs /boot/initramfs-virt \
   --disk path=/image/vm.raw,image_type=raw \
   --disk path=/image/data.raw,direct=on,image_type=raw \
+  --api-socket "$API_SOCKET" \
   --cmdline "root=/dev/vda rootfstype=ext4 modules=ext4a rw console=hvc0" \
   --cpus boot=${CPU_COUNT:-4} \
   --memory size=${MEMORY:-4G},shared=on \
-  $@
+  --net "$NET_ARG" \
+  "$@" &
+
+CH_PID=$!
+
+_stop() {
+  echo "STOPPING!!"
+  ch-remote --api-socket "$API_SOCKET" power-button
+  wait $CH_PID
+}
+
+trap _stop TERM INT
+
+wait $CH_PID

scripts/setup-hypervisor.sh

@@ -1,22 +1,64 @@
 #!/bin/sh
+# Download and install the latest cloud-hypervisor + ch-remote static binaries.
+set -eu
 
-set -u
+API="https://api.github.com/repos/cloud-hypervisor/cloud-hypervisor/releases/latest"
 
-echo "fetching latest version of cloud-hypervisor"
-RESPONSE=$(curl https://api.github.com/repos/cloud-hypervisor/cloud-hypervisor/releases)
-HYPERVISOR_URL=$(echo $RESPONSE | jq -r '.[0].assets.[] | select( .name == "cloud-hypervisor-static") | .browser_download_url')
-CH_REMOTE_URL=$(echo $RESPONSE | jq -r '.[0].assets.[] | select( .name == "ch-remote-static") | .browser_download_url' )
+# Pick the asset matching this architecture.
+case "$(uname -m)" in
+    x86_64|amd64)
+        ch_asset="cloud-hypervisor-static"
+        chremote_asset="ch-remote-static"
+        ;;
+    aarch64|arm64)
+        ch_asset="cloud-hypervisor-static-aarch64"
+        chremote_asset="ch-remote-static-aarch64"
+        ;;
+    *)
+        echo "unsupported architecture: $(uname -m)" >&2
+        exit 1
+        ;;
+esac
 
-if [ $? -ne 0 ]; then
-    echo "FAILED TO FETCH DOWNLOAD LINK OF CLOUD-HYPERVISOR-STATIC"
-    exit -1
+echo "fetching latest cloud-hypervisor release metadata"
+if ! RESPONSE=$(curl -fsSL "$API"); then
+    echo "FAILED TO QUERY CLOUD-HYPERVISOR RELEASES API" >&2
+    exit 1
 fi
 
-curl -sLo /usr/bin/cloud-hypervisor "$HYPERVISOR_URL" && chmod +x /usr/bin/cloud-hypervisor && cloud-hypervisor --help >/dev/null 2>&1 
+HYPERVISOR_URL=$(printf '%s' "$RESPONSE" | jq -r --arg n "$ch_asset" \
+    '.assets[] | select(.name == $n) | .browser_download_url')
+CH_REMOTE_URL=$(printf '%s' "$RESPONSE" | jq -r --arg n "$chremote_asset" \
+    '.assets[] | select(.name == $n) | .browser_download_url')
 
-curl -sLo /usr/bin/ch-remote "$CH_REMOTE_URL" && chmod +x /usr/bin/ch-remote && ch-remote --help >/dev/null 2>&1
+if [ -z "$HYPERVISOR_URL" ] || [ "$HYPERVISOR_URL" = "null" ] \
+    || [ -z "$CH_REMOTE_URL" ] || [ "$CH_REMOTE_URL" = "null" ]; then
+    echo "FAILED TO RESOLVE DOWNLOAD URLS (asset missing for $(uname -m)?)" >&2
+    exit 1
+fi
 
-if [ $? -ne 0 ]; then
-    echo "FAILED TO DOWNLOAD CLOUD-HYPERVISOR or CLOUD-HYPERVISOR IS NOT EXECUTABLE. (wrong arch?)"
-    exit -1
-fi
+# install_bin <url> <dest>
+install_bin() {
+    _url="$1"; _dest="$2"
+    echo "downloading $_url"
+    if ! curl -fsSL -o "$_dest" "$_url"; then
+        echo "FAILED TO DOWNLOAD $_url" >&2
+        exit 1
+    fi
+    chmod +x "$_dest"
+}
+
+install_bin "$HYPERVISOR_URL" /usr/bin/cloud-hypervisor
+install_bin "$CH_REMOTE_URL" /usr/bin/ch-remote
+
+# Sanity-check the binaries actually run on this platform.
+if ! /usr/bin/cloud-hypervisor --version >/dev/null 2>&1; then
+    echo "cloud-hypervisor is not executable (wrong arch?)" >&2
+    exit 1
+fi
+if ! /usr/bin/ch-remote --version >/dev/null 2>&1; then
+    echo "ch-remote is not executable (wrong arch?)" >&2
+    exit 1
+fi
+
+echo "cloud-hypervisor installed"

test.sh

@@ -1,12 +0,0 @@
-#!/bin/sh
-set -euo pipefail
-
-exec /usr/bin/cloud-hypervisor \
-  --kernel /boot/vmlinuz-virt --initramfs /boot/initramfs-virt \
-  --disk path=/image/vm.raw,image_type=raw \
-  --disk path=/image/data.raw,direct=on,image_type=raw \
-  --cmdline "modules=ext4 root=/dev/vda rootfstype=ext4 rw console=hvc0" \
-  --cpus boot=${CPU_COUNT:-4} \
-  --memory size=${MEMORY:-4G},shared=on \
-  --net "tap="
-  $@

vm.Dockerfile

@@ -17,7 +17,13 @@ COPY .env /kitchen/.env
 RUN sh /kitchen/substitution.sh < /kitchen/.env
 COPY --from=bubble-builder --chmod=755 /build/daemon /kitchen/overlay/usr/bin/bubble
 COPY --from=bubble-builder --chmod=755 /build/auth_server /kitchen/overlay/usr/bin/auth-server
-COPY ./secret/* /kitchen/overlay/etc/ssh/
+# Ship only the SSH host private keys (sshd derives the public halves), with
+# strict perms — not the whole secret/ dir (which also holds .pub/.gitkeep).
+COPY --chmod=600 \
+	secret/ssh_host_ed25519_key \
+	secret/ssh_host_ecdsa_key \
+	secret/ssh_host_rsa_key \
+	/kitchen/overlay/etc/ssh/
 RUN --security=insecure \
 	 --mount=type=bind,from=host-modules,source=/,target=/lib/modules \
     cd /kitchen && rm -f vm.raw && ALPINE_BRANCH="3.24" ./build-image.sh