Skip to content

Commit f9fe1ab

Browse files
struanbstruanb
committed
Refactor docker swarm test and add support for modified MTU
- RunCVM nodes are launched on a custom network with a custom MTU - Nodes detect their MTU, then launch dockerd --mtu accordingly - Nodes create their own docker_gwbridge network with custom MTU - After swarm init, primary node recreates ingress network - Refactored test simplifies and standardises in-node docker.sh logic
1 parent 0424996 commit f9fe1ab

File tree

4 files changed

+214
-97
lines changed

4 files changed

+214
-97
lines changed

‎README.md

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,14 @@ Gain another interactive console on `ubuntu1`:
3636
docker exec -it ubuntu1 bash
3737
```
3838

39+
Launch a 3-node Docker Swarm on a network with 9000 MTU and, on the swarm, an http global service:
40+
41+
```console
42+
git clone https://github.com/newsnowlabs/runcvm.git && \
43+
cd runcvm/tests/00-http-docker-swarm && \
44+
./test
45+
```
46+
3947
## RunCVM-in-Portainer walk-through
4048

4149
[![Playing around with RunCVM, a docker runtime plugin](https://i.ytimg.com/vi/OENaWDlCWKg/maxresdefault.jpg)](https://www.youtube.com/watch?v=OENaWDlCWKg "Playing around with RunCVM, a docker runtime plugin")
@@ -89,9 +97,7 @@ RunCVM is free and open-source, licensed under the Apache Licence, Version 2.0.
8997
- Run unusual container workloads, like `dockerd` and `systemd` that will not run in standard container runtimes
9098
- Maintain a similar experience within a RunCVM VM as within a container: process table, network interfaces, stdio, exit code handling should broadly similar to maximise compatibility
9199
- Container start/stop/kill semantics respected, where possible providing clean VM shutdown on stop
92-
- VM console accessible as one would expect using `docker run -it`, `docker start -ai` and `docker attach` (but stderr is not yet separated from stdout)
93-
- Support for `docker exec`
94-
- Good support for most other `docker container` subcommands
100+
- VM console accessible as one would expect using `docker run -it`, `docker start -ai` and `docker attach` (and so on), generally good support for other `docker container` subcommands
95101
- Efficient container startup, by using virtiofs to serve a container's filesystem directly to a VM (instead of unpacking an image into a backing file)
96102
- Improved security compared to the standard container runtime, and as much security as possible without compromising the simplicity of the implementation
97103
- Command-line and image-embedded options for customising the a container's VM specifications, devices, kernel
@@ -178,7 +184,7 @@ In the below summary of RunCVM's current main features and limitations, [+] is u
178184
- [-] Bind-mounting host sockets or devices, and `--device` is unsupported
179185
- Networking
180186
- [+] The default bridge network is supported
181-
- [+] Custom/user-defined networks specified using `--network` are supported, including Docker DNS resolution of container names
187+
- [+] Custom/user-defined networks specified using `--network` are supported, including Docker DNS resolution of container names and respect for custom network MTU
182188
- [+] `--publish` (or `-p`) is supported
183189
- [+] `--dns`, `--dns-option`, `--dns-search` are supported
184190
- [+] `--ip` is supported

‎tests/00-http-docker-swarm/node/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
FROM alpine
1+
FROM alpine:3.18
22

3-
RUN apk add --update docker bash curl
3+
RUN apk add --update --no-cache docker bash curl jq iproute2
44

55
ADD docker.sh /usr/local/bin/
66

‎tests/00-http-docker-swarm/node/docker.sh

Lines changed: 99 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
#!/bin/bash
22

3+
log() {
4+
echo "$1"
5+
}
6+
37
node_state() {
48
NodeState=$(docker info --format '{{.Swarm.LocalNodeState}}')
59
IsManager=$(docker info --format '{{.Swarm.ControlAvailable}}')
@@ -17,100 +21,136 @@ cgroupfs_mount() {
1721
mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
1822
fi
1923

20-
if true; then
21-
(
22-
cd /sys/fs/cgroup
23-
for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
24-
mkdir -p $sys
25-
if ! mountpoint -q $sys; then
26-
if ! mount -n -t cgroup -o $sys cgroup $sys; then
27-
rmdir $sys || true
28-
fi
29-
fi
30-
done
31-
)
32-
fi
24+
cd /sys/fs/cgroup
25+
for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
26+
mkdir -p $sys
27+
if ! mountpoint -q $sys; then
28+
if ! mount -n -t cgroup -o $sys cgroup $sys; then
29+
rmdir $sys || true
30+
fi
31+
fi
32+
done
3333

3434
if ! mountpoint -q /sys/fs/cgroup/unified; then
3535
mkdir -p /sys/fs/cgroup/unified
3636
mount -t cgroup2 cgroup2 /sys/fs/cgroup/unified
3737
fi
38+
39+
true
3840
}
3941

40-
cgroupfs_mount || true
42+
cgroupfs_mount
4143

4244
ulimit -u unlimited
4345

4446
modprobe ip_vs
4547

4648
h=$(hostname)
4749

48-
while true; do dockerd >>/var/log/dockerd.log 2>&1; done &
50+
log "Checking network ..."
51+
DOCKER_IF=$(ip -json route show | jq -j '.[] | select(.dst == "default") | .dev')
52+
read -r DOCKER_IF_IP DOCKER_IF_MTU <<< \
53+
$(ip -json addr show eth0 | jq -j '.[0] | .addr_info[0].local, " ", .mtu')
54+
55+
log "- DOCKER_IF_IP=$DOCKER_IF_IP DOCKER_IF_MTU=$DOCKER_IF_MTU"
4956
50-
echo "> ($h) Waiting for dockerd to start ..."
51-
while ! docker ps >/dev/null 2>&1; do
57+
# Start dockerd and keep it running
58+
log "Launching dockerd ..."
59+
while true; do dockerd --mtu $DOCKER_IF_MTU >>/var/log/dockerd.log 2>&1; done &
60+
61+
for i in $(seq 1 10 | sort -nr)
62+
do
63+
log "Waiting for dockerd to start (#$i) ..."
64+
docker ps >/dev/null 2>1 && break
65+
[ $i -eq 1 ] && exit 1
5266
sleep 0.5
5367
done
5468
55-
echo "> ($h) dockerd started"
69+
log "dockerd started"
5670
5771
node_state
58-
echo "> ($h) docker swarm: node state = $NodeState; manager=$IsManager"
72+
log "docker swarm: node state = $NodeState; manager=$IsManager"
73+
74+
log "Creating docker_gwbridge network with MTU $DOCKER_IF_MTU"
75+
docker network create -d bridge \
76+
--subnet 172.18.0.0/16 \
77+
--opt com.docker.network.bridge.name=docker_gwbridge \
78+
--opt com.docker.network.bridge.enable_icc=false \
79+
--opt com.docker.network.bridge.enable_ip_masquerade=true \
80+
--opt com.docker.network.driver.mtu=$DOCKER_IF_MTU \
81+
docker_gwbridge
5982
6083
if [ "$NodeState" = "inactive" ] || [ "$NodeState" = "pending" ]; then
6184
62-
if [ -f /swarm/worker ]; then
85+
if [ "$NODE" != "1" ]; then
86+
87+
for i in $(seq 1 20 | sort -nr)
88+
do
89+
log "Waiting for swarm manager startup (#$i) ..."
90+
[ -f /swarm/worker ] && break
91+
[ $i -eq 1 ] && exit 1
92+
sleep 1
93+
done
6394
64-
while ! . /swarm/worker
95+
log "Swarm manager has started up."
96+
for i in $(seq 1 20 | sort -nr)
6597
do
98+
log "Joining swarm (#$i) ..."
99+
. /swarm/worker && break
100+
[ $i -eq 1 ] && exit 1
66101
sleep 0.5
67102
done
68103
69-
echo "> ($h) Joined swarm!"
104+
log "Joined swarm!"
70105
71106
else
72-
if docker swarm init >/dev/null; then
73-
docker swarm join-token worker | grep docker >/swarm/worker
74-
75-
echo "> ($h) Initialised swarm!"
76-
77-
echo "> ($h) Sleeping 10s to wait for other nodes ..."
78-
sleep 10
79-
80-
echo "> ($h) Listing nodes"
81-
docker node ls
82-
echo
83-
84-
echo "> ($h) Creating 'http' service (please be patient) ..."
85-
docker service create --name=http --mode=global -p 80:80 \
86-
alpine ash -c 'apk update && apk add mini_httpd && mkdir -p /www && echo "<!DOCTYPE html>
87-
<html lang="en">
88-
<head>
89-
<meta charset="utf-8" />
90-
<title>hostname $HOSTNAME</title>
91-
</head>
92-
<body>
93-
Server is online
94-
</body>
95-
</html>" >/www/index.html && mini_httpd -d /www -D -l /dev/stdout'
96-
echo
97-
98-
echo "> ($h) Itemising 'http' service ..."
99-
docker service ps http
100-
echo
101-
102-
echo "> ($h) 'http' service launched"
107+
108+
log "Initialising swarm ..."
109+
if ! docker swarm init >/dev/null; then
110+
log "Swarm initialisation FAILED!"
111+
exit 1
103112
fi
113+
114+
log "Swarm initialised!"
115+
116+
log "Removing default ingress ..."
117+
echo y | docker network rm ingress
118+
119+
log "Creating new ingress with MTU $DOCKER_IF_MTU"
120+
docker network create \
121+
--driver=overlay \
122+
--ingress \
123+
--subnet=10.0.0.0/24 \
124+
--gateway=10.0.0.2 \
125+
--opt com.docker.network.driver.mtu=$DOCKER_IF_MTU \
126+
ingress
127+
128+
log "Writing swarm 'join token' to shared storage and waiting for other nodes ..."
129+
docker swarm join-token worker | grep docker >/swarm/worker
130+
131+
for i in $(seq 1 30 | sort -nr)
132+
do
133+
nodes=$(docker node ls --format '{{json .}}' | wc -l)
134+
log "Waiting for remaining $((NODES-nodes)) of $NODES nodes to join swarm (#$i) ..."
135+
[ $nodes -eq $NODES ] && break
136+
[ $i -eq 1 ] && log "Swarm failed!" && exit 1
137+
sleep 1
138+
done
139+
140+
log "Swarm nodes started:"
141+
docker node ls
142+
echo
143+
144+
# Log this trigger line last.
145+
log "Swarm complete!"
146+
104147
fi
105148
fi
106149
107150
node_state
108151
if [ "$NodeState" = "active" ] && [ "$IsManager" = "true" ]; then
109-
echo "> ($h) Now running: docker service logs -n 0 -f http"
110-
echo
111-
docker service logs -n 0 -f http
152+
log "Manager ready"
112153
fi
113154
114-
echo "> ($h) Dropping to shell. Type CTRL+D to exit"
115-
echo
116-
bash -i
155+
log "Looping indefinitely ..."
156+
while true; do sleep infinity; done

0 commit comments

Comments
 (0)