Skip to content

Commit f482ef9

Browse files
struanbstruanb
committed
Extend Docker swarm test to support Docker Ingress Routing Daemon
- Set DIRD=1 to enable installation of https://github.com/newsnowlabs/docker-ingress-routing-daemon on the swarm - This serves as a more advanced simulation of a Docker swarm as well as an integration test for DIRD.
1 parent 0dca292 commit f482ef9

File tree

3 files changed

+106
-28
lines changed

3 files changed

+106
-28
lines changed

‎tests/00-http-docker-swarm/node/Dockerfile

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ FROM alpine:3.18
22

33
RUN apk add --update --no-cache docker bash curl jq iproute2
44

5-
ADD docker.sh /usr/local/bin/
5+
ADD docker.sh https://raw.githubusercontent.com/newsnowlabs/docker-ingress-routing-daemon/main/docker-ingress-routing-daemon /usr/local/bin/
6+
7+
RUN chmod 755 /usr/local/bin/docker-ingress-routing-daemon
68

79
ENTRYPOINT ["docker.sh"]

‎tests/00-http-docker-swarm/node/docker.sh

Lines changed: 69 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,21 @@ cgroupfs_mount() {
3939
true
4040
}
4141

42+
detect_ingress_ip() {
43+
local FILE="$1"
44+
local IP
45+
46+
for i in $(seq 1 30 | sort -nr)
47+
do
48+
IP=$(docker network inspect ingress --format '{{index (split (index .Containers "ingress-sbox").IPv4Address "/") 0}}' 2>/dev/null)
49+
[ -n "$IP" ] && break
50+
[ $i -eq 1 ] && log "Ingress IP detection for this node failed!" && exit 1
51+
sleep 0.5
52+
done
53+
54+
echo -n "$IP" >$FILE
55+
}
56+
4257
cgroupfs_mount
4358

4459
ulimit -u unlimited
@@ -55,7 +70,7 @@ read -r DOCKER_IF_IP DOCKER_IF_MTU <<< \
5570
log "- DOCKER_IF_IP=$DOCKER_IF_IP DOCKER_IF_MTU=$DOCKER_IF_MTU"
5671
5772
# Start dockerd and keep it running
58-
log "Launching dockerd ..."
73+
log "Launching 'dockerd --mtu $DOCKER_IF_MTU' ..."
5974
while true; do dockerd --mtu $DOCKER_IF_MTU >>/var/log/dockerd.log 2>&1; done &
6075
6176
for i in $(seq 1 10 | sort -nr)
@@ -113,20 +128,23 @@ if [ "$NodeState" = "inactive" ] || [ "$NodeState" = "pending" ]; then
113128
114129
log "Swarm initialised!"
115130
116-
log "Removing default ingress ..."
117-
echo y | docker network rm ingress
131+
if [ -n "$MTU" ] && [ "$MTU" -gt 0 ]; then
118132
119-
log "Creating new ingress with MTU $DOCKER_IF_MTU"
120-
docker network create \
121-
--driver=overlay \
122-
--ingress \
123-
--subnet=10.0.0.0/24 \
124-
--gateway=10.0.0.2 \
125-
--opt com.docker.network.driver.mtu=$DOCKER_IF_MTU \
126-
ingress
133+
log "Removing default ingress ..."
134+
echo y | docker network rm ingress
127135
128-
log "Writing swarm 'join token' to shared storage and waiting for other nodes ..."
129-
docker swarm join-token worker | grep docker >/swarm/worker
136+
log "Creating new ingress with MTU $DOCKER_IF_MTU"
137+
docker network create \
138+
--driver=overlay \
139+
--ingress \
140+
--subnet=10.0.0.0/24 \
141+
--gateway=10.0.0.1 \
142+
--opt com.docker.network.driver.mtu=$DOCKER_IF_MTU \
143+
ingress
144+
fi
145+
146+
log "Writing swarm 'join token' to shared storage and waiting for other nodes ..."
147+
mkdir -p /swarm/nodes && docker swarm join-token worker | grep docker >/swarm/worker
130148
131149
for i in $(seq 1 30 | sort -nr)
132150
do
@@ -140,11 +158,46 @@ if [ "$NodeState" = "inactive" ] || [ "$NodeState" = "pending" ]; then
140158
log "Swarm nodes started:"
141159
docker node ls
142160
echo
161+
162+
fi
163+
164+
log "Log memory consumption ..."
165+
free
166+
167+
# Log this trigger line last
168+
# BUT before running DIRD.
169+
log "Swarm complete!"
170+
171+
# Optionally run DIRD.
172+
# Do this after logging "Swarm complete" so that test script proceeds to launch the service;
173+
# as, until the service is launched, the nodes' ingress network IPs will not yet be defined.
174+
if [ "$DIRD" = "1" ]; then
143175
144-
# Log this trigger line last.
145-
log "Swarm complete!"
146-
176+
log "Detecting node ingress network IP ..."
177+
detect_ingress_ip /swarm/nodes/$NODE
178+
log "Detected node ingress network IP '$(cat /swarm/nodes/$NODE)'"
179+
180+
log "Waiting for all nodes' ingress network IPs ..."
181+
for i in $(seq 1 30 | sort -nr)
182+
do
183+
[ $(ls /swarm/nodes/ | wc -l) -eq $NODES ] && break
184+
[ $i -eq 1 ] && log "Ingress IP detection for all nodes failed!" && exit 1
185+
sleep 0.5
186+
done
187+
188+
for n in $(ls /swarm/nodes)
189+
do
190+
IPs+="$(cat /swarm/nodes/$n),"
191+
echo "$n: '$(cat /swarm/nodes/$n)'"
192+
done
193+
194+
IPs=$(echo $IPs | sed 's/,$//')
195+
196+
log "Running docker-ingress-routing-daemon --preexisting --ingress-gateway-ips $IPs --install ..."
197+
while true; do /usr/local/bin/docker-ingress-routing-daemon --preexisting --ingress-gateway-ips "$IPs" --install; sleep 1; done &
198+
147199
fi
200+
148201
fi
149202
150203
node_state

‎tests/00-http-docker-swarm/test

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,23 @@
11
#!/bin/bash -e
22

33
NODE=00-http-docker-swarm-node
4-
NODES=3
5-
MTU=9000
4+
5+
# Number of nodes
6+
NODES=${NODES:-3}
7+
8+
# Network MTU to deploy in Docker network, RunCVM container VM nodes, and on Docker and swarm ingress network running on those nodes.
9+
MTU=${MTU:-9000}
10+
11+
# Set to "1" to enable installation of https://github.com/newsnowlabs/docker-ingress-routing-daemon on the swarm
12+
DIRD=${DIRD:-0}
13+
14+
# Set to "1" to disable cleanup of Docker image
15+
NO_CLEAN_IMAGE=${NO_CLEAN_IMAGE:-0}
616

717
log() {
818
local opts
919
if [ "$1" = "-n" ]; then opts="-n"; shift; fi
10-
echo $opts "($h) > $1"
20+
echo $opts "$(date +%Y%m%d.%H%M%S) ($h) > $1"
1121
}
1222

1323
nodes() {
@@ -33,7 +43,7 @@ _cleanup() {
3343
fi
3444
echo
3545

36-
if [ -n "$IMAGE" ]; then
46+
if [ -n "$IMAGE" ] && [ "$NO_CLEAN_IMAGE" != "1" ]; then
3747
log "Cleaning up temporary image ..."
3848
docker rmi $IMAGE
3949
echo
@@ -83,8 +93,13 @@ log "Build image ..."
8393
docker build --iidfile /tmp/iid -f node/Dockerfile node/
8494
IMAGE=$(cat /tmp/iid)
8595

86-
log "Creating network 'runcvm-mtu' with MTU $MTU ..."
87-
docker network create --opt com.docker.network.driver.mtu=$MTU --scope=local runcvm-mtu
96+
if [ -n "$MTU" ] && [ "$MTU" -gt 0 ]; then
97+
log "Creating network 'runcvm-mtu' with MTU $MTU ..."
98+
docker network create --opt com.docker.network.driver.mtu=$MTU --scope=local runcvm-mtu
99+
else
100+
log "Creating network 'runcvm-mtu' with default (unspecified) MTU ..."
101+
docker network create --scope=local runcvm-mtu
102+
fi
88103

89104
log "Launching $NODES x RunCVM nodes with image $IMAGE ..."
90105
for n in $(seq 1 $NODES)
@@ -101,11 +116,14 @@ do
101116
--rm \
102117
--runtime=runcvm \
103118
--network=runcvm-mtu \
104-
-p $((8080+$n-1)):80 \
119+
--publish=$((8080+$n-1)):80 \
105120
--name=$NODE$n \
106121
--hostname=$NODE$n \
122+
--memory=256m \
107123
--env=NODE=$n \
108124
--env=NODES=$NODES \
125+
--env=MTU=$MTU \
126+
--env=DIRD=$DIRD \
109127
--mount=type=volume,src=swarm,dst=/swarm \
110128
$diskopt \
111129
$IMAGE
@@ -128,7 +146,7 @@ mini_httpd -d /www -D -l /dev/stdout -c '**.cgi'
128146
_EOE_
129147
)"
130148

131-
for i in $(seq 1 20 | sort -nr)
149+
for i in $(seq 1 200 | sort -nr)
132150
do
133151
replicas=$(docker exec ${NODE}1 docker service ls --format='{{ .Replicas }}' --filter='Name=http')
134152
log "Waiting for remainder of $replicas replicas to launch (#$i) ..."
@@ -142,9 +160,14 @@ sleep 1
142160
# Allow final test to complete, even if we encounter errors
143161
set +e
144162

145-
DOCKER_IPV4="10.0.0."
163+
if [ "$DIRD" = "1" ]; then
164+
DOCKER_IPV4=$(docker network inspect runcvm-mtu --format='{{(index .IPAM.Config 0).Gateway}}')
165+
else
166+
DOCKER_IPV4="10.0.0."
167+
fi
168+
169+
log "Running $NODE test looking for '$DOCKER_IPV4' at $(date) ..."
146170

147-
log "Running $NODE test looking for $DOCKER_IPV4 at $(date) ..."
148171
ERRORS=0
149172
TESTS=0
150173
for loop in $(seq 1 250)
@@ -156,7 +179,7 @@ do
156179
# Uncomment if running inside a Dockside devtainer (which must be preconnected to a precreated runcvm-mtu Docker network).
157180
# host=http://$NODE$((i+1)):80/
158181

159-
response=$(curl -is $host)
182+
response=$(curl --max-time 1 -is $host)
160183
ERROR=$?
161184

162185
if [ $ERROR -eq 0 ]; then

0 commit comments

Comments
 (0)