Skip to content

Commit c1032ae

Browse files
aslushnikovaslushnikov
authored
devops: simplify building webkit on linux bots (#3127)
Originally there was no way to build both webkit-gtk and webkit-wpe in the same checkout. As a result, we were: - building webkit-gtk & uploading it to CDN - building webkit-wpe & uploading it to CDN - downloading webkit-gtk & webkit-wpe from CDN and compiling a single webkit-gtk-wpe folder that we uploaded to CDN As of today, however, we can build WPE and GTK ports together. This patch starts using this to drastically simplify build process: - build webkit-gtk and webkit-wpe - use `archive.sh` script that compiles a single folder with both ports - upload archive to the cloud This should also fix currently failing webkit build that fails to assemble the `webkit-gtk-wpe` folder (I missed these codepaths while preparing for Ubuntu-20.04 bot). References #2745
1 parent d234dac commit c1032ae

File tree

6 files changed

+88
-264
lines changed

6 files changed

+88
-264
lines changed

‎browser_patches/buildbots/buildbot-ubuntu-18.04.sh

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,4 @@ git pull origin master
6666
../checkout_build_archive_upload.sh firefox-ubuntu-18.04 >/tmp/$(basename $0)--firefox.log || true
6767

6868
git pull origin master
69-
../checkout_build_archive_upload.sh webkit-gtk-ubuntu-18.04 >/tmp/$(basename $0)--webkit-gtk.log || true
70-
../checkout_build_archive_upload.sh webkit-wpe-ubuntu-18.04 >/tmp/$(basename $0)--webkit-wpe.log || true
71-
../checkout_build_archive_upload.sh webkit-gtk-wpe-ubuntu-18.04 >/tmp/$(basename $0)--webkit-gtk-wpe.log || true
69+
../checkout_build_archive_upload.sh webkit-ubuntu-18.04 >/tmp/$(basename $0)--webkit.log || true

‎browser_patches/buildbots/buildbot-ubuntu-20.04.sh

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,4 @@ fi
6363
touch "$IS_FIRST_RUN_FILE"
6464

6565
git pull origin master
66-
../checkout_build_archive_upload.sh webkit-gtk-ubuntu-20.04 >/tmp/$(basename $0)--webkit-gtk.log || true
67-
../checkout_build_archive_upload.sh webkit-wpe-ubuntu-20.04 >/tmp/$(basename $0)--webkit-wpe.log || true
68-
../checkout_build_archive_upload.sh webkit-gtk-wpe-ubuntu-20.04 >/tmp/$(basename $0)--webkit-gtk-wpe.log || true
66+
../checkout_build_archive_upload.sh webkit-ubuntu-20.04 >/tmp/$(basename $0)--webkit.log || true

‎browser_patches/checkout_build_archive_upload.sh

Lines changed: 3 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ fi
3232

3333
BROWSER_NAME=""
3434
EXTRA_BUILD_ARGS=""
35-
EXTRA_ARCHIVE_ARGS=""
3635
BUILD_FLAVOR="$1"
3736
BUILD_BLOB_NAME=""
3837
EXPECTED_HOST_OS=""
@@ -56,40 +55,12 @@ elif [[ "$BUILD_FLAVOR" == "firefox-win64" ]]; then
5655
EXTRA_BUILD_ARGS="--win64"
5756
EXPECTED_HOST_OS="MINGW"
5857
BUILD_BLOB_NAME="firefox-win64.zip"
59-
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-ubuntu-18.04" ]]; then
60-
BROWSER_NAME="webkit"
61-
EXTRA_BUILD_ARGS="--gtk"
62-
EXTRA_ARCHIVE_ARGS="--gtk"
63-
EXPECTED_HOST_OS="Ubuntu"
64-
EXPECTED_HOST_OS_VERSION="18.04"
65-
BUILD_BLOB_NAME="minibrowser-gtk-ubuntu-18.04.zip"
66-
elif [[ "$BUILD_FLAVOR" == "webkit-wpe-ubuntu-18.04" ]]; then
67-
BROWSER_NAME="webkit"
68-
EXTRA_BUILD_ARGS="--wpe"
69-
EXTRA_ARCHIVE_ARGS="--wpe"
70-
EXPECTED_HOST_OS="Ubuntu"
71-
EXPECTED_HOST_OS_VERSION="18.04"
72-
BUILD_BLOB_NAME="minibrowser-wpe-ubuntu-18.04.zip"
73-
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe-ubuntu-18.04" ]]; then
58+
elif [[ "$BUILD_FLAVOR" == "webkit-ubuntu-18.04" ]]; then
7459
BROWSER_NAME="webkit"
7560
EXPECTED_HOST_OS="Ubuntu"
7661
EXPECTED_HOST_OS_VERSION="18.04"
7762
BUILD_BLOB_NAME="minibrowser-gtk-wpe-ubuntu-18.04.zip"
78-
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-ubuntu-20.04" ]]; then
79-
BROWSER_NAME="webkit"
80-
EXTRA_BUILD_ARGS="--gtk"
81-
EXTRA_ARCHIVE_ARGS="--gtk"
82-
EXPECTED_HOST_OS="Ubuntu"
83-
EXPECTED_HOST_OS_VERSION="20.04"
84-
BUILD_BLOB_NAME="minibrowser-gtk-ubuntu-20.04.zip"
85-
elif [[ "$BUILD_FLAVOR" == "webkit-wpe-ubuntu-20.04" ]]; then
86-
BROWSER_NAME="webkit"
87-
EXTRA_BUILD_ARGS="--wpe"
88-
EXTRA_ARCHIVE_ARGS="--wpe"
89-
EXPECTED_HOST_OS="Ubuntu"
90-
EXPECTED_HOST_OS_VERSION="20.04"
91-
BUILD_BLOB_NAME="minibrowser-wpe-ubuntu-20.04.zip"
92-
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe-ubuntu-20.04" ]]; then
63+
elif [[ "$BUILD_FLAVOR" == "webkit-ubuntu-20.04" ]]; then
9364
BROWSER_NAME="webkit"
9465
EXPECTED_HOST_OS="Ubuntu"
9566
EXPECTED_HOST_OS_VERSION="20.04"
@@ -161,20 +132,6 @@ else
161132
fi
162133

163134
function generate_and_upload_browser_build {
164-
# webkit-gtk-wpe is a special build doesn't need to be built.
165-
if [[ "$BUILD_FLAVOR" == webkit-gtk-wpe-* ]]; then
166-
echo "-- combining binaries together"
167-
if ! ./webkit/download_gtk_and_wpe_and_zip_together.sh $ZIP_PATH; then
168-
return 10
169-
fi
170-
echo "-- uploading"
171-
if ! ./upload.sh $BUILD_BLOB_PATH $ZIP_PATH; then
172-
return 11
173-
fi
174-
return 0
175-
fi
176-
177-
# Other browser flavors follow typical build flow.
178135
echo "-- preparing checkout"
179136
if ! ./prepare_checkout.sh $BROWSER_NAME; then
180137
return 20
@@ -191,7 +148,7 @@ function generate_and_upload_browser_build {
191148
fi
192149

193150
echo "-- archiving to $ZIP_PATH"
194-
if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH "$EXTRA_ARCHIVE_ARGS"; then
151+
if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH; then
195152
return 23
196153
fi
197154

‎browser_patches/download.sh

Lines changed: 0 additions & 70 deletions
This file was deleted.

‎browser_patches/webkit/archive.sh

Lines changed: 83 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,14 @@ set -e
33
set +x
44

55
if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
6-
echo "usage: $(basename $0) [output-absolute-path] [--wpe|--gtk]"
6+
echo "usage: $(basename $0) [output-absolute-path]"
77
echo
88
echo "Generate distributable .zip archive from ./checkout folder that was previously built."
99
echo
1010
exit 0
1111
fi
1212

1313
ZIP_PATH=$1
14-
LINUX_FLAVOR=$2
1514
if [[ $ZIP_PATH != /* ]]; then
1615
echo "ERROR: path $ZIP_PATH is not absolute"
1716
exit 1
@@ -45,80 +44,101 @@ main() {
4544
fi
4645
}
4746

47+
copyLibrariesForWPE() {
48+
# Expect target directory to be passed in as first argument.
49+
local tmpdir=$1
50+
51+
# copy runner
52+
cp -t $tmpdir $SCRIPTS_DIR/pw_run.sh
53+
54+
# copy all relevant binaries
55+
cp -t $tmpdir ./WebKitBuild/WPE/Release/bin/MiniBrowser ./WebKitBuild/WPE/Release/bin/WPE*Process
56+
# copy all relevant shared objects
57+
# - exclude gstreamer plugins
58+
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/MiniBrowser \
59+
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
60+
| grep -v '/libgst.*so' \
61+
| xargs cp -t $tmpdir
62+
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/WPENetworkProcess \
63+
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
64+
| grep -v '/libgst.*so' \
65+
| xargs cp -t $tmpdir
66+
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/WPEWebProcess \
67+
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
68+
| grep -v '/libgst.*so' \
69+
| xargs cp -t $tmpdir
70+
# Copy some wayland libraries required for Web Process
71+
if ls WebKitBuild/WPE/DependenciesWPE/Root/lib/libva\-* 2>&1 >/dev/null; then
72+
cp -d -t $tmpdir WebKitBuild/WPE/DependenciesWPE/Root/lib/libva\-*
73+
fi
74+
# Injected bundle is loaded dynamicly via dlopen => not bt listed by ldd.
75+
cp -t $tmpdir WebKitBuild/WPE/Release/lib/libWPEInjectedBundle.so
76+
if test -d $PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib/gio/modules/; then
77+
mkdir -p $tmpdir/gio/modules
78+
cp -t $tmpdir/gio/modules $PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib/gio/modules/*
79+
fi
80+
81+
cd $tmpdir
82+
ln -s libWPEBackend-fdo-1.0.so.1 libWPEBackend-fdo-1.0.so
83+
cd -
84+
85+
# Strip copied binaries.
86+
cd $tmpdir
87+
strip --strip-unneeded * || true
88+
cd -
89+
}
90+
91+
copyLibrariesForGTK() {
92+
# Expect target directory to be passed in as first argument.
93+
local tmpdir=$1
94+
95+
# copy runner
96+
cp -t $tmpdir $SCRIPTS_DIR/pw_run.sh
97+
98+
# copy all relevant binaries
99+
cp -t $tmpdir ./WebKitBuild/GTK/Release/bin/MiniBrowser ./WebKitBuild/GTK/Release/bin/WebKit*Process
100+
# copy all relevant shared objects
101+
# - exclude gstreamer plugins
102+
# - exclude libdrm
103+
LD_LIBRARY_PATH="$PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib" ldd WebKitBuild/GTK/Release/bin/MiniBrowser \
104+
| grep -o '[^ ]*WebKitBuild/GTK/[^ ]*' \
105+
| grep -v '/libgst.*so' \
106+
| grep -v '/libdrm.so' \
107+
| xargs cp -t $tmpdir
108+
109+
# Injected bundle is loaded dynamicly via dlopen => not bt listed by ldd.
110+
cp -t $tmpdir WebKitBuild/GTK/Release/lib/libwebkit2gtkinjectedbundle.so
111+
if test -d $PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib/gio/modules; then
112+
mkdir -p $tmpdir/gio/modules
113+
cp -t $tmpdir/gio/modules $PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib/gio/modules/*
114+
fi
115+
116+
# we failed to nicely build libgdk_pixbuf - expect it in the env
117+
rm -f $tmpdir/libgdk_pixbuf*
118+
119+
# tar resulting directory and cleanup TMP.
120+
cd $tmpdir
121+
strip --strip-unneeded * || true
122+
cd -
123+
}
124+
48125
createZipForLinux() {
49126
# create a TMP directory to copy all necessary files
50127
local tmpdir=$(mktemp -d -t webkit-deploy-XXXXXXXXXX)
51128
mkdir -p $tmpdir
129+
mkdir -p $tmpdir/minibrowser-gtk
130+
mkdir -p $tmpdir/minibrowser-wpe
52131

53132
# copy runner
54133
cp -t $tmpdir $SCRIPTS_DIR/pw_run.sh
55134
# copy protocol
56135
node $SCRIPTS_DIR/concat_protocol.js > $tmpdir/protocol.json
57136

58-
if [[ "$LINUX_FLAVOR" == "--wpe" ]]; then
59-
# copy all relevant binaries
60-
cp -t $tmpdir ./WebKitBuild/WPE/Release/bin/MiniBrowser ./WebKitBuild/WPE/Release/bin/WPE*Process
61-
# copy all relevant shared objects
62-
# - exclude gstreamer plugins
63-
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/MiniBrowser \
64-
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
65-
| grep -v '/libgst.*so' \
66-
| xargs cp -t $tmpdir
67-
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/WPENetworkProcess \
68-
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
69-
| grep -v '/libgst.*so' \
70-
| xargs cp -t $tmpdir
71-
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/WPEWebProcess \
72-
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
73-
| grep -v '/libgst.*so' \
74-
| xargs cp -t $tmpdir
75-
# Copy libvpx.so.5 as Ubuntu 20.04 comes with libvpx.so.6
76-
ldd WebKitBuild/WPE/Release/bin/MiniBrowser | grep -o '[^ ]*\/libvpx.so.5[^ ]*' | xargs cp -t $tmpdir
77-
# Copy some wayland libraries required for Web Process t
78-
if ls WebKitBuild/WPE/DependenciesWPE/Root/lib/libva\-* 2>&1 >/dev/null; then
79-
cp -d -t $tmpdir WebKitBuild/WPE/DependenciesWPE/Root/lib/libva\-*
80-
fi
81-
# Injected bundle is loaded dynamicly via dlopen => not bt listed by ldd.
82-
cp -t $tmpdir WebKitBuild/WPE/Release/lib/libWPEInjectedBundle.so
83-
if test -d $PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib/gio/modules/; then
84-
mkdir -p $tmpdir/gio/modules
85-
cp -t $tmpdir/gio/modules $PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib/gio/modules/*
86-
fi
87-
88-
cd $tmpdir
89-
ln -s libWPEBackend-fdo-1.0.so.1 libWPEBackend-fdo-1.0.so
90-
cd -
91-
elif [[ "$LINUX_FLAVOR" == "--gtk" ]]; then
92-
# copy all relevant binaries
93-
cp -t $tmpdir ./WebKitBuild/GTK/Release/bin/MiniBrowser ./WebKitBuild/GTK/Release/bin/WebKit*Process
94-
# copy all relevant shared objects
95-
# - exclude gstreamer plugins
96-
# - exclude libdrm
97-
LD_LIBRARY_PATH="$PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib" ldd WebKitBuild/GTK/Release/bin/MiniBrowser \
98-
| grep -o '[^ ]*WebKitBuild/GTK/[^ ]*' \
99-
| grep -v '/libgst.*so' \
100-
| grep -v '/libdrm.so' \
101-
| xargs cp -t $tmpdir
102-
103-
# Injected bundle is loaded dynamicly via dlopen => not bt listed by ldd.
104-
cp -t $tmpdir WebKitBuild/GTK/Release/lib/libwebkit2gtkinjectedbundle.so
105-
# Copy libvpx.so.5 as Ubuntu 20.04 comes with libvpx.so.6
106-
ldd WebKitBuild/GTK/Release/bin/MiniBrowser | grep -o '[^ ]*\/libvpx.so.5[^ ]*' | xargs cp -t $tmpdir
107-
if test -d $PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib/gio/modules; then
108-
mkdir -p $tmpdir/gio/modules
109-
cp -t $tmpdir/gio/modules $PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib/gio/modules/*
110-
fi
111-
112-
# we failed to nicely build libgdk_pixbuf - expect it in the env
113-
rm -f $tmpdir/libgdk_pixbuf*
114-
else
115-
echo "ERROR: must specify --gtk or --wpe"
116-
exit 1
117-
fi
137+
copyLibrariesForWPE $tmpdir/minibrowser-wpe
138+
copyLibrariesForGTK $tmpdir/minibrowser-gtk
118139

119140
# tar resulting directory and cleanup TMP.
120141
cd $tmpdir
121-
strip --strip-unneeded * || true
122142
zip --symlinks -r $ZIP_PATH ./
123143
cd -
124144
rm -rf $tmpdir

0 commit comments

Comments
 (0)