diff --git a/fah-gpu-bastet/.env-example b/fah-gpu-bastet/.env-example new file mode 100644 index 0000000..f25394c --- /dev/null +++ b/fah-gpu-bastet/.env-example @@ -0,0 +1,3 @@ +VERSION=25.11.0 +CUDA_RELEASE=11.7.1 +ROCM_RELEASE=6.4.4 diff --git a/fah-gpu-bastet/Dockerfile b/fah-gpu-bastet/Dockerfile new file mode 100644 index 0000000..629ef41 --- /dev/null +++ b/fah-gpu-bastet/Dockerfile @@ -0,0 +1,61 @@ +# Folding@home GPU Container + +# CUDA 11.7.1 is the lowest version of ubuntu22.04 jammy +ARG CUDA_RELEASE=11.7.1 + +FROM nvidia/cuda:${CUDA_RELEASE}-base-ubuntu22.04 +LABEL description="Official Folding@home GPU Container for Bastet" + +ARG ROCM_RELEASE=6.4.4 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + kmod \ + polkitd \ + python3 \ + python3-websocket \ + curl \ + # setup OpenCL ICD for nVidia + && mkdir -p /etc/OpenCL/vendors \ + && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd \ + # get ROCm OpenCL runtime packages + && mkdir -p --mode=0755 /etc/apt/keyrings \ + && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor > /etc/apt/keyrings/rocm.gpg \ + && printf "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_RELEASE}/ jammy main" \ + | tee /etc/apt/sources.list.d/rocm.list \ + && printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' \ + | tee /etc/apt/preferences.d/rocm-pin-600 \ + && apt-get update && apt-get install -y --no-install-recommends \ + rocm-opencl-runtime \ + rocm-smi-lib \ + rocminfo \ + # create libOpenCL.so instead of install ocl-icd-opencl-dev + && ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /usr/lib/x86_64-linux-gnu/libOpenCL.so \ + # add users and groups to properly install fah-client.deb + && useradd -ms /bin/bash fah-client \ + && groupadd -r render \ + && usermod -aG render,video fah-client \ + && mkdir /fah \ + && chown fah-client:fah-client /fah \ + # download and verify checksum + && curl -fsSL \ + https://download.foldingathome.org/releases/public/fah-client/debian-10-64bit/release/fah-client_8.4.9_amd64.deb \ + -o fah-client.deb \ + && echo "098f4abf7c172e543de4391e86eacd10655e67d53fd4b0ae05633e2b57f9e07c fah-client.deb" \ + | sha256sum -c --strict - \ + # install and cleanup + && DEBIAN_FRONTEND=noninteractive dpkg --install --force-depends fah-client.deb \ + && rm -rf fah-client.deb \ + && apt-get purge --autoremove -y curl \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + +ENV LD_LIBRARY_PATH=/opt/rocm/lib +ENV PATH=$PATH:/opt/rocm/bin + +USER fah-client +WORKDIR "/fah" +VOLUME ["/fah"] +EXPOSE 7396 + +# allow access outside container (allow option accepts just one entry) +ENTRYPOINT ["fah-client", "--allow", "0/0", "--http-addresses", "0.0.0.0:7396"] diff --git a/fah-gpu-bastet/Dockerfile.amd b/fah-gpu-bastet/Dockerfile.amd new file mode 100644 index 0000000..fb5266d --- /dev/null +++ b/fah-gpu-bastet/Dockerfile.amd @@ -0,0 +1,59 @@ +# Folding@home AMD ROCm Container + +# To reduce the image size, install the minimal packages for OpenCL +# instead of AMD's official ROCm image. +FROM ubuntu:22.04 +LABEL description="Folding@home AMD ROCm Container for Bastet" + +ARG ROCM_RELEASE=6.4.4 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + kmod \ + polkitd \ + python3 \ + python3-websocket \ + curl \ + gnupg \ + ca-certificates \ + # get ROCm OpenCL runtime packages + && mkdir -p --mode=0755 /etc/apt/keyrings \ + && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor > /etc/apt/keyrings/rocm.gpg \ + && printf "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_RELEASE}/ jammy main" \ + | tee /etc/apt/sources.list.d/rocm.list \ + && printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' \ + | tee /etc/apt/preferences.d/rocm-pin-600 \ + && apt-get update && apt-get install -y --no-install-recommends \ + rocm-opencl-runtime \ + rocm-smi-lib \ + rocminfo \ + # create libOpenCL.so instead of install ocl-icd-opencl-dev + && ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /usr/lib/x86_64-linux-gnu/libOpenCL.so \ + # add users and groups to properly install fah-client.deb + && useradd -ms /bin/bash fah-client \ + && groupadd -r render \ + && usermod -aG render,video fah-client \ + && mkdir /fah \ + && chown fah-client:fah-client /fah \ + # download and verify checksum + && curl -fsSL \ + https://download.foldingathome.org/releases/public/fah-client/debian-10-64bit/release/fah-client_8.4.9_amd64.deb \ + -o fah-client.deb \ + && echo "098f4abf7c172e543de4391e86eacd10655e67d53fd4b0ae05633e2b57f9e07c fah-client.deb" \ + | sha256sum -c --strict - \ + # install and cleanup + && DEBIAN_FRONTEND=noninteractive dpkg --install --force-depends fah-client.deb \ + && rm -rf fah-client.deb \ + && apt-get purge --autoremove -y curl \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + +ENV LD_LIBRARY_PATH=/opt/rocm/lib +ENV PATH=$PATH:/opt/rocm/bin + +USER fah-client +WORKDIR "/fah" +VOLUME ["/fah"] +EXPOSE 7396 + +# allow access outside container (allow option accepts just one entry) +ENTRYPOINT ["fah-client", "--allow", "0/0", "--http-addresses", "0.0.0.0:7396"] diff --git a/fah-gpu-bastet/Dockerfile.nvidia b/fah-gpu-bastet/Dockerfile.nvidia new file mode 100644 index 0000000..6e63032 --- /dev/null +++ b/fah-gpu-bastet/Dockerfile.nvidia @@ -0,0 +1,45 @@ +# Folding@home nVidia CUDA Container + +# CUDA 11.7.1 is the lowest version of ubuntu22.04 jammy +ARG CUDA_RELEASE=11.7.1 + +FROM nvidia/cuda:${CUDA_RELEASE}-base-ubuntu22.04 +LABEL description="Folding@home nVidia CUDA Container for Bastet" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + polkitd \ + python3 \ + python3-websocket \ + curl \ + clinfo \ + # setup OpenCL ICD for nVidia + && mkdir -p /etc/OpenCL/vendors \ + && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd \ + # create libOpenCL.so instead of install ocl-icd-opencl-dev + && ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /usr/lib/x86_64-linux-gnu/libOpenCL.so \ + # add users and groups to properly install fah-client.deb + && useradd -ms /bin/bash fah-client \ + && groupadd -r render \ + && usermod -aG render,video fah-client \ + && mkdir /fah \ + && chown fah-client:fah-client /fah \ + # download and verify checksum + && curl -fsSL \ + https://download.foldingathome.org/releases/public/fah-client/debian-10-64bit/release/fah-client_8.4.9_amd64.deb \ + -o fah-client.deb \ + && echo "098f4abf7c172e543de4391e86eacd10655e67d53fd4b0ae05633e2b57f9e07c fah-client.deb" \ + | sha256sum -c --strict - \ + # install and cleanup + && DEBIAN_FRONTEND=noninteractive dpkg --install --force-depends fah-client.deb \ + && rm -rf fah-client.deb \ + && apt-get purge --autoremove -y curl \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + +USER fah-client +WORKDIR "/fah" +VOLUME ["/fah"] +EXPOSE 7396 + +# allow access outside container (allow option accepts just one entry) +ENTRYPOINT ["fah-client", "--allow", "0/0", "--http-addresses", "0.0.0.0:7396"] diff --git a/fah-gpu-bastet/README.md b/fah-gpu-bastet/README.md new file mode 100644 index 0000000..eba8452 --- /dev/null +++ b/fah-gpu-bastet/README.md @@ -0,0 +1,117 @@ +# Folding@home GPU Container for Bastet (FaH client v8.x) + +## Goals + +- Latest Docker image for the new FAH client, Bastet (v8.x). +- Support for nVidia, AMD, or both. +- Provides a sandbox testing environment for the new FAH client and core. + + +## Notes + +- GPU drivers must be installed on the host machine since they are part of the OS kernel. + - nVidia: A package with a name similar to `nvidia-driver-???` + - AMD: In most cases, these are already included in the base kernel. +- If using an AMD GPU, pay attention to the `render` group ID associated with `/dev/dri/renderD*` permissions. +- Unlike FAH client v7, fah-client Bastet does not read `config.xml` after creating `client.db`. +- You should control fah-client via the [v8-4.foldingathome.org](https://v8-4.foldingathome.org/) web control or `fahctl` command. + - Fah-client does not start folding by default. + - You must enable GPUs and configure the number of CPU cores to use for folding. + - For more details, see [V8.4 Client Guide](https://foldingathome.org/guides/v8-4-client-guide/#v8-software-interface). +- The V8 client recognizes *users* and user *machines* separately. + - User data: username, team, passkey + - User machine data: `/etc/machine-id`, machine name, account token + - If `/etc/machine-id` is changed, previous folding works will be **invalid**. +- `Passkey` and `token` are different. + - Passkey: evidence of who the **user** is + - Token: **machine** password used to connect to foldingathome.org + + +## How to run on your desktop + +### Using `docker run` + +For the nVidia host machine +```bash +# Run container with GPUs, name it "fah0", map user and /fah volume +docker run --gpus all --name fah0 -d -p 7396:7396 --user "$(id -u):$(id -g)" \ + -v $HOME/fah:/fah -v /etc/machine-id:/etc/machine-id:ro \ + foldingathome/fah-gpu-bastet:cuda +``` + +For the AMD GPU host machine +```bash +docker run --device=/dev/kfd --device=/dev/dri \ + --security-opt seccomp=unconfined \ + --group-add video --group-add "$(getent group render | cut -d: -f3)" \ + --name fah0 -d -p 7396:7396 --user "$(id -u):$(id -g)" \ + -v $HOME/fah:/fah -v /etc/machine-id:/etc/machine-id:ro \ + foldingathome/fah-gpu-bastet:rocm +``` + +For both +```bash +docker run --gpus all --device=/dev/kfd --device=/dev/dri \ + --security-opt seccomp=unconfined \ + --group-add video --group-add "$(getent group render | cut -d: -f3)" \ + --name fah0 -d -p 7396:7396 --user "$(id -u):$(id -g)" \ + -v $HOME/fah:/fah -v /etc/machine-id:/etc/machine-id:ro \ + foldingathome/fah-gpu-bastet:cuda-rocm +``` + +Once the container starts, you can access it with [v8-4.foldingathome.org](https://v8-4.foldingathome.org/) + + +### Using `docker compose` + +Before you begin, check the `compose-{platform}.yml` file and uncomment any option or change any options. + +For the nVidia host machine +```bash +docker compose -f compose-cuda.yml up -d +``` + +For the AMD GPU host machine +```bash +export RENDER_GID="$(getent group render | cut -d: -f3)" +docker compose -f compose-rocm.yml up -d +``` + +For both +```bash +export RENDER_GID="$(getent group render | cut -d: -f3)" +docker compose up -d +``` + +**Note:** You can set the `RENDER_GID` variable in your `.env` file. +```sh +RENDER_GID=999 # replace 999 with your machine's render group ID +``` + + +## How to build a custom image + +You may set the CUDA and ROCm versions by creating the `.env` file, as `docker compose` consults it. Please check the `.env-example` file as an example. + +For the nVidia host machine +```bash +docker compose -f compose-cuda.yml build --pull +``` + +For the AMD GPU host machine +```bash +docker compose -f compose-rocm.yml build --pull +``` + +For both +```bash +docker compose build --pull +``` + + +## Maintenance Tips + +You can reduce the size of the `client.db` file **when fah-client is not running** by running this command: +```bash +sqlite3 client.db "VACUUM;" +``` diff --git a/fah-gpu-bastet/compose-cuda.yml b/fah-gpu-bastet/compose-cuda.yml new file mode 100644 index 0000000..88108d0 --- /dev/null +++ b/fah-gpu-bastet/compose-cuda.yml @@ -0,0 +1,21 @@ +services: + client: + image: foldingathome/fah-gpu-bastet:cuda + build: + context: . + dockerfile: Dockerfile.nvidia + args: + CUDA_RELEASE: "${CUDA_RELEASE:-11.7.1}" + tags: + - "foldingathome/fah-gpu-bastet:cuda${CUDA_RELEASE:-11.7.1}" + - "foldingathome/fah-gpu-bastet:${VERSION:-25.11.0}-cuda" + - "foldingathome/fah-gpu-bastet:${VERSION:-25.11.0}-cuda${CUDA_RELEASE:-11.7.1}" + ports: + - "7396:7396" + # Options for nVidia CUDA + gpus: all + # Options for setting local directory + # user: "1000:1000" + # volumes: + # - "/etc/machine-id:/etc/machine-id:ro" + # - "${HOME}/fah:/fah" diff --git a/fah-gpu-bastet/compose-rocm.yml b/fah-gpu-bastet/compose-rocm.yml new file mode 100644 index 0000000..b4ff508 --- /dev/null +++ b/fah-gpu-bastet/compose-rocm.yml @@ -0,0 +1,28 @@ +services: + client: + image: foldingathome/fah-gpu-bastet:rocm + build: + context: . + dockerfile: Dockerfile.amd + args: + ROCM_RELEASE: "${ROCM_RELEASE:-6.4.4}" + tags: + - "foldingathome/fah-gpu-bastet:rocm${ROCM_RELEASE:-6.4.4}" + - "foldingathome/fah-gpu-bastet:${VERSION:-25.11.0}-rocm" + - "foldingathome/fah-gpu-bastet:${VERSION:-25.11.0}-rocm${ROCM_RELEASE:-6.4.4}" + ports: + - "7396:7396" + # Options for AMD ROCm + devices: + - "/dev/kfd:/dev/kfd" + - "/dev/dri:/dev/dri" + security_opt: + - seccomp=unconfined + group_add: + - "video" + - "${RENDER_GID:-render}" + # Options for setting local directory + # user: "1000:1000" + # volumes: + # - "/etc/machine-id:/etc/machine-id:ro" + # - "${HOME}/fah:/fah" diff --git a/fah-gpu-bastet/compose.yml b/fah-gpu-bastet/compose.yml new file mode 100644 index 0000000..98def30 --- /dev/null +++ b/fah-gpu-bastet/compose.yml @@ -0,0 +1,30 @@ +services: + client: + image: foldingathome/fah-gpu-bastet:cuda-rocm + build: + context: . + args: + CUDA_RELEASE: "${CUDA_RELEASE:-11.7.1}" + ROCM_RELEASE: "${ROCM_RELEASE:-6.4.4}" + tags: + - "foldingathome/fah-gpu-bastet:cuda${CUDA_RELEASE:-11.7.1}-rocm${ROCM_RELEASE:-6.4.4}" + - "foldingathome/fah-gpu-bastet:${VERSION:-25.11.0}-cuda-rocm" + - "foldingathome/fah-gpu-bastet:${VERSION:-25.11.0}-cuda${CUDA_RELEASE:-11.7.1}-rocm${ROCM_RELEASE:-6.4.4}" + ports: + - "7396:7396" + # Options for nVidia CUDA + gpus: all + # Options for AMD ROCm + devices: + - "/dev/kfd:/dev/kfd" + - "/dev/dri:/dev/dri" + security_opt: + - seccomp=unconfined + group_add: + - "video" + - "${RENDER_GID:-render}" + # Options for setting local directory + # user: "1000:1000" + # volumes: + # - "/etc/machine-id:/etc/machine-id:ro" + # - "${HOME}/fah:/fah"