From 99aec47c76dbb63f7f74a7c524581ce0403ed236 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Thu, 14 Mar 2024 17:27:34 +0800 Subject: [PATCH] chore: drop root in container --- .github/workflows/container.yml | 3 +-- Runner.Dockerfile | 5 +++- Server.Dockerfile | 2 +- docker-compose.yml | 6 ++--- docker-entrypoint.sh | 42 ++++++++++++++++++++++++++++--- internal/service/runner/nsjail.go | 4 +-- 6 files changed, 49 insertions(+), 13 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 0e26a08..6abf817 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -13,7 +13,6 @@ jobs: sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force sudo df -h - uses: actions/checkout@v4 # reference: https://github.com/containers/podman/discussions/17868 @@ -28,7 +27,7 @@ jobs: path: | ~/.local/share/containers ~/.config/containers - key: ${{ runner.os }}-${{ hashFiles('**/*.Dockerfile', 'build_image.sh', 'VERSION') }} + key: ${{ runner.os }}-${{ hashFiles('**/*.Dockerfile', 'build_image.sh', 'docker-entrypoint.sh', 'VERSION') }} - name: Login to Container Registry uses: redhat-actions/podman-login@v1 with: diff --git a/Runner.Dockerfile b/Runner.Dockerfile index fa3dfc3..b5a5b12 100644 --- a/Runner.Dockerfile +++ b/Runner.Dockerfile @@ -15,7 +15,7 @@ FROM docker.io/library/debian:bookworm-slim WORKDIR /app RUN apt-get update && apt-get upgrade -y \ - && apt-get install -y ca-certificates libnl-route-3-200 libprotobuf32 tini \ + && apt-get install -y ca-certificates gosu libnl-route-3-200 libprotobuf32 tini \ && apt-get clean && rm -rf /var/lib/apt/lists # rootfs @@ -31,5 +31,8 @@ COPY --from=base /builder/config.docker.yaml /app COPY --from=base /builder/docker-entrypoint.sh /app COPY --from=base /builder/woj /app +# tell entrypoint to setup runner env +ENV RUNNER_IMAGE=1 + # reap zombies ENTRYPOINT ["/usr/bin/tini", "/app/docker-entrypoint.sh"] diff --git a/Server.Dockerfile b/Server.Dockerfile index 18a8a48..9252f37 100644 --- a/Server.Dockerfile +++ b/Server.Dockerfile @@ -12,7 +12,7 @@ RUN find /app -type f -name "*.map" -delete FROM docker.io/library/alpine WORKDIR /app -RUN apk --no-cache add tzdata ca-certificates bash +RUN apk --no-cache add tzdata ca-certificates bash su-exec COPY --from=base /builder/config.docker.yaml /app COPY --from=base /builder/docker-entrypoint.sh /app diff --git a/docker-compose.yml b/docker-compose.yml index 2d6ca3f..a24aa2a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: server: - image: git.0x7f.app/woj/woj-server:1.3.1-dev + image: quay.io/ldcraft/woj-server:1.3.1-dev restart: unless-stopped healthcheck: test: [ "CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8000/health" ] @@ -27,9 +27,10 @@ services: - "8000:8000" runner: - image: git.0x7f.app/woj/woj-runner:1.3.1-dev + image: quay.io/ldcraft/woj-runner:1.3.1-dev restart: unless-stopped command: runner + # moby/moby#42040, enable privileged option to make cgroup2 mount as rw privileged: true cap_add: - SYS_ADMIN @@ -40,7 +41,6 @@ services: - STORAGE_SECRET_KEY=secret_key - STORAGE_BUCKET=woj - DEVELOPMENT=true - - SETUP_CGROUP=true volumes: - runner:/app/resource/runner/user - /etc/localtime:/etc/localtime:ro diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index dea0112..c4c73fa 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -136,11 +136,45 @@ setup_cgroups() { sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/nsjail/cgroup.controllers > /sys/fs/cgroup/nsjail/cgroup.subtree_control } -if [ -n "$SETUP_CGROUP" ]; then - setup_cgroups -fi +setup_user_runner() { + # runner use debian as base image + groupadd --gid 1000 woj || true + useradd --gid 1000 --uid 1000 woj || true + # runner data + chown -R woj:woj /app/resource/runner/problem + chown -R woj:woj /app/resource/runner/tmp + chown -R woj:woj /app/resource/runner/user + # rootfs mount path + chown -R woj:woj /app/resource/runner/framework/rootfs/full/woj + chown -R woj:woj /app/resource/runner/framework/rootfs/run/woj + # cgroups + chown -R woj:woj /sys/fs/cgroup/nsjail + chown -R woj:root /sys/fs/cgroup/cgroup.procs + # create run dir: nsjail will try to use it + mkdir -p /run/user/1000 + chown -R woj:woj /run/user/1000 + chmod 700 /run/user/1000 +} +setup_user_server() { + # server use alpine as base image + addgroup -g 1000 -S woj || true + adduser -u 1000 -S -G woj woj || true +} + +setup_user() { + if [ -n "$RUNNER_IMAGE" ]; then + setup_user_runner + else + setup_user_server + fi +} + +if [ -n "$RUNNER_IMAGE" ]; then setup_cgroups; fi generate_config +setup_user log_info "starting woj" -exec /app/woj "$@" +if [ -n "$RUNNER_IMAGE" ]; then EXEC=gosu; else EXEC=su-exec; fi +exec $EXEC woj /app/woj "$@" + diff --git a/internal/service/runner/nsjail.go b/internal/service/runner/nsjail.go index cb56d3d..57a8508 100644 --- a/internal/service/runner/nsjail.go +++ b/internal/service/runner/nsjail.go @@ -113,9 +113,9 @@ func (s *service) JailRun(arg *RunArgs) (RuntimeStatus, error) { // build args args := []string{ - "--quiet", + "--really_quiet", "--use_cgroupv2", - "--disable_rlimits", // Rust requires this + "--disable_rlimits", "-m", "none:/tmp:tmpfs:size=67108864", // 64MB tmpfs "-E", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", // following envs must sync with resource/runner