Use wasi-sdk 20 pre-release for tests with threads (#2021)

`wasi-sdk-20` pre-release can be used to avoid building `wasi-libc` to enable threads. It's not possible to use `wasi-sdk-20` pre-release on Ubuntu 20.04 because of incompatibility with the glibc version: ```bash /opt/wasi-sdk/bin/clang: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by /opt/wasi-sdk/bin/clang) ```
2023-03-17 13:02:03 +01:00 · 2023-03-17 13:02:03 +01:00 · 83659fa42a
commit 83659fa42a
parent 23e9a356e5
7 changed files with 50 additions and 58 deletions
--- a/.github/workflows/compilation_on_android_ubuntu.yml
+++ b/.github/workflows/compilation_on_android_ubuntu.yml
@ -337,9 +337,13 @@ jobs:
          - os: ubuntu-20.04
            wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz"
            wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
+            wasi_sdk_folder_name: "wasi-sdk-19.0"
+            wasi_sysroot_option: "-DWASI_SYSROOT=`pwd`/../../../core/deps/wasi-libc/sysroot"
          - os: ubuntu-22.04
-            wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz"
+            wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20%2Bthreads/wasi-sdk-20.0.threads-linux.tar.gz"
            wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
+            wasi_sdk_folder_name: "wasi-sdk-20.0+threads"
+            wasi_sysroot_option: ""
    steps:
      - name: checkout
        uses: actions/checkout@v3
@ -349,7 +353,7 @@ jobs:
          cd /opt
          sudo wget ${{ matrix.wasi_sdk_release }}
          sudo tar -xzf wasi-sdk-*.tar.gz
-          sudo mv wasi-sdk-19.0 wasi-sdk
+          sudo mv ${{ matrix.wasi_sdk_folder_name }} wasi-sdk

      - name: download and install wabt
        run: |
@ -359,6 +363,7 @@ jobs:
          sudo mv wabt-1.0.31 wabt

      - name: build wasi-libc (needed for wasi-threads)
+        if: matrix.os == 'ubuntu-20.04'
        run: |
          mkdir wasi-libc
          cd wasi-libc
@ -431,15 +436,22 @@ jobs:
        run: |
          cd samples/wasi-threads
          mkdir build && cd build
-          cmake -DWASI_SYSROOT=`pwd`/../../../core/deps/wasi-libc/sysroot ..
+          cmake ${{ matrix.wasi_sysroot_option }} ..
          cmake --build . --config Release --parallel 4
          ./iwasm wasm-apps/no_pthread.wasm

  test:
-    needs: [build_iwasm, build_llvm_libraries_on_ubuntu_2004, build_wamrc]
-    runs-on: ubuntu-20.04
+    needs:
+      [
+        build_iwasm,
+        build_llvm_libraries_on_ubuntu_2004,
+        build_llvm_libraries_on_ubuntu_2204,
+        build_wamrc
+      ]
+    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
+        os: [ubuntu-20.04, ubuntu-22.04]
        running_mode:
          [
            "classic-interp",
@ -457,12 +469,21 @@ jobs:
            $THREADS_TEST_OPTIONS,
            $WASI_TEST_OPTIONS,
          ]
-        wasi_sdk_release:
-          [
-            "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz",
-          ]
-        llvm_cache_key:
-          ["${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}"]
+        include:
+          - os: ubuntu-20.04
+            llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}
+            wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz"
+            wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
+            wasi_sdk_folder_name: "wasi-sdk-19.0"
+            wasi_sysroot_option: "WASI_SYSROOT_OPTION='--sysroot ../../../../../core/deps/wasi-libc/sysroot'"
+            ubuntu_version: "20.04"
+          - os: ubuntu-22.04
+            llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }}
+            wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20%2Bthreads/wasi-sdk-20.0.threads-linux.tar.gz"
+            wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
+            wasi_sdk_folder_name: "wasi-sdk-20.0+threads"
+            wasi_sysroot_option: ""
+            ubuntu_version: "22.04"
        exclude:
          # uncompatiable modes and features
          # classic-interp and fast-interp don't support simd
@ -503,10 +524,10 @@ jobs:
          cd /opt
          sudo wget ${{ matrix.wasi_sdk_release }}
          sudo tar -xzf wasi-sdk-*.tar.gz
-          sudo mv wasi-sdk-19.0 wasi-sdk
+          sudo mv ${{ matrix.wasi_sdk_folder_name }} wasi-sdk

      - name: build wasi-libc (needed for wasi-threads)
-        if: matrix.test_option == '$WASI_TEST_OPTIONS'
+        if: matrix.os == 'ubuntu-20.04' && matrix.test_option == '$WASI_TEST_OPTIONS'
        run: |
          mkdir wasi-libc
          cd wasi-libc
@ -557,12 +578,12 @@ jobs:

      - name: Build WASI thread tests
        if: matrix.test_option == '$WASI_TEST_OPTIONS'
-        run: WASI_SYSROOT=../../../../../core/deps/wasi-libc/sysroot bash build.sh
+        run: ${{ matrix.wasi_sysroot_option }} bash build.sh
        working-directory: ./core/iwasm/libraries/lib-wasi-threads/test/

      - name: build socket api tests
        if: matrix.test_option == '$WASI_TEST_OPTIONS'
-        run: WASI_SYSROOT=../../../../../core/deps/wasi-libc/sysroot bash build.sh
+        run: ${{ matrix.wasi_sysroot_option }} bash build.sh
        working-directory: ./core/iwasm/libraries/lib-socket/test/

      - name: run tests
@ -577,7 +598,7 @@ jobs:
          # Add another apt repository as some packages cannot
          # be downloaded with the github default repository
          sudo curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc &&
-          sudo apt-add-repository https://packages.microsoft.com/ubuntu/20.04/prod &&
+          sudo apt-add-repository https://packages.microsoft.com/ubuntu/${{ matrix.ubuntu_version }}/prod &&
          sudo apt-get update &&
          sudo apt install -y g++-multilib lib32gcc-9-dev

--- a/.github/workflows/compilation_on_macos.yml
+++ b/.github/workflows/compilation_on_macos.yml
@ -249,7 +249,7 @@ jobs:
        os: [macos-latest]
        wasi_sdk_release:
          [
-            "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-macos.tar.gz",
+            "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20%2Bthreads/wasi-sdk-20.0.threads-macos.tar.gz",
          ]
        wabt_release:
          [
@ -264,7 +264,7 @@ jobs:
          cd /opt
          sudo wget ${{ matrix.wasi_sdk_release }}
          sudo tar -xzf wasi-sdk-*.tar.gz
-          sudo mv wasi-sdk-19.0 wasi-sdk
+          sudo mv wasi-sdk-20.0+threads wasi-sdk

      - name: download and install wabt
        run: |
@ -273,22 +273,6 @@ jobs:
          sudo tar -xzf wabt-1.0.31-*.tar.gz
          sudo mv wabt-1.0.31 wabt

-      - name: build wasi-libc (needed for wasi-threads)
-        run: |
-          mkdir wasi-libc
-          cd wasi-libc
-          git init
-          # "Rename thread_spawn import" commit on main branch
-          git fetch https://github.com/WebAssembly/wasi-libc \
-            8f5275796a82f8ecfd0833a4f3f444fa37ed4546
-          git checkout FETCH_HEAD
-          make \
-            AR=/opt/wasi-sdk/bin/llvm-ar \
-            NM=/opt/wasi-sdk/bin/llvm-nm \
-            CC=/opt/wasi-sdk/bin/clang \
-            THREAD_MODEL=posix
-        working-directory: core/deps
-
      - name: Build Sample [basic]
        run: |
          cd samples/basic
@ -339,6 +323,6 @@ jobs:
        run: |
          cd samples/wasi-threads
          mkdir build && cd build
-          cmake -DWASI_SYSROOT=`pwd`/../../../core/deps/wasi-libc/sysroot ..
+          cmake ..
          cmake --build . --config Release --parallel 4
          ./iwasm wasm-apps/no_pthread.wasm
--- a/core/iwasm/libraries/lib-socket/test/build.sh
+++ b/core/iwasm/libraries/lib-socket/test/build.sh
@ -3,18 +3,17 @@
 # Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

-set -ueo pipefail
+set -eo pipefail
 CC="${CC:=/opt/wasi-sdk/bin/clang}"
 files=("tcp_udp.c" "nslookup.c")
-WASI_SYSROOT=${WASI_SYSROOT:=~/dev/wasi-libc/sysroot}

 for file in "${files[@]}"
 do
    echo $file
    $CC \
+        $WASI_SYSROOT_OPTION \
        --target=wasm32-wasi-threads \
        -I../inc \
-        --sysroot $WASI_SYSROOT \
        ../src/wasi/wasi_socket_ext.c -pthread -ftls-model=local-exec \
        -Wl,--allow-undefined \
        -Wl,--strip-all,--no-entry \
--- a/core/iwasm/libraries/lib-wasi-threads/test/build.sh
+++ b/core/iwasm/libraries/lib-wasi-threads/test/build.sh
@ -5,8 +5,8 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #

+set -eo pipefail
 CC=${CC:=/opt/wasi-sdk/bin/clang}
-WASI_SYSROOT=${WASI_SYSROOT:=~/dev/wasi-libc/sysroot}
 WAMR_DIR=../../../../..

 for test_c in *.c; do
@ -14,7 +14,7 @@ for test_c in *.c; do

    echo "Compiling $test_c to $test_wasm"
    $CC \
-        --sysroot $WASI_SYSROOT \
+        $WASI_SYSROOT_OPTION \
        -target wasm32-wasi-threads \
        -pthread -ftls-model=local-exec \
        -z stack-size=32768 \
--- a/samples/wasi-threads/CMakeLists.txt
+++ b/samples/wasi-threads/CMakeLists.txt
@ -7,13 +7,6 @@ include(CheckPIESupported)

 project(wasi_threads_sample)

-if (NOT DEFINED WASI_SYSROOT)
-  message (WARNING "Custom sysroot with threads enabled is required to build wasi threads samples.
-Please note that current wasi-sdk doesn't ship with threads enabled.
-Run cmake command with -DWASI_SYSROOT=/path/to/sysroot/with/threads to compile samples.")
-  return ()
-endif ()
-
 ################  runtime settings  ################
 string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM)
 if (APPLE)
--- a/samples/wasi-threads/README.md
+++ b/samples/wasi-threads/README.md
@ -1,21 +1,13 @@
 # "WASI threads" sample introduction

-Currently, since the `wasi-sdk` does not have thread support in the latest release, make sure to have [wasi-libc](https://github.com/WebAssembly/wasi-libc) installed. Build it with threads enabled, e.g.
-
-```shell
-make \
-    AR=/opt/wasi-sdk/bin/llvm-ar \
-    NM=/opt/wasi-sdk/bin/llvm-nm \
-    CC=/opt/wasi-sdk/bin/clang \
-    THREAD_MODEL=posix
-```
+To run the sample, `wasi-sdk` >= 20 is required.

 ## Build and run the samples

 ```shell
 $ mkdir build
 $ cd build
-$ cmake -DWASI_SYSROOT=/path/to/wasi-libc/sysroot ..
+$ cmake ..
 $ make
 ...
 $ ./iwasm wasm-apps/no_pthread.wasm
--- a/samples/wasi-threads/wasm-apps/CMakeLists.txt
+++ b/samples/wasi-threads/wasm-apps/CMakeLists.txt
@ -11,7 +11,10 @@ if (NOT DEFINED WASI_SDK_DIR)
  set (WASI_SDK_DIR               "/opt/wasi-sdk")
 endif ()

-set (CMAKE_SYSROOT                  "${WASI_SYSROOT}")
+if (DEFINED WASI_SYSROOT)
+  set (CMAKE_SYSROOT                "${WASI_SYSROOT}")
+endif ()
+
 set (CMAKE_C_COMPILER               "${WASI_SDK_DIR}/bin/clang")
 set (CMAKE_ASM_COMPILER               "${WASI_SDK_DIR}/bin/clang")
 set (CMAKE_EXE_LINKER_FLAGS         "-target wasm32-wasi-threads")