diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5ddba8f3..7ccfa246 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -12,7 +12,7 @@ ENV TZ=Asian/Shanghai # hadolint ignore=DL3008 RUN apt-get update \ && apt-get install -y apt-transport-https apt-utils build-essential \ - ca-certificates curl g++-multilib git gnupg \ + ca-certificates ccache curl g++-multilib git gnupg \ libgcc-9-dev lib32gcc-9-dev lsb-release \ ninja-build ocaml ocamlbuild python2.7 \ software-properties-common tree tzdata \ @@ -20,6 +20,15 @@ RUN apt-get update \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* +# +# binaryen +ARG BINARYEN_VER=111 +WORKDIR /opt +RUN wget -c --progress=dot:giga https://github.com/WebAssembly/binaryen/releases/download/version_${BINARYEN_VER}/binaryen-version_${BINARYEN_VER}-x86_64-linux.tar.gz \ + && tar xf binaryen-version_${BINARYEN_VER}-x86_64-linux.tar.gz \ + && ln -sf /opt/binaryen-version_111 /opt/binaryen \ + && rm binaryen-version_${BINARYEN_VER}-x86_64-linux.tar.gz + # # CMAKE (https://apt.kitware.com/) SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -31,25 +40,26 @@ RUN wget --progress=dot:giga -O - https://apt.kitware.com/keys/kitware-archive-l && apt-get install -y kitware-archive-keyring --no-install-recommends \ && apt-get install -y cmake --no-install-recommends \ && apt-get clean -y \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* # # install emsdk WORKDIR /opt RUN git clone https://github.com/emscripten-core/emsdk.git +ARG EMSDK_VER=3.0.0 WORKDIR /opt/emsdk RUN git pull \ - && ./emsdk install 2.0.26 \ - && ./emsdk activate 2.0.26 \ + && ./emsdk install ${EMSDK_VER} \ + && ./emsdk activate ${EMSDK_VER} \ && echo "source /opt/emsdk/emsdk_env.sh" >> /root/.bashrc # # install wasi-sdk -ARG WASI_SDK_VER=16 +ARG WASI_SDK_VER=19 RUN wget -c --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VER}/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz -P /opt \ && tar xf /opt/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz -C /opt \ - && ln -fs /opt/wasi-sdk-${WASI_SDK_VER}.0 /opt/wasi-sdk \ + && ln -sf /opt/wasi-sdk-${WASI_SDK_VER}.0 /opt/wasi-sdk \ && rm /opt/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz # @@ -57,29 +67,29 @@ RUN wget -c --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases ARG WABT_VER=1.0.29 RUN wget -c --progress=dot:giga https://github.com/WebAssembly/wabt/releases/download/${WABT_VER}/wabt-${WABT_VER}-ubuntu.tar.gz -P /opt \ && tar xf /opt/wabt-${WABT_VER}-ubuntu.tar.gz -C /opt \ - && ln -fs /opt/wabt-${WABT_VER} /opt/wabt \ + && ln -sf /opt/wabt-${WABT_VER} /opt/wabt \ && rm /opt/wabt-${WABT_VER}-ubuntu.tar.gz # # install bazelisk ARG BAZELISK_VER=1.12.0 -RUN mkdir /opt/bazelisk \ +RUN mkdir /opt/bazelisk \ && wget -c --progress=dot:giga https://github.com/bazelbuild/bazelisk/releases/download/v${BAZELISK_VER}/bazelisk-linux-amd64 -P /opt/bazelisk \ && chmod a+x /opt/bazelisk/bazelisk-linux-amd64 \ && ln -fs /opt/bazelisk/bazelisk-linux-amd64 /opt/bazelisk/bazel # # install clang+llvm +ARG LLVM_VER=14 +RUN apt-get purge -y clang-10 llvm-10 && apt autoremove -y WORKDIR /etc/apt/apt.conf.d RUN touch 99verfiy-peer.conf \ && echo "Acquire { https::Verify-Peer false }" > 99verfiy-peer.conf -WORKDIR /tmp +WORKDIR /tmp RUN wget --progress=dot:giga https://apt.llvm.org/llvm.sh \ && chmod a+x ./llvm.sh \ - && /tmp/llvm.sh 12 all \ - && ln -sf /usr/bin/clang-format-12 /usr/bin/clang-format \ - && rm -rf /tmp/* + && ./llvm.sh ${LLVM_VER} all # # [Optional] @@ -96,17 +106,28 @@ RUN apt-get update \ # Install required python packages # hadolint ignore=DL3013 RUN python3 -m pip install --no-cache-dir --upgrade pip \ - && pip3 install --no-cache-dir --user black nose pycparser pylint + && pip3 install --no-cache-dir black nose pycparser pylint -# set path, PS and clean up -ENV PATH "/opt/bazelisk:/opt/clang-llvm/bin:${PATH}" -RUN echo "export PATH=/opt/bazelisk:/opt/clang-llvm/bin:${PATH}" >> /root/.bashrc \ - && printf "%s\n" "PS1='\n[ \u@wamr-dev-docker \W ]\n$ '" >> /root/.bashrc \ +# +# Install github-cli. It doens't work as a feature of devcontainer.json +RUN cd /tmp \ + && wget https://github.com/cli/cli/releases/download/v2.20.2/gh_2.20.2_linux_amd64.deb \ + && dpkg -i gh_2.20.2_linux_amd64.deb + +# +# Install NodeJS +RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - +RUN apt-get install -y nodejs + +# set path +ENV PATH="/opt/bazelisk:/usr/lib/llvm-${LLVM_VER}/bin:${PATH}" +ENV CC=/usr/lib/llvm-${LLVM_VER}/bin/clang CXX=/usr/lib/llvm-${LLVM_VER}/bin/clang++ +RUN printf "%s\n" "PS1='\n[ \u@wamr-dev-docker \W ]\n$ '" >> /root/.bashrc \ && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /tmp/* # set workdir when container run -VOLUME /workspace -WORKDIR /workspace \ No newline at end of file +VOLUME /workspaces +WORKDIR /workspaces diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 976f5beb..24e1bdfd 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,5 @@ // Copyright (C) 2019 Intel Corporation. All rights reserved. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - // For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: // https://github.com/microsoft/vscode-dev-containers/tree/v0.195.0/containers/cpp { @@ -10,7 +9,12 @@ // Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 // Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon "args": { - "VARIANT": "ubuntu-20.04" + "BINARYEN_VER": "111", + "EMSDK_VER": "3.0.0", + "LLVM_VER": "15", + "VARIANT": "ubuntu-20.04", + "WASI_SDK_VER": "19", + "WABT_VER": "1.0.31" } }, "runArgs": [ @@ -27,12 +31,10 @@ // Add the IDs of extensions you want installed when the container is created. "extensions": [ "dtsvet.vscode-wasm", - "esbenp.prettier-vscode", + "llvm-vs-code-extensions.vscode-clangd", "ms-python.python", "ms-python.vscode-pylance", "ms-vscode.cmake-tools", - "ms-vscode.cpptools", - "twxs.cmake" ] } }, diff --git a/.github/workflows/build_llvm_libraries.yml b/.github/workflows/build_llvm_libraries.yml index 6eaa2be7..b82b89e6 100644 --- a/.github/workflows/build_llvm_libraries.yml +++ b/.github/workflows/build_llvm_libraries.yml @@ -5,23 +5,42 @@ name: Reusable workflow-build_llvm_libraries on: workflow_call: inputs: - runs-on: + os: required: true type: string + arch: + required: true + type: string + outputs: + cache_key: + description: "A cached key of LLVM libraries" + value: ${{ jobs.build_llvm_libraries.outputs.key}} jobs: build_llvm_libraries: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ${{ fromJson(inputs.runs-on) }} + runs-on: ${{ inputs.os }} + outputs: + key: ${{ steps.create_lib_cache_key.outputs.key}} steps: - name: checkout uses: actions/checkout@v3 + - name: retrive the last commit ID + id: get_last_commit + run: echo "last_commit=$(/usr/bin/env python3 ./build_llvm.py --llvm-ver)" >> $GITHUB_OUTPUT + working-directory: build-scripts + + # Bump the prefix number to evict all previous caches and + # enforce a clean build, in the unlikely case that some + # weird build error occur and llvm/build becomes a potential + # suspect. + - name: form the cache key of libraries + id: create_lib_cache_key + run: echo "key=0-llvm-libraries-${{ inputs.os }}-${{ inputs.arch }}-${{ steps.get_last_commit.outputs.last_commit }}" >> $GITHUB_OUTPUT + - name: Cache LLVM libraries - id: cache_llvm + id: retrieve_llvm_libs uses: actions/cache@v3 with: path: | @@ -30,10 +49,39 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ${{ matrix.os }}-build-llvm_libraries_ex + key: ${{ steps.create_lib_cache_key.outputs.key}} - - name: Build llvm - id: build_llvm - if: ${{ steps.cache_llvm.outputs.cache-hit != 'true' }} - run: /usr/bin/env python3 ./build_llvm.py --arch X86 WebAssembly + - uses: actions/cache@v3 + with: + path: ~/.ccache + key: 0-ccache-${{ inputs.os }}-${{ steps.get_last_commit.outputs.last_commit }} + restore-keys: | + 0-ccache-${{ inputs.os }} + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && inputs.os == 'ubuntu-20.04' + + - uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: 0-ccache-${{ inputs.os }}-${{ steps.get_last_commit.outputs.last_commit }} + restore-keys: | + 0-ccache-${{ inputs.os }} + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && inputs.os == 'ubuntu-22.04' + + - run: sudo apt install -y ccache ninja-build + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'ubuntu') + + - uses: actions/cache@v3 + with: + path: ~/Library/Caches/ccache + key: 0-ccache-${{ inputs.os }}-${{ steps.get_last_commit.outputs.last_commit }} + restore-keys: | + 0-ccache-${{ inputs.os }} + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'macos') + + - run: brew install ccache ninja + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'macos') + + - name: Build LLVM libraries + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: /usr/bin/env python3 ./build_llvm.py --arch ${{ inputs.arch }} working-directory: build-scripts diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 6980e273..93528b14 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -53,8 +53,6 @@ env: FAST_JIT_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" LLVM_LAZY_JIT_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" LLVM_EAGER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" - # LLVM - LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex" # For Spec Test DEFAULT_TEST_OPTIONS: "-s spec -b -P" MULTI_MODULES_TEST_OPTIONS: "-s spec -b -M -P" @@ -64,23 +62,37 @@ env: WASI_TEST_OPTIONS: "-s wasi_certification" jobs: - build_llvm_libraries: + build_llvm_libraries_on_ubuntu_2004: uses: ./.github/workflows/build_llvm_libraries.yml with: - runs-on: "['ubuntu-20.04', 'ubuntu-22.04']" + os: "ubuntu-20.04" + arch: "X86" + + build_llvm_libraries_on_ubuntu_2204: + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "ubuntu-22.04" + arch: "X86" build_wamrc: - needs: [build_llvm_libraries] + needs: + [build_llvm_libraries_on_ubuntu_2004, build_llvm_libraries_on_ubuntu_2204] runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, ubuntu-22.04] + include: + - os: ubuntu-20.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} steps: - name: checkout uses: actions/checkout@v3 + # since jobs.id can't contain the dot character + # it is hard to use `format` to assemble the cache key - name: Get LLVM libraries - id: cache_llvm + id: retrieve_llvm_libs uses: actions/cache@v3 with: path: | @@ -89,10 +101,10 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} + key: ${{ matrix.llvm_cache_key }} - name: Quit if cache miss - if: steps.cache_llvm.outputs.cache-hit != 'true' + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: Build wamrc @@ -103,7 +115,8 @@ jobs: working-directory: wamr-compiler build_iwasm: - needs: [build_llvm_libraries] + needs: + [build_llvm_libraries_on_ubuntu_2004, build_llvm_libraries_on_ubuntu_2204] runs-on: ${{ matrix.os }} strategy: matrix: @@ -187,13 +200,21 @@ jobs: # Fast-JIT mode doesn't support android(X86-32) - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS platform: android + # only test andorid on ubuntu latest + - os: ubuntu-20.04 + platform: android + include: + - os: ubuntu-20.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} steps: - name: checkout uses: actions/checkout@v3 # only download llvm cache when needed - name: Get LLVM libraries - id: cache_llvm + id: retrieve_llvm_libs if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') uses: actions/cache@v3 with: @@ -203,10 +224,10 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} + key: ${{ matrix.llvm_cache_key }} - name: Quit if cache miss - if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.cache_llvm.outputs.cache-hit != 'true') + if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: Build iwasm @@ -217,7 +238,13 @@ jobs: working-directory: product-mini/platforms/${{ matrix.platform }} build_samples_wasm_c_api: - needs: [build_iwasm, build_llvm_libraries, build_wamrc] + needs: + [ + build_iwasm, + build_llvm_libraries_on_ubuntu_2004, + build_llvm_libraries_on_ubuntu_2204, + build_wamrc, + ] runs-on: ${{ matrix.os }} strategy: matrix: @@ -233,18 +260,23 @@ jobs: os: [ubuntu-20.04, ubuntu-22.04] wasi_sdk_release: [ - "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz", + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz", ] wabt_release: [ - "https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz", + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz", ] + include: + - os: ubuntu-20.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} steps: - name: checkout uses: actions/checkout@v3 - name: Get LLVM libraries - id: cache_llvm + id: retrieve_llvm_libs if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) uses: actions/cache@v3 with: @@ -254,18 +286,18 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} + key: ${{ matrix.llvm_cache_key }} - name: Quit if cache miss - if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) && (steps.cache_llvm.outputs.cache-hit != 'true') + if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: download and install wabt run: | cd /opt sudo wget ${{ matrix.wabt_release }} - sudo tar -xzf wabt-1.0.24-*.tar.gz - sudo mv wabt-1.0.24 wabt + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo mv wabt-1.0.31 wabt - name: Build wamrc if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) @@ -277,19 +309,9 @@ jobs: - name: Build Sample [wasm-c-api] run: | - mkdir build && cd build - cmake .. ${{ matrix.make_options }} - cmake --build . --config Release --parallel 4 - ./callback - ./callback_chain - ./empty_imports - ./global - ./hello - ./hostref - ./memory - ./reflect - ./table - ./trap + cmake -S . -B build ${{ matrix.make_options }} + cmake --build build --config Release --parallel 4 + ctest --test-dir build working-directory: samples/wasm-c-api build_samples_others: @@ -298,14 +320,13 @@ jobs: strategy: matrix: os: [ubuntu-20.04, ubuntu-22.04] - wasi_sdk_release: - [ - "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz", - ] - wabt_release: - [ - "https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz", - ] + include: + - os: ubuntu-20.04 + wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz" + wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz" + - os: ubuntu-22.04 + wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz" + wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz" steps: - name: checkout uses: actions/checkout@v3 @@ -314,15 +335,15 @@ jobs: run: | cd /opt sudo wget ${{ matrix.wasi_sdk_release }} - sudo tar -xzf wasi-sdk-12.0-*.tar.gz - sudo mv wasi-sdk-12.0 wasi-sdk + sudo tar -xzf wasi-sdk-*.tar.gz + sudo mv wasi-sdk-19.0 wasi-sdk - name: download and install wabt run: | cd /opt sudo wget ${{ matrix.wabt_release }} - sudo tar -xzf wabt-1.0.24-*.tar.gz - sudo mv wabt-1.0.24 wabt + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo mv wabt-1.0.31 wabt - name: Build Sample [basic] run: | @@ -378,7 +399,7 @@ jobs: working-directory: ./samples/simple test: - needs: [build_iwasm, build_llvm_libraries, build_wamrc] + needs: [build_iwasm, build_llvm_libraries_on_ubuntu_2004, build_wamrc] runs-on: ubuntu-20.04 strategy: matrix: @@ -392,6 +413,8 @@ jobs: $THREADS_TEST_OPTIONS, $WASI_TEST_OPTIONS, ] + llvm_cache_key: + ["${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}"] exclude: # uncompatiable modes and features # classic-interp and fast-interp don't support simd @@ -432,7 +455,7 @@ jobs: #only download llvm libraries in jit and aot mode - name: Get LLVM libraries if: env.USE_LLVM == 'true' - id: cache_llvm + id: retrieve_llvm_libs uses: actions/cache@v3 with: path: | @@ -441,10 +464,10 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }} + key: ${{ matrix.llvm_cache_key }} - name: Quit if cache miss - if: env.USE_LLVM == 'true' && steps.cache_llvm.outputs.cache-hit != 'true' + if: env.USE_LLVM == 'true' && steps.retrieve_llvm_libs.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: run tests diff --git a/.github/workflows/compilation_on_macos.yml b/.github/workflows/compilation_on_macos.yml index 73ebec89..7ad395b2 100644 --- a/.github/workflows/compilation_on_macos.yml +++ b/.github/workflows/compilation_on_macos.yml @@ -51,26 +51,28 @@ env: FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" LLVM_LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" LLVM_EAGER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" - LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex" jobs: build_llvm_libraries: uses: ./.github/workflows/build_llvm_libraries.yml with: - runs-on: "['macos-latest']" + os: "macos-latest" + arch: "X86" build_wamrc: needs: [build_llvm_libraries] runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest] + include: + - os: macos-latest + llvm_cache_key: ${{ needs.build_llvm_libraries.outputs.cache_key }} steps: - name: checkout uses: actions/checkout@v3 - name: Get LLVM libraries - id: cache_llvm + id: retrieve_llvm_libs uses: actions/cache@v3 with: path: | @@ -79,10 +81,10 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} + key: ${{ matrix.llvm_cache_key }} - name: Quit if cache miss - if: steps.cache_llvm.outputs.cache-hit != 'true' + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: Build wamrc @@ -166,13 +168,16 @@ jobs: make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + include: + - os: macos-latest + llvm_cache_key: ${{ needs.build_llvm_libraries.outputs.cache_key }} steps: - name: checkout uses: actions/checkout@v3 # only download llvm cache when needed - name: Get LLVM libraries - id: cache_llvm + id: retrieve_llvm_libs if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') uses: actions/cache@v3 with: @@ -182,10 +187,10 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} + key: ${{ matrix.llvm_cache_key }} - name: Quit if cache miss - if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.cache_llvm.outputs.cache-hit != 'true') + if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: Build iwasm @@ -210,8 +215,14 @@ jobs: #$AOT_BUILD_OPTIONS, ] os: [macos-latest] - wasi_sdk_release: ["https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-macos.tar.gz"] - wabt_release: ["https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-macos.tar.gz"] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-macos.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-macos-12.tar.gz", + ] steps: - name: checkout uses: actions/checkout@v3 @@ -220,24 +231,14 @@ jobs: run: | cd /opt sudo wget ${{ matrix.wabt_release }} - sudo tar -xzf wabt-1.0.24-*.tar.gz - sudo mv wabt-1.0.24 wabt + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo mv wabt-1.0.31 wabt - name: Build Sample [wasm-c-api] run: | - mkdir build && cd build - cmake .. ${{ matrix.make_options }} - cmake --build . --config Release --parallel 4 - ./callback - ./callback_chain - ./empty_imports - ./global - ./hello - ./hostref - ./memory - ./reflect - ./table - ./trap + cmake -S . -B build ${{ matrix.make_options }} + cmake --build build --config Release --parallel 4 + ctest --test-dir build working-directory: samples/wasm-c-api build_samples_others: @@ -246,8 +247,14 @@ jobs: strategy: matrix: os: [macos-latest] - wasi_sdk_release: ["https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-macos.tar.gz"] - wabt_release: ["https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-macos.tar.gz"] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-macos.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-macos-12.tar.gz", + ] steps: - name: checkout uses: actions/checkout@v3 @@ -256,15 +263,15 @@ jobs: run: | cd /opt sudo wget ${{ matrix.wasi_sdk_release }} - sudo tar -xzf wasi-sdk-12.0-*.tar.gz - sudo mv wasi-sdk-12.0 wasi-sdk + sudo tar -xzf wasi-sdk-*.tar.gz + sudo mv wasi-sdk-19.0 wasi-sdk - name: download and install wabt run: | cd /opt sudo wget ${{ matrix.wabt_release }} - sudo tar -xzf wabt-1.0.24-*.tar.gz - sudo mv wabt-1.0.24 wabt + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo mv wabt-1.0.31 wabt - name: Build Sample [basic] run: | diff --git a/.github/workflows/compilation_on_sgx.yml b/.github/workflows/compilation_on_sgx.yml index 70eeaec0..5975c262 100644 --- a/.github/workflows/compilation_on_sgx.yml +++ b/.github/workflows/compilation_on_sgx.yml @@ -51,13 +51,13 @@ env: FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" LLVM_LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" LLVM_EAGER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" - LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex" jobs: build_llvm_libraries: uses: ./.github/workflows/build_llvm_libraries.yml with: - runs-on: "['ubuntu-20.04']" + os: "ubuntu-20.04" + arch: "X86" build_iwasm: runs-on: ${{ matrix.os }} @@ -131,7 +131,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + include: + - os: ubuntu-20.04 + llvm_cache_key: ${{ needs.build_llvm_libraries.outputs.cache_key }} steps: - name: install SGX SDK and necessary libraries run: | @@ -150,7 +152,7 @@ jobs: uses: actions/checkout@v3 - name: Get LLVM libraries - id: cache_llvm + id: retrieve_llvm_libs uses: actions/cache@v3 with: path: | @@ -159,10 +161,10 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} + key: ${{ matrix.llvm_cache_key }} - name: Quit if cache miss - if: steps.cache_llvm.outputs.cache-hit != 'true' + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: Build wamrc @@ -189,11 +191,11 @@ jobs: os: [ubuntu-20.04] wasi_sdk_release: [ - "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz", + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz", ] wabt_release: [ - "https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz", + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz", ] steps: - name: checkout @@ -203,8 +205,8 @@ jobs: run: | cd /opt sudo wget ${{ matrix.wabt_release }} - sudo tar -xzf wabt-1.0.24-*.tar.gz - sudo mv wabt-1.0.24 wabt + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo mv wabt-1.0.31 wabt - name: install SGX SDK and necessary libraries run: | @@ -221,19 +223,9 @@ jobs: - name: Build Sample [wasm-c-api] run: | - mkdir build && cd build - cmake .. ${{ matrix.make_options }} - cmake --build . --config Release --parallel 4 - ./callback - ./callback_chain - ./empty_imports - ./global - ./hello - ./hostref - ./memory - ./reflect - ./table - ./trap + cmake -S . -B build ${{ matrix.make_options }} + cmake --build build --config Release --parallel 4 + ctest --test-dir build working-directory: samples/wasm-c-api build_samples_others: @@ -244,11 +236,11 @@ jobs: os: [ubuntu-20.04] wasi_sdk_release: [ - "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz", + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz", ] wabt_release: [ - "https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz", + "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz", ] steps: - name: checkout @@ -258,15 +250,15 @@ jobs: run: | cd /opt sudo wget ${{ matrix.wasi_sdk_release }} - sudo tar -xzf wasi-sdk-12.0-*.tar.gz - sudo mv wasi-sdk-12.0 wasi-sdk + sudo tar -xzf wasi-sdk-*.tar.gz + sudo mv wasi-sdk-19.0 wasi-sdk - name: download and install wabt run: | cd /opt sudo wget ${{ matrix.wabt_release }} - sudo tar -xzf wabt-1.0.24-*.tar.gz - sudo mv wabt-1.0.24 wabt + sudo tar -xzf wabt-1.0.31-*.tar.gz + sudo mv wabt-1.0.31 wabt - name: install SGX SDK and necessary libraries run: | @@ -334,6 +326,7 @@ jobs: matrix: running_mode: ["classic-interp", "fast-interp", "aot"] test_option: ["-x -p -s spec -b -P", "-x -p -s spec -S -b -P"] + llvm_cache_key: ["${{ needs.build_llvm_libraries.outputs.cache_key }}"] # classic-interp and fast-interp don't support simd exclude: - running_mode: "classic-interp" @@ -347,7 +340,7 @@ jobs: - name: Get LLVM libraries if: matrix.running_mode == 'aot' - id: cache_llvm + id: retrieve_llvm_libs uses: actions/cache@v3 with: path: | @@ -356,10 +349,10 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }} + key: ${{ matrix.llvm_cache_key }} - name: Quit if cache miss - if: matrix.running_mode == 'aot' && steps.cache_llvm.outputs.cache-hit != 'true' + if: matrix.running_mode == 'aot' && steps.retrieve_llvm_libs.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: install SGX SDK and necessary libraries diff --git a/.github/workflows/release_process.yml b/.github/workflows/release_process.yml index 824a0823..88f595dd 100644 --- a/.github/workflows/release_process.yml +++ b/.github/workflows/release_process.yml @@ -123,7 +123,7 @@ jobs: runner: ubuntu-20.04 upload_url: ${{ needs.create_release.outputs.upload_url }} ver_num: ${{ needs.create_tag.outputs.new_ver}} - wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz + wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz release_wamr_sdk_on_ubuntu_2204: needs: [create_tag, create_release] @@ -133,7 +133,7 @@ jobs: runner: ubuntu-22.04 upload_url: ${{ needs.create_release.outputs.upload_url }} ver_num: ${{ needs.create_tag.outputs.new_ver}} - wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz + wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz release_wamr_sdk_on_macos: needs: [create_tag, create_release] @@ -143,7 +143,7 @@ jobs: runner: macos-latest upload_url: ${{ needs.create_release.outputs.upload_url }} ver_num: ${{ needs.create_tag.outputs.new_ver}} - wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-macos.tar.gz + wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-macos.tar.gz # # vscode extension cross-platform diff --git a/.github/workflows/spec_test_on_nuttx.yml b/.github/workflows/spec_test_on_nuttx.yml index 03f99f73..bf39c140 100644 --- a/.github/workflows/spec_test_on_nuttx.yml +++ b/.github/workflows/spec_test_on_nuttx.yml @@ -16,7 +16,8 @@ jobs: build_llvm_libraries: uses: ./.github/workflows/build_llvm_libraries.yml with: - runs-on: "['ubuntu-22.04']" + os: "ubuntu-22.04" + arch: "ARM RISCV AArch64" spec_test_on_qemu: runs-on: ${{ matrix.os }} @@ -37,6 +38,7 @@ jobs: "-t aot", "-t aot -X" ] + llvm_cache_key: [ "${{ needs.build_llvm_libraries.outputs.cache_key }}" ] steps: - name: Install Utilities run: | @@ -72,7 +74,7 @@ jobs: path: apps/interpreters/wamr/wamr - name: Get LLVM libraries - id: cache_llvm + id: retrieve_llvm_libs uses: actions/cache@v3 with: path: | @@ -81,10 +83,10 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} + key: ${{ matrix.llvm_cache_key }} - name: Quit if cache miss - if: steps.cache_llvm.outputs.cache-hit != 'true' + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: Copy LLVM diff --git a/.gitignore b/.gitignore index a7abd7c1..8b4dce9b 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ samples/socket-api/wasm-src/inc/pthread.h **/__pycache__ tests/benchmarks/coremark/coremark* + +samples/workload/include/** +!samples/workload/include/.gitkeep diff --git a/build-scripts/build_llvm.py b/build-scripts/build_llvm.py index bc0daf1f..dd211f30 100755 --- a/build-scripts/build_llvm.py +++ b/build-scripts/build_llvm.py @@ -21,28 +21,31 @@ def clone_llvm(dst_dir, llvm_repo, llvm_branch): llvm_dir = dst_dir.joinpath("llvm").resolve() if not llvm_dir.exists(): - print(f"Clone llvm to {llvm_dir} ...") GIT_CLONE_CMD = f"git clone --depth 1 --branch {llvm_branch} {llvm_repo} llvm" subprocess.check_output(shlex.split(GIT_CLONE_CMD), cwd=dst_dir) - else: - print(f"There is an LLVM local repo in {llvm_dir}, clean and keep using it") return llvm_dir -def build_llvm(llvm_dir, platform, backends, projects): +def query_llvm_version(llvm_dir): + GIT_LOG_CMD = f"git log --format=format:'%h' -1" + return subprocess.check_output( + shlex.split(GIT_LOG_CMD), cwd=llvm_dir, universal_newlines=True, text=True + ) + + +def build_llvm(llvm_dir, platform, backends, projects, use_clang=False): LLVM_COMPILE_OPTIONS = [ '-DCMAKE_BUILD_TYPE:STRING="Release"', "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", "-DLLVM_APPEND_VC_REV:BOOL=ON", - "-DLLVM_BUILD_BENCHMARKS:BOOL=OFF", - "-DLLVM_BUILD_DOCS:BOOL=OFF", "-DLLVM_BUILD_EXAMPLES:BOOL=OFF", "-DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF", "-DLLVM_BUILD_TESTS:BOOL=OFF", - "-DLLVM_CCACHE_BUILD:BOOL=OFF", + "-DLLVM_CCACHE_BUILD:BOOL=ON", "-DLLVM_ENABLE_BINDINGS:BOOL=OFF", "-DLLVM_ENABLE_IDE:BOOL=OFF", + "-DLLVM_ENABLE_LIBEDIT=OFF", "-DLLVM_ENABLE_TERMINFO:BOOL=OFF", "-DLLVM_ENABLE_ZLIB:BOOL=OFF", "-DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF", @@ -54,6 +57,18 @@ def build_llvm(llvm_dir, platform, backends, projects): "-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON", ] + # use clang/clang++/lld. but macos doesn't support lld + if not sys.platform.startswith("darwin") and use_clang: + if shutil.which("clang") and shutil.which("clang++") and shutil.which("lld"): + os.environ["CC"] = "clang" + os.environ["CXX"] = "clang++" + LLVM_COMPILE_OPTIONS.append('-DLLVM_USE_LINKER:STRING="lld"') + print("Use the clang toolchain") + else: + print("Can not find clang, clang++ and lld, keep using the gcc toolchain") + else: + print("Use the gcc toolchain") + LLVM_EXTRA_COMPILE_OPTIONS = { "arc": [ '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="ARC"', @@ -99,8 +114,10 @@ def build_llvm(llvm_dir, platform, backends, projects): lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve() if lib_llvm_core_library.exists(): - print(f"Please remove {build_dir} manually and try again") - return build_dir + print( + f"It has already been fully compiled. If want to a re-build, please remove {build_dir} manually and try again" + ) + return None compile_options = " ".join( LLVM_COMPILE_OPTIONS @@ -119,10 +136,11 @@ def build_llvm(llvm_dir, platform, backends, projects): CONFIG_CMD += " -G'Unix Makefiles'" else: CONFIG_CMD += " -A x64" - print(f"{CONFIG_CMD}") + else: + CONFIG_CMD += " -G'Ninja'" subprocess.check_call(shlex.split(CONFIG_CMD), cwd=build_dir) - BUILD_CMD = f"cmake --build . --target package --parallel {os.cpu_count()}" + ( + BUILD_CMD = "cmake --build . --target package" + ( " --config Release" if "windows" == platform else "" ) subprocess.check_call(shlex.split(BUILD_CMD), cwd=build_dir) @@ -133,23 +151,25 @@ def build_llvm(llvm_dir, platform, backends, projects): def repackage_llvm(llvm_dir): build_dir = llvm_dir.joinpath("./build").resolve() - packs = [f for f in build_dir.glob("LLVM-13*.tar.gz")] + packs = [f for f in build_dir.glob("LLVM-*.tar.gz")] if len(packs) > 1: - raise Exception("Find more than one LLVM-13*.tar.gz") + raise Exception("Find more than one LLVM-*.tar.gz") if not packs: return llvm_package = packs[0].name - # mv build/LLVM-13.0.0*.gz . + # mv build/LLVM-*.gz . shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir)) # rm -r build shutil.rmtree(str(build_dir)) # mkdir build build_dir.mkdir() - # tar xf ./LLVM-13.0.0-*.tar.gz --strip-components=1 --directory=build + # tar xf ./LLVM-*.tar.gz --strip-components=1 --directory=build CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}" subprocess.check_call(shlex.split(CMD), cwd=llvm_dir) + # rm ./LLVM-1*.gz + os.remove(llvm_dir.joinpath(llvm_package).resolve()) def main(): @@ -184,8 +204,17 @@ def main(): choices=["clang", "lldb"], help="identify extra LLVM projects, separate by space, like '--project clang lldb'", ) + parser.add_argument( + "--llvm-ver", + action="store_true", + help="return the version info of generated llvm libraries", + ) + parser.add_argument( + "--use-clang", + action="store_true", + help="use clang instead of gcc", + ) options = parser.parse_args() - print(f"options={options}") # if the "platform" is not identified in the command line option, # detect it @@ -199,12 +228,10 @@ def main(): else: platform = options.platform - print(f"========== Build LLVM for {platform} ==========\n") - llvm_repo_and_branch = { "arc": { "repo": "https://github.com/llvm/llvm-project.git", - "branch": "release/13.x", + "branch": "release/15.x", }, "xtensa": { "repo": "https://github.com/espressif/llvm-project.git", @@ -212,7 +239,7 @@ def main(): }, "default": { "repo": "https://github.com/llvm/llvm-project.git", - "branch": "release/13.x", + "branch": "release/15.x", }, } @@ -225,19 +252,22 @@ def main(): deps_dir = current_dir.joinpath("../core/deps").resolve() try: - print(f"==================== CLONE LLVM ====================") llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"]) llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"]) - print() - print(f"==================== BUILD LLVM ====================") - build_llvm(llvm_dir, platform, options.arch, options.project) + if options.llvm_ver: + commit_hash = query_llvm_version(llvm_dir) + print(commit_hash) + return commit_hash is not None - print() - print(f"==================== PACKAGE LLVM ====================") - repackage_llvm(llvm_dir) + if ( + build_llvm( + llvm_dir, platform, options.arch, options.project, options.use_clang + ) + is not None + ): + repackage_llvm(llvm_dir) - print() return True except subprocess.CalledProcessError: return False diff --git a/doc/build_wasm_app.md b/doc/build_wasm_app.md index a10a1a4c..40f1b89d 100644 --- a/doc/build_wasm_app.md +++ b/doc/build_wasm_app.md @@ -3,12 +3,12 @@ Prepare WASM building environments ================================== -For C and C++, WASI-SDK version 12.0+ is the major tool supported by WAMR to build WASM applications. Also, we can use [Emscripten SDK (EMSDK)](https://github.com/emscripten-core/emsdk), but it is not recommended. And there are some other compilers such as the standard clang compiler, which might also work [here](./other_wasm_compilers.md). +For C and C++, WASI-SDK version 19.0+ is the major tool supported by WAMR to build WASM applications. Also, we can use [Emscripten SDK (EMSDK)](https://github.com/emscripten-core/emsdk), but it is not recommended. And there are some other compilers such as the standard clang compiler, which might also work [here](./other_wasm_compilers.md). To install WASI SDK, please download the [wasi-sdk release](https://github.com/CraneStation/wasi-sdk/releases) and extract the archive to default path `/opt/wasi-sdk`. The official *wasi-sdk release* doesn't fully support *latest 128-bit SIMD spec* yet. WAMR provides a script in [build-wasi-sdk](../test-tools/build-wasi-sdk/) to generate -another wasi-sdk with *llvm-13* from source code and installs it at *../test-tools/wasi-sdk*. If you plan to build WASM applications with *latest 128-bit SIMD*, please use it instead of the official release. +another wasi-sdk with *llvm-15* from source code and installs it at *../test-tools/wasi-sdk*. If you plan to build WASM applications with *latest 128-bit SIMD*, please use it instead of the official release. And [sample workloads](../samples/workload) are using the self-compiled wasi-sdk. diff --git a/samples/wasm-c-api/CMakeLists.txt b/samples/wasm-c-api/CMakeLists.txt index a6494624..e2be3b8f 100644 --- a/samples/wasm-c-api/CMakeLists.txt +++ b/samples/wasm-c-api/CMakeLists.txt @@ -136,26 +136,13 @@ include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) set(MM_UTIL src/utils/multi_module_utils.c) # build executable for each .c -set(EXAMPLES - callback - callback_chain - clone - empty_imports - global - hello - hostref - memory - reflect - table - threads - trap -) - -if(WAMR_BUILD_JIT AND WAMR_BUILD_LAZY_JIT) - if((${WAMR_BUILD_JIT} EQUAL 1) AND (${WAMR_BUILD_LAZY_JIT} EQUAL 1)) - list(APPEND EXAMPLES serialize) - endif() -endif() +list(APPEND EXAMPLES callback callback_chain empty_imports global hello hostref memory reflect table trap) +# FIXME enable both in the future +#list(APPEND EXAMPLES clone threads) +# FIXME +# if(WAMR_BUILD_JIT EQUAL 1 AND WAMR_BUILD_LAZY_JIT EQUAL 0) +# list(APPEND EXAMPLES serialize) +# endif() check_pie_supported() diff --git a/samples/wasm-c-api/README.md b/samples/wasm-c-api/README.md index 51b8642a..b2327670 100644 --- a/samples/wasm-c-api/README.md +++ b/samples/wasm-c-api/README.md @@ -4,9 +4,9 @@ Before staring, we need to download and intall [WABT](https://github.com/WebAsse ``` shell $ cd /opt -$ wget https://github.com/WebAssembly/wabt/releases/download/1.0.19/wabt-1.0.19-ubuntu.tar.gz -$ tar -xzf wabt-1.0.19-ubuntu.tar.gz -$ mv wabt-1.0.19 wabt +$ wget https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz +$ tar -xzf wabt-1.0.31-ubuntu.tar.gz +$ mv wabt-1.0.31 wabt ``` By default, all samples are compiled and run in "interpreter" mode. @@ -47,4 +47,4 @@ $ ./global $ ... $ ./callback $ ... -``` \ No newline at end of file +``` diff --git a/samples/workload/CMakeLists.txt b/samples/workload/CMakeLists.txt new file mode 100644 index 00000000..5e5dca28 --- /dev/null +++ b/samples/workload/CMakeLists.txt @@ -0,0 +1,116 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(wasm_workloads) + +####################################### +add_subdirectory(bwa) +add_subdirectory(meshoptimizer) +add_subdirectory(wasm-av1) + +####################################### +include(ExternalProject) + +################ iwasm ################ +ExternalProject_Add(iwasm + PREFIX + iwasm-build + BUILD_ALWAYS + YES + SOURCE_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/../../product-mini/platforms/linux + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/../../product-mini/platforms/linux -B build -DWAMR_BUILD_LIBC_EMCC=1 + BUILD_COMMAND + ${CMAKE_COMMAND} --build build + INSTALL_COMMAND + # FIXME: replace with --install + ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/iwasm-build/src/iwasm-build/build/iwasm + ${CMAKE_CURRENT_BINARY_DIR}/iwasm +) + +################ wamrc ################ +ExternalProject_Add(wamrc + PREFIX + wamrc-build + BUILD_ALWAYS + YES + SOURCE_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/../../wamr-compiler + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/../../wamr-compiler -B build + BUILD_COMMAND + ${CMAKE_COMMAND} --build build + INSTALL_COMMAND + # FIXME: replace with --install + ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/wamrc-build/src/wamrc-build/build/wamrc + ${CMAKE_CURRENT_BINARY_DIR}/wamrc +) + +################ .aot ################ +add_custom_target( + bwa_to_aot + ALL + DEPENDS + bwa wamrc + COMMAND + ./wamrc -o bwa.aot ./bwa/bwa.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_custom_target( + codecbench_to_aot + ALL + DEPENDS + codecbench wamrc + COMMAND + ./wamrc -o codecbench.aot ./meshoptimizer/codecbench.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_custom_target( + av1_to_aot + ALL + DEPENDS + av1 wamrc + COMMAND + ./wamrc -o testavx.aot ./wasm-av1/testavx.opt.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +################ smoking test ################ +include(CTest) + +add_test( + NAME + run_bwa + COMMAND + ./iwasm --dir=. ./bwa.aot index ./bwa/hs38DH-extra.fa + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_test( + NAME + run_codecbench + COMMAND + ./iwasm codecbench.aot + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_test( + NAME + run_av1 + COMMAND + ./iwasm --dir=. testavx.aot ./wasm-av1/elephants_dream_480p24.ivf + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) \ No newline at end of file diff --git a/samples/workload/README.md b/samples/workload/README.md index dd94e05c..0e6a3a41 100644 --- a/samples/workload/README.md +++ b/samples/workload/README.md @@ -1,41 +1,30 @@ -All workloads have similar requirment of software dependencies, including -**emsdk**, **wabt** and **binaryen** +All workloads have similar requirment of software dependencies, including **emsdk** and **binaryen** -> There might be slight differences when using MacOS and other Linux distro than Ubuntu. This document only target -Ubuntu 18.04 as example. +> There might be slight differences when using MacOS and other Linux distro than Ubuntu. This document targets +Ubuntu 20.04 as an example. ## Installation instructions -use [preparation.sh](./preparation.sh) to install all dependencies before compiling any workload. +use [preparation.sh](./preparation.sh) to install all dependencies before compiling any workload. Or use [*vscode DevContainer*](../../.devcontainer/) -for details, the script includes below steps: - -- **wabt**. Install - [latest release](https://github.com/WebAssembly/wabt/releases/download/1.0.23/wabt-1.0.23-ubuntu.tar.gz) - to */opt/wabt* - -``` bash -$ wget https://github.com/WebAssembly/wabt/releases/download/${WABT_VER}/${WABT_FILE} -$ tar zxf ${WABT_FILE} -C /opt -$ ln -sf /opt/wabt-${WABT_VER} /opt/wabt -``` +The script installs below software: - **emsdk**. Refer to [the guide](https://emscripten.org/docs/getting_started/downloads.html). Don't forget to activate emsdk and set up environment variables. Verify it with `echo ${EMSDK}`. Please be sure to install and activate the building - of 2.0.26 + of 3.0.0 ``` bash $ cd /opt $ git clone https://github.com/emscripten-core/emsdk.git $ cd emsdk $ git pull -$ ./emsdk install 2.0.26 -$ ./emsdk activate 2.0.26 +$ ./emsdk install 3.0.0 +$ ./emsdk activate 3.0.0 $ echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc ``` - **binaryen**. Install - [latest release](https://github.com/WebAssembly/binaryen/releases/download/version_101/binaryen-version_101-x86_64-linux.tar.gz) + [latest release](https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-linux.tar.gz) to */opt/binaryen* ``` bash diff --git a/samples/workload/XNNPACK/CMakeLists.txt b/samples/workload/XNNPACK/CMakeLists.txt index 3f67367a..532544f9 100644 --- a/samples/workload/XNNPACK/CMakeLists.txt +++ b/samples/workload/XNNPACK/CMakeLists.txt @@ -15,8 +15,9 @@ ExternalProject_Add(xnnpack GIT_PROGRESS ON SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack UPDATE_COMMAND git checkout . - && git reset --hard 4d738aef36872669e4bba05a4b259149ba8e62e1 - && cmake -E copy ${CMAKE_CURRENT_SOURCE_DIR}/benchmark.patch ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/third_party + && git reset --hard 4570a7151aa4f3e57eca14a575eeff6bb13e26be + && cmake -E copy ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/google3/third_party/XNNPACK/microkernels.bzl + ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/ && git apply ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack.patch CONFIGURE_COMMAND "" # grep xnnpack_benchmark -A 1 BUILD.bazel \ @@ -24,70 +25,123 @@ ExternalProject_Add(xnnpack # | awk '{print $3}' \ # | sed -e 's/\"//g' -e 's/,//g' -e 's/^/\/\/:/g' BUILD_COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack - && bazel --output_user_root=build_user_output build -c opt --config=wasm + && bazel --output_user_root=build-user-output build -c opt --config=wasm //:qs8_dwconv_bench.wasm + //:qs8_f32_vcvt_bench.wasm //:qs8_gemm_bench.wasm //:qs8_requantization_bench.wasm //:qs8_vadd_bench.wasm //:qs8_vaddc_bench.wasm + //:qs8_vcvt_bench.wasm + //:qs8_vlrelu_bench.wasm + //:qs8_vmul_bench.wasm + //:qs8_vmulc_bench.wasm + //:qu8_f32_vcvt_bench.wasm //:qu8_gemm_bench.wasm //:qu8_requantization_bench.wasm //:qu8_vadd_bench.wasm //:qu8_vaddc_bench.wasm + //:qu8_vcvt_bench.wasm + //:qu8_vlrelu_bench.wasm + //:qu8_vmul_bench.wasm + //:qu8_vmulc_bench.wasm + //:bf16_gemm_bench.wasm //:f16_igemm_bench.wasm //:f16_gemm_bench.wasm + //:f16_raddstoreexpminusmax_bench.wasm //:f16_spmm_bench.wasm - //:f16_vrelu_bench.wasm + //:f16_vsigmoid_bench.wasm + //:f16_f32_vcvt_bench.wasm //:f32_igemm_bench.wasm //:f32_conv_hwc_bench.wasm + //:f16_conv_hwc2chw_bench.wasm + //:f16_gavgpool_cw_bench.wasm + //:f32_gavgpool_cw_bench.wasm //:f32_conv_hwc2chw_bench.wasm //:f16_dwconv_bench.wasm //:f32_dwconv_bench.wasm //:f32_dwconv2d_chw_bench.wasm + //:f16_dwconv2d_chw_bench.wasm + //:f32_f16_vcvt_bench.wasm + //:xx_transpose_bench.wasm + //:x8_transpose_bench.wasm + //:x16_transpose_bench.wasm + //:x24_transpose_bench.wasm + //:x32_transpose_bench.wasm + //:x64_transpose_bench.wasm //:f32_gemm_bench.wasm + //:f32_qs8_vcvt_bench.wasm + //:f32_qu8_vcvt_bench.wasm //:f32_raddexpminusmax_bench.wasm //:f32_raddextexp_bench.wasm //:f32_raddstoreexpminusmax_bench.wasm //:f32_rmax_bench.wasm //:f32_spmm_bench.wasm //:f32_softmax_bench.wasm + //:f16_velu_bench.wasm //:f32_velu_bench.wasm //:f32_vhswish_bench.wasm + //:f32_vlrelu_bench.wasm //:f32_vrelu_bench.wasm //:f32_vscaleexpminusmax_bench.wasm //:f32_vscaleextexp_bench.wasm //:f32_vsigmoid_bench.wasm + //:f16_vsqrt_bench.wasm //:f32_vsqrt_bench.wasm //:f32_im2col_gemm_bench.wasm //:rounding_bench.wasm + //:s16_rmaxabs_bench.wasm + //:s16_window_bench.wasm + //:u32_filterbank_accumulate_bench.wasm + //:u32_filterbank_subtract_bench.wasm + //:u32_vlog_bench.wasm + //:u64_u32_vsqrtshift_bench.wasm + //:i16_vlshift_bench.wasm + //:cs16_vsquareabs_bench.wasm + //:cs16_bfly4_bench.wasm + //:cs16_fftr_bench.wasm + //:x8_lut_bench.wasm + //:abs_bench.wasm //:average_pooling_bench.wasm //:bankers_rounding_bench.wasm //:ceiling_bench.wasm //:channel_shuffle_bench.wasm + //:convert_bench.wasm //:convolution_bench.wasm //:deconvolution_bench.wasm //:elu_bench.wasm //:floor_bench.wasm //:global_average_pooling_bench.wasm //:hardswish_bench.wasm + //:leaky_relu_bench.wasm //:max_pooling_bench.wasm + //:negate_bench.wasm //:sigmoid_bench.wasm //:prelu_bench.wasm //:softmax_bench.wasm + //:square_bench.wasm //:square_root_bench.wasm //:truncation_bench.wasm + //:f16_gemm_e2e_bench.wasm //:f32_dwconv_e2e_bench.wasm //:f32_gemm_e2e_bench.wasm //:qs8_dwconv_e2e_bench.wasm //:qs8_gemm_e2e_bench.wasm + //:qu8_gemm_e2e_bench.wasm //:qu8_dwconv_e2e_bench.wasm //:end2end_bench.wasm + //:f16_exp_ulp_eval.wasm + //:f16_expminus_ulp_eval.wasm + //:f16_expm1minus_ulp_eval.wasm + //:f16_sigmoid_ulp_eval.wasm + //:f16_sqrt_ulp_eval.wasm //:f32_exp_ulp_eval.wasm //:f32_expminus_ulp_eval.wasm //:f32_expm1minus_ulp_eval.wasm //:f32_extexp_ulp_eval.wasm //:f32_sigmoid_ulp_eval.wasm //:f32_sqrt_ulp_eval.wasm + //:f32_tanh_ulp_eval.wasm INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/bazel-out/wasm-opt/bin/ ${CMAKE_BINARY_DIR}/wasm-opt diff --git a/samples/workload/XNNPACK/README.md b/samples/workload/XNNPACK/README.md index f6a20729..7984d9ce 100644 --- a/samples/workload/XNNPACK/README.md +++ b/samples/workload/XNNPACK/README.md @@ -24,7 +24,7 @@ Firstly please build iwasm with simd, libc-emcc and lib-pthread support: ``` bash $ cd /product-mini/platforms/linux/ $ mkdir build && cd build -$ cmake .. -DWAMR_BUILD_SIMD=1 -DWAMR_BUILD_LIBC_EMCC=1 -DWAMR_BUILD_LIB_PTHREAD=1 +$ cmake .. -DWAMR_BUILD_LIBC_EMCC=1 -DWAMR_BUILD_LIB_PTHREAD=1 $ make ``` @@ -42,7 +42,7 @@ Then compile wasm file to aot file and run: ``` shell $ cd /samples/workload/XNNPACK/xnnpack/bazel-bin -$ wamrc --enable-simd -o average_pooling_bench.aot average_pooling_bench.wasm (or other wasm files) +$ wamrc -o average_pooling_bench.aot average_pooling_bench.wasm (or other wasm files) $ iwasm average_pooling_bench.aot ``` diff --git a/samples/workload/XNNPACK/build_workload.sh b/samples/workload/XNNPACK/build_workload.sh deleted file mode 120000 index a31afa92..00000000 --- a/samples/workload/XNNPACK/build_workload.sh +++ /dev/null @@ -1 +0,0 @@ -../docker/build_workload.sh \ No newline at end of file diff --git a/samples/workload/XNNPACK/xnnpack.patch b/samples/workload/XNNPACK/xnnpack.patch index 7eed678b..f7d0a01d 100644 --- a/samples/workload/XNNPACK/xnnpack.patch +++ b/samples/workload/XNNPACK/xnnpack.patch @@ -1,8 +1,8 @@ diff --git a/.bazelrc b/.bazelrc -index ec740f38..29f9d56e 100644 +index 688279da1..376996885 100644 --- a/.bazelrc +++ b/.bazelrc -@@ -49,4 +49,9 @@ build:ios_fat --watchos_cpus=armv7k +@@ -53,4 +53,9 @@ build:ios_fat --watchos_cpus=armv7k build:macos --apple_platform_type=macos build:macos_arm64 --config=macos @@ -11,42 +11,26 @@ index ec740f38..29f9d56e 100644 +build:macos_arm64 --cpu=darwin_arm64 + +build:wasm --cpu=wasm -+build:wasm --copt=-msimd128 ++build:wasm --features=wasm_simd +build:wasm --crosstool_top=@emsdk//emscripten_toolchain:everything +build:wasm --host_crosstool_top=@bazel_tools//tools/cpp:toolchain -diff --git a/BUILD.bazel b/BUILD.bazel -index 3fc8139f..c893356d 100644 ---- a/BUILD.bazel -+++ b/BUILD.bazel -@@ -11988,7 +11988,6 @@ config_setting( - values = { - "crosstool_top": "@emsdk//emscripten_toolchain:everything", - "cpu": "wasm", -- "copt": "-msimd128", - "copt": "-mrelaxed-simd", - }, - ) diff --git a/WORKSPACE b/WORKSPACE -index c58e76b6..30934678 100644 +index cd8960ffa..5d3e685f4 100644 --- a/WORKSPACE +++ b/WORKSPACE -@@ -21,6 +21,7 @@ http_archive( - name = "com_google_benchmark", - strip_prefix = "benchmark-master", - urls = ["https://github.com/google/benchmark/archive/master.zip"], -+ patches = ["@//third_party:benchmark.patch"], - ) - - # FP16 library, used for half-precision conversions -@@ -84,6 +85,19 @@ http_archive( - ], +@@ -92,8 +92,25 @@ http_archive( + ], ) ++load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( -+ name = "emsdk", -+ strip_prefix = "emsdk-2.0.26/bazel", -+ url = "https://github.com/emscripten-core/emsdk/archive/refs/tags/2.0.26.tar.gz", -+ sha256 = "79e7166aa8eaae6e52cef1363b2d8db795d03684846066bc51f9dcf905dd58ad", ++ name = "emsdk", ++ # Use emsdk-3.0.0 since the larger version may: ++ # - compress the wasm file into a tar file but not directly generate wasm file ++ # - generate incomplete implementation of libc API, e.g. throw exception in getentropy ++ strip_prefix = "emsdk-3.0.0/bazel", ++ url = "https://github.com/emscripten-core/emsdk/archive/refs/tags/3.0.0.tar.gz", ++ sha256 = "a41dccfd15be9e85f923efaa0ac21943cbab77ec8d39e52f25eca1ec61a9ac9e" +) + +load("@emsdk//:deps.bzl", emsdk_deps = "deps") @@ -56,13 +40,17 @@ index c58e76b6..30934678 100644 +emsdk_emscripten_deps() + # Android NDK location and version is auto-detected from $ANDROID_NDK_HOME environment variable - android_ndk_repository(name = "androidndk") +-android_ndk_repository(name = "androidndk") ++#android_ndk_repository(name = "androidndk") + # Android SDK location and API is auto-detected from $ANDROID_HOME environment variable +-android_sdk_repository(name = "androidsdk") ++#android_sdk_repository(name = "androidsdk") diff --git a/build_defs.bzl b/build_defs.bzl -index fbadb400..e496b78d 100644 +index b8217a18d..da232966e 100644 --- a/build_defs.bzl +++ b/build_defs.bzl -@@ -430,7 +430,7 @@ def xnnpack_benchmark(name, srcs, copts = [], deps = [], tags = []): +@@ -380,7 +380,7 @@ def xnnpack_benchmark(name, srcs, copts = [], deps = [], tags = []): explicitly specified. """ native.cc_binary( @@ -72,7 +60,7 @@ index fbadb400..e496b78d 100644 copts = xnnpack_std_cxxopts() + [ "-Iinclude", diff --git a/emscripten.bzl b/emscripten.bzl -index 130d5f16..2696ad54 100644 +index f1557a7b1..7f964a094 100644 --- a/emscripten.bzl +++ b/emscripten.bzl @@ -25,12 +25,19 @@ def xnnpack_emscripten_benchmark_linkopts(): @@ -84,7 +72,7 @@ index 130d5f16..2696ad54 100644 - "-s EXIT_RUNTIME=1", + "-s ERROR_ON_UNDEFINED_SYMBOLS=0", "-s ALLOW_MEMORY_GROWTH=1", - "-s TOTAL_MEMORY=445644800", # 425M + "-s TOTAL_MEMORY=536870912", # 512M - "--pre-js $(location :preamble.js.lds)", + "-s USE_PTHREADS=0", + "-s STANDALONE_WASM=1", @@ -99,11 +87,33 @@ index 130d5f16..2696ad54 100644 ] def xnnpack_emscripten_deps(): +diff --git a/src/log.c b/src/log.c +index 5715f2f85..4b3e4261b 100644 +--- a/src/log.c ++++ b/src/log.c +@@ -55,7 +55,7 @@ + #endif + + #if XNN_LOG_TO_STDIO +-static void xnn_vlog(int output_handle, const char* prefix, size_t prefix_length, const char* format, va_list args) { ++void xnn_vlog(int output_handle, const char* prefix, size_t prefix_length, const char* format, va_list args) { + char stack_buffer[XNN_LOG_STACK_BUFFER_SIZE]; + char* heap_buffer = NULL; + char* out_buffer = &stack_buffer[0]; diff --git a/third_party/cpuinfo.BUILD b/third_party/cpuinfo.BUILD -index 128d683e..f6c287c4 100644 +index 1997f4e3a..5e03c43af 100644 --- a/third_party/cpuinfo.BUILD +++ b/third_party/cpuinfo.BUILD -@@ -343,5 +343,5 @@ config_setting( +@@ -150,7 +150,7 @@ cc_library( + "src/arm/midr.h", + ], + deps = [ +- "@clog", ++ "//deps/clog" + ], + ) + +@@ -352,5 +352,5 @@ config_setting( config_setting( name = "emscripten", diff --git a/samples/workload/bwa/CMakeLists.bwa_wasm.txt b/samples/workload/bwa/CMakeLists.bwa_wasm.txt index c68b942f..a8d1d882 100644 --- a/samples/workload/bwa/CMakeLists.bwa_wasm.txt +++ b/samples/workload/bwa/CMakeLists.bwa_wasm.txt @@ -1,11 +1,14 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required (VERSION 3.0) +cmake_minimum_required (VERSION 3.14) project(bwa_wasm C) -include(${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/preparation.cmake) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake) + +################ dependencies ################ +find_package(Binaryen 111 REQUIRED) ################ LIBZ ################ set(LIBZ_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../libz) @@ -86,12 +89,6 @@ add_executable(${PROJECT_NAME} ${BWA_SOURCE}) set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME bwa.wasm) -target_include_directories(${PROJECT_NAME} - PRIVATE - ${WASI_SDK_HOME}/share/wasi-sysroot/include/libc/musl - ${WASI_SDK_HOME}/share/wasi-sysroot/include/sse -) - target_compile_definitions(${PROJECT_NAME} PRIVATE USE_MALLOC_WRAPPERS @@ -117,7 +114,7 @@ target_link_libraries(${PROJECT_NAME} z_wasm wasi-emulated-process-clocks) add_custom_target(bwa_wasm_opt ALL COMMAND - ${WASM_OPT} -Oz --enable-simd -o bwa.opt.wasm bwa.wasm + ${Binaryen_WASM_OPT} -Oz --enable-simd -o bwa.opt.wasm bwa.wasm BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/bwa.opt.wasm WORKING_DIRECTORY diff --git a/samples/workload/bwa/CMakeLists.txt b/samples/workload/bwa/CMakeLists.txt index 7f7e3182..731dd4c7 100644 --- a/samples/workload/bwa/CMakeLists.txt +++ b/samples/workload/bwa/CMakeLists.txt @@ -1,11 +1,19 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required (VERSION 2.8...3.16) +cmake_minimum_required (VERSION 3.14) project(bwa_wasm) -include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/preparation.cmake) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) + +################ dependencies ################ +find_package(Python3 REQUIRED) +find_package(WASISDK 16.0 REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) ####################################### include(ExternalProject) @@ -13,7 +21,7 @@ include(ExternalProject) ################ libz ################ ExternalProject_Add(libz_src GIT_REPOSITORY https://github.com/madler/zlib.git - GIT_TAG master + GIT_TAG 04f42ceca40f73e2978b50e93806c2a18c1281fc GIT_PROGRESS ON GIT_SHALLOW ON SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libz @@ -27,7 +35,7 @@ ExternalProject_Add(libz_src ################ bwa ################ ExternalProject_Add(bwa GIT_REPOSITORY https://github.com/lh3/bwa.git - GIT_TAG master + GIT_TAG 139f68fc4c3747813783a488aef2adc86626b01b GIT_PROGRESS ON GIT_SHALLOW ON SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/bwa @@ -37,10 +45,29 @@ ExternalProject_Add(bwa && ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.bwa_wasm.txt CMakeLists.txt && git apply ../bwa.patch CONFIGURE_COMMAND ${CMAKE_COMMAND} - -DWASI_SDK_PREFIX=${WASI_SDK_HOME} - -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_HOME}/share/cmake/wasi-sdk.cmake - -DCMAKE_SYSROOT=${WASI_SDK_HOME}/share/wasi-sysroot + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + -DCMAKE_SYSROOT=${WASISDK_SYSROOT} + -DCMAKE_C_FLAGS=-isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/sse\ -isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/libc/musl ${CMAKE_CURRENT_SOURCE_DIR}/bwa BUILD_COMMAND make bwa_wasm_opt - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ./bwa.opt.wasm ${CMAKE_BINARY_DIR}/bwa.wasm + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different ./bwa.opt.wasm ${CMAKE_CURRENT_BINARY_DIR}/bwa.wasm ) + +################ bwa data ################ +ExternalProject_Add(bwa-kit + PREFIX bwa-kit + URL https://sourceforge.net/projects/bio-bwa/files/bwakit/bwakit-0.7.15_x64-linux.tar.bz2/download + URL_HASH SHA256=0a7b11971bc7916b68e9df35a364afe77cb3000df02ffb3a6fbd1aff9be5878c + DOWNLOAD_NAME bwakit-0.7.15_x64-linux.tar.bz2 + DOWNLOAD_EXTRACT_TIMESTAMP ON + DOWNLOAD_NO_EXTRACT OFF + DOWNLOAD_NO_PROGRESS ON + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/bwa-kit/src/bwa-kit/resource-GRCh38/hs38DH-extra.fa + ${CMAKE_CURRENT_BINARY_DIR}/hs38DH-extra.fa +) \ No newline at end of file diff --git a/samples/workload/bwa/README.md b/samples/workload/bwa/README.md index e826bb61..a8fbe3e6 100644 --- a/samples/workload/bwa/README.md +++ b/samples/workload/bwa/README.md @@ -33,7 +33,7 @@ Firstly please build iwasm with simd support: ``` shell $ cd /product-mini/platforms/linux/ $ mkdir build && cd build -$ cmake .. -DWAMR_BUILD_SIMD=1 +$ cmake .. $ make ``` @@ -41,6 +41,6 @@ Then compile wasm file to aot file and run: ``` shell $ cd /samples/workload/bwa/build -$ /wamr-compiler/build/wamrc --enable-simd -o bwa.aot bwa.wasm +$ /wamr-compiler/build/wamrc -o bwa.aot bwa.wasm $ /product-mini/platforms/linux/iwasm --dir=. bwa.aot index hs38DH.fa ``` diff --git a/samples/workload/bwa/build_workload.sh b/samples/workload/bwa/build_workload.sh deleted file mode 120000 index a31afa92..00000000 --- a/samples/workload/bwa/build_workload.sh +++ /dev/null @@ -1 +0,0 @@ -../docker/build_workload.sh \ No newline at end of file diff --git a/samples/workload/cmake/FindBinaryen.cmake b/samples/workload/cmake/FindBinaryen.cmake new file mode 100644 index 00000000..b4a64786 --- /dev/null +++ b/samples/workload/cmake/FindBinaryen.cmake @@ -0,0 +1,43 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# +# Output below variables: +# - Binaryen_HOME. the installation location +# + +include(CMakePrintHelpers) +include(FindPackageHandleStandardArgs) + +file(GLOB Binaryen_SEARCH_PATH "/opt/binaryen*") +find_path(Binaryen_HOME + NAMES bin/wasm-opt + PATHS ${Binaryen_SEARCH_PATH} + NO_CMAKE_FIND_ROOT_PATH + NO_SYSTEM_ENVIRONMENT_PATH + REQUIRED +) + +execute_process( + COMMAND ${Binaryen_HOME}/bin/wasm-opt --version + OUTPUT_VARIABLE WASM_OPT_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +string(REGEX MATCH version_[0-9]+ Binaryen_VERSION_tmp ${WASM_OPT_OUTPUT}) +string(REGEX MATCH [0-9]+ Binaryen_VERSION ${Binaryen_VERSION_tmp}) + +#cmake_print_variables(Binaryen_VERSION_tmp Binaryen_VERSION) + +find_package_handle_standard_args(Binaryen REQUIRED_VARS Binaryen_HOME VERSION_VAR Binaryen_VERSION) + +if(Binaryen_FOUND) + mark_as_advanced(Binaryen_SEARCH_PATH) + mark_as_advanced(Binaryen_VERSION_tmp) + mark_as_advanced(Binaryen_VERSION) + mark_as_advanced(WASM_OPT_OUTPUT) + + set(Binaryen_WASM_OPT ${Binaryen_HOME}/bin/wasm-opt) +else() + # TODO: install WASISDK +endif() diff --git a/samples/workload/cmake/FindWASISDK.cmake b/samples/workload/cmake/FindWASISDK.cmake new file mode 100644 index 00000000..fff8aea4 --- /dev/null +++ b/samples/workload/cmake/FindWASISDK.cmake @@ -0,0 +1,38 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# +# Output below variables: +# - WASISDK_HOME. the installation location +# - WASISDK_SYSROOT. where wasi-sysroot is +# - WASISDK_TOOLCHAIN. where wasi-sdk.cmake is +# + +include(CMakePrintHelpers) +include(FindPackageHandleStandardArgs) + +file(GLOB WASISDK_SEARCH_PATH "/opt/wasi-sdk-*") +find_path(WASISDK_HOME + NAMES share/wasi-sysroot + PATHS ${WASISDK_SEARCH_PATH} + NO_CMAKE_FIND_ROOT_PATH + NO_SYSTEM_ENVIRONMENT_PATH + REQUIRED +) + +string(REGEX MATCH [0-9]+\.[0-9]+\.*[0-9]* WASISDK_VERSION ${WASISDK_HOME}) + +#cmake_print_variables(WASISDK_HOME WASISDK_VERSION) +find_package_handle_standard_args(WASISDK REQUIRED_VARS WASISDK_HOME VERSION_VAR WASISDK_VERSION) + +if(WASISDK_FOUND) + mark_as_advanced(WASISDK_SEARCH_PATH) + mark_as_advanced(WASISDK_VERSION) + + set(WASISDK_CC_COMMAND ${WASISDK_HOME}/bin/clang) + set(WASISDK_CXX_COMMAND ${WASISDK_HOME}/bin/clang++) + set(WASISDK_SYSROOT ${WASISDK_HOME}/share/wasi-sysroot) + set(WASISDK_TOOLCHAIN ${WASISDK_HOME}/share/cmake/wasi-sdk.cmake) +else() + # TODO: install WASISDK +endif() diff --git a/samples/workload/cmake/preparation.cmake b/samples/workload/cmake/preparation.cmake deleted file mode 100644 index 326943c8..00000000 --- a/samples/workload/cmake/preparation.cmake +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -####################################### -include(ExternalProject) - -file(REAL_PATH ../../.. WAMR_ROOT - BASE_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} -) - -find_path(WASI_SDK_PARENT - name wasi-sdk - PATHS ${WAMR_ROOT}/test-tools/ - NO_DEFAULT_PATH - NO_CMAKE_FIND_ROOT_PATH -) - -if(NOT WASI_SDK_PARENT) - message(FATAL_ERROR - "can not find 'wasi-sdk' under ${WAMR_ROOT}/test-tools, " - "please run ${WAMR_ROOT}/test-tools/build-wasi-sdk/build_wasi_sdk.py " - "to build wasi-sdk and try again" - ) -endif() - -set(WASI_SDK_HOME ${WASI_SDK_PARENT}/wasi-sdk) -message(CHECK_START "Detecting WASI-SDK at ${WASI_SDK_HOME}") -if(EXISTS "${WASI_SDK_HOME}/share/cmake/wasi-sdk.cmake") - message(CHECK_PASS "found") -else() - message(CHECK_FAIL "not found") -endif() - -################ BINARYEN ################ -find_program(WASM_OPT - NAMES wasm-opt - PATHS /opt/binaryen-version_101/bin /opt/binaryen/bin - NO_DEFAULT_PATH - NO_CMAKE_FIND_ROOT_PATH -) - -if(NOT WASM_OPT) - message(FATAL_ERROR - "can not find wasm-opt. " - "please download it from " - "https://github.com/WebAssembly/binaryen/releases/download/version_101/binaryen-version_101-x86_64-linux.tar.gz " - "and install it under /opt" - ) -endif() diff --git a/samples/workload/docker/build_workload.sh b/samples/workload/docker/build_workload.sh deleted file mode 100755 index 640cca97..00000000 --- a/samples/workload/docker/build_workload.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# - -readonly SCRIPT_PATH=$(dirname "$(realpath "$0")") -readonly ROOT=$(realpath "${SCRIPT_PATH}"/../../../) -readonly CURRENT_PATH=$(pwd) -readonly CURRENT_RELATIVE_ROOT=$(realpath --relative-base ${ROOT} ${CURRENT_PATH}) -readonly VARIANT=$(lsb_release -c | awk '{print $2}') - -docker build \ - --build-arg VARIANT=${VARIANT} \ - --memory 4G --cpu-quota 50000 \ - -t wamr_dev_${VARIANT}:0.1 -f "${ROOT}"/.devcontainer/Dockerfile "${ROOT}"/.devcontainer && - docker run --rm -it \ - --memory 4G \ - --cpus ".5" \ - --name workload_build_env \ - --mount type=bind,source="${ROOT}",target=/workspace \ - wamr_dev_${VARIANT}:0.1 \ - /bin/bash -c "\ - pwd \ - && pushd ${CURRENT_RELATIVE_ROOT} \ - && rm -rf build \ - && mkdir build \ - && pushd build \ - && cmake .. \ - && cmake --build . --config Release \ - && popd \ - && popd \ - && echo 'Go and find out results under ${CURRENT_RELATIVE_ROOT}/build' " diff --git a/samples/workload/include/.gitkeep b/samples/workload/include/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/samples/workload/meshoptimizer/CMakeLists.txt b/samples/workload/meshoptimizer/CMakeLists.txt index 172de8d6..cb490f91 100644 --- a/samples/workload/meshoptimizer/CMakeLists.txt +++ b/samples/workload/meshoptimizer/CMakeLists.txt @@ -1,11 +1,19 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required (VERSION 3.0) +cmake_minimum_required (VERSION 3.14) project(bench-meshoptimizer) -include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/preparation.cmake) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) + +################ dependencies ################ +find_package(Python3 REQUIRED) +find_package(WASISDK 16.0 REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) ################ MESHOPTIMIZER ################ include(ExternalProject) @@ -13,7 +21,7 @@ include(ExternalProject) ExternalProject_Add(codecbench PREFIX codecbench GIT_REPOSITORY https://github.com/zeux/meshoptimizer.git - GIT_TAG master + GIT_TAG f926b288264522e1b331a41b07ba40167f396913 GIT_SHALLOW ON GIT_PROGRESS ON SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/meshoptimizer @@ -21,10 +29,10 @@ ExternalProject_Add(codecbench && ${CMAKE_COMMAND} -E echo "Applying patch" && git apply ${CMAKE_CURRENT_SOURCE_DIR}/codecbench.patch CONFIGURE_COMMAND ${CMAKE_COMMAND} - -DWASI_SDK_PREFIX=${WASI_SDK_HOME} - -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_HOME}/share/cmake/wasi-sdk.cmake - -DCMAKE_SYSROOT=${WASI_SDK_HOME}/share/wasi-sysroot + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + -DCMAKE_SYSROOT=${WASISDK_SYSROOT} ${CMAKE_CURRENT_SOURCE_DIR}/meshoptimizer BUILD_COMMAND make codecbench - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ./codecbench.wasm ${CMAKE_BINARY_DIR}/codecbench.wasm + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different ./codecbench.wasm ${CMAKE_CURRENT_BINARY_DIR}/codecbench.wasm ) diff --git a/samples/workload/meshoptimizer/README.md b/samples/workload/meshoptimizer/README.md index ceefb572..466cd875 100644 --- a/samples/workload/meshoptimizer/README.md +++ b/samples/workload/meshoptimizer/README.md @@ -44,14 +44,14 @@ Firstly please build iwasm with simd support: ``` shell $ cd /product-mini/platforms/linux/ $ mkdir build && cd build -$ cmake .. -DWAMR_BUILD_SIMD=1 +$ cmake .. $ make ``` Then compile wasm file to aot file and run: ``` shell -$ /wamr-compiler/build/wamrc --enable-simd -o codecbench.aot codecbench.wasm +$ /wamr-compiler/build/wamrc -o codecbench.aot codecbench.wasm $ /product-mini/platforms/linux/build/iwasm codecbench.aot ``` diff --git a/samples/workload/meshoptimizer/build_workload.sh b/samples/workload/meshoptimizer/build_workload.sh deleted file mode 120000 index a31afa92..00000000 --- a/samples/workload/meshoptimizer/build_workload.sh +++ /dev/null @@ -1 +0,0 @@ -../docker/build_workload.sh \ No newline at end of file diff --git a/samples/workload/preparation.sh b/samples/workload/preparation.sh index 3f49f0cf..47b11ac5 100755 --- a/samples/workload/preparation.sh +++ b/samples/workload/preparation.sh @@ -5,13 +5,13 @@ # readonly BUILD_CONTENT="/tmp/build_content" -readonly WABT_VER=1.0.23 +readonly WABT_VER=1.0.31 readonly WABT_FILE="wabt-${WABT_VER}-ubuntu.tar.gz" -readonly CMAKE_VER=3.16.2 +readonly CMAKE_VER=3.25.1 readonly CMAKE_FILE="cmake-${CMAKE_VER}-Linux-x86_64.sh" -readonly BINARYEN_VER=version_101 +readonly BINARYEN_VER=version_111 readonly BINARYEN_FILE="binaryen-${BINARYEN_VER}-x86_64-linux.tar.gz" -readonly BAZEL_VER=3.7.0 +readonly BAZEL_VER=6.0.0 readonly BAZEL_FILE=bazel-${BAZEL_VER}-installer-linux-x86_64.sh function DEBUG() { @@ -57,8 +57,8 @@ function install_emsdk() { git clone https://github.com/emscripten-core/emsdk.git cd emsdk git pull - ./emsdk install 2.0.26 - ./emsdk activate 2.0.26 + ./emsdk install 3.1.28 + ./emsdk activate 3.1.28 echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc } diff --git a/samples/workload/tensorflow/build.sh b/samples/workload/tensorflow/build.sh index 7289e617..d997113f 100755 --- a/samples/workload/tensorflow/build.sh +++ b/samples/workload/tensorflow/build.sh @@ -98,11 +98,11 @@ make WAMRC_CMD="$(pwd)/wamrc" cd ${OUT_DIR} if [[ $1 == '--sgx' ]]; then - ${WAMRC_CMD} --enable-simd -sgx -o benchmark_model.aot benchmark_model.wasm + ${WAMRC_CMD} -sgx -o benchmark_model.aot benchmark_model.wasm elif [[ $1 == '--threads' ]]; then - ${WAMRC_CMD} --enable-simd --enable-multi-thread -o benchmark_model.aot benchmark_model.wasm + ${WAMRC_CMD} --enable-multi-thread -o benchmark_model.aot benchmark_model.wasm else - ${WAMRC_CMD} --enable-simd -o benchmark_model.aot benchmark_model.wasm + ${WAMRC_CMD} -o benchmark_model.aot benchmark_model.wasm fi # 4. build iwasm with pthread and libc_emcc enable @@ -112,14 +112,14 @@ fi if [[ $1 == '--sgx' ]]; then cd ${WAMR_PLATFORM_DIR}/linux-sgx rm -fr build && mkdir build - cd build && cmake .. -DWAMR_BUILD_SIMD=1 -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1 + cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1 make cd ../enclave-sample make else cd ${WAMR_PLATFORM_DIR}/linux rm -fr build && mkdir build - cd build && cmake .. -DWAMR_BUILD_SIMD=1 -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1 + cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1 make fi diff --git a/samples/workload/wasm-av1/CMakeLists.avx_wasm.txt b/samples/workload/wasm-av1/CMakeLists.avx_wasm.txt index d1149612..409665b3 100644 --- a/samples/workload/wasm-av1/CMakeLists.avx_wasm.txt +++ b/samples/workload/wasm-av1/CMakeLists.avx_wasm.txt @@ -1,14 +1,14 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required (VERSION 2.8...3.16) +cmake_minimum_required (VERSION 3.14) project(testavx) -include(${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/preparation.cmake) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake) -# a workaround to let aom find our non-public headers -include_directories(${WASI_SDK_HOME}/share/wasi-sysroot/include/libc/musl) +################ dependencies ################ +find_package(Binaryen 111 REQUIRED) ################ AOM ################ set(ENABLE_CCACHE ON) @@ -62,7 +62,7 @@ add_dependencies(${PROJECT_NAME} aom) add_custom_target(${PROJECT_NAME}_opt ALL COMMAND - ${WASM_OPT} -Oz --enable-simd -o ${PROJECT_NAME}.opt.wasm ${PROJECT_NAME}.wasm + ${Binaryen_WASM_OPT} -Oz --enable-simd -o ${PROJECT_NAME}.opt.wasm ${PROJECT_NAME}.wasm BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.opt.wasm WORKING_DIRECTORY diff --git a/samples/workload/wasm-av1/CMakeLists.txt b/samples/workload/wasm-av1/CMakeLists.txt index 8b1f0df0..6fea31ea 100644 --- a/samples/workload/wasm-av1/CMakeLists.txt +++ b/samples/workload/wasm-av1/CMakeLists.txt @@ -1,11 +1,19 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required (VERSION 2.8...3.16) +cmake_minimum_required (VERSION 3.14) project(av1_wasm) -include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/preparation.cmake) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) + +################ dependencies ################ +find_package(Python3 REQUIRED) +find_package(WASISDK 16.0 REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) ####################################### include(ExternalProject) @@ -23,10 +31,14 @@ ExternalProject_Add(av1 && ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.avx_wasm.txt CMakeLists.txt && git apply ../av1-clang.patch CONFIGURE_COMMAND ${CMAKE_COMMAND} - -DWASI_SDK_PREFIX=${WASI_SDK_HOME} - -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_HOME}/share/cmake/wasi-sdk.cmake - -DCMAKE_SYSROOT=${WASI_SDK_HOME}/share/wasi-sysroot + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + -DCMAKE_SYSROOT=${WASISDK_SYSROOT} + -DCMAKE_C_FLAGS=-isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/sse\ -isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/libc/musl ${CMAKE_CURRENT_SOURCE_DIR}/av1 BUILD_COMMAND make testavx_opt - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy testavx.opt.wasm ${CMAKE_CURRENT_BINARY_DIR}/testavx.wasm + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different + testavx.opt.wasm + ${CMAKE_CURRENT_SOURCE_DIR}/av1/third_party/samples/elephants_dream_480p24.ivf + ${CMAKE_CURRENT_BINARY_DIR} ) diff --git a/samples/workload/wasm-av1/README.md b/samples/workload/wasm-av1/README.md index e7a92f2a..2166fe6a 100644 --- a/samples/workload/wasm-av1/README.md +++ b/samples/workload/wasm-av1/README.md @@ -39,7 +39,7 @@ Firstly please build iwasm with simd support: ``` shell $ cd /product-mini/platforms/linux/ $ mkdir build && cd build -$ cmake .. -DWAMR_BUILD_SIMD=1 -DWAMR_BUILD_LIBC_EMCC=1 +$ cmake .. -DWAMR_BUILD_LIBC_EMCC=1 $ make ``` @@ -47,7 +47,7 @@ Then compile wasm file to aot file and run: ``` shell $ cd -$ /wamr-compiler/build/wamrc --enable-simd -o testavx.aot testavx.wasm +$ /wamr-compiler/build/wamrc -o testavx.aot testavx.wasm # copy sample data like /samples/workload/wasm-av1/av1/third_party/samples/elephants_dream_480p24.ivf # make sure you declare the access priority of the directory in which the sample data is $ /product-mini/platforms/linux/build/iwasm --dir=. testavx.aot elephants_dream_480p24.ivf diff --git a/samples/workload/wasm-av1/build.sh b/samples/workload/wasm-av1/build.sh index 7f82c6c5..efa17eca 100755 --- a/samples/workload/wasm-av1/build.sh +++ b/samples/workload/wasm-av1/build.sh @@ -85,12 +85,12 @@ cd build && cmake .. make # 3.2 compile wasm-av1.wasm to wasm-av1.aot cd ${OUT_DIR} -${WAMRC_CMD} --enable-simd -o testavx.aot testavx.wasm +${WAMRC_CMD} -o testavx.aot testavx.wasm # 4. build iwasm with pthread and libc_emcc enable cd ${WAMR_PLATFORM_DIR}/linux rm -fr build && mkdir build -cd build && cmake .. -DWAMR_BUILD_SIMD=1 -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1 +cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1 make # 5. run wasm-av1 with iwasm diff --git a/samples/workload/wasm-av1/build_workload.sh b/samples/workload/wasm-av1/build_workload.sh deleted file mode 120000 index a31afa92..00000000 --- a/samples/workload/wasm-av1/build_workload.sh +++ /dev/null @@ -1 +0,0 @@ -../docker/build_workload.sh \ No newline at end of file diff --git a/test-tools/build-wasi-sdk/patches/wasi_libc.patch b/test-tools/build-wasi-sdk/patches/wasi_libc.patch deleted file mode 100644 index e236735b..00000000 --- a/test-tools/build-wasi-sdk/patches/wasi_libc.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/expected/wasm32-wasi/predefined-macros.txt b/expected/wasm32-wasi/predefined-macros.txt -index c1bb19e..954f3b5 100644 ---- a/expected/wasm32-wasi/predefined-macros.txt -+++ b/expected/wasm32-wasi/predefined-macros.txt -@@ -3002,6 +3002,8 @@ - #define __alignof_is_defined 1 - #define __bitop(x,i,o) ((x)[(i)/8] o (1<<(i)%8)) - #define __bool_true_false_are_defined 1 -+#define __clang_literal_encoding__ "UTF-8" -+#define __clang_wide_literal_encoding__ "UTF-32" - #define __inline inline - #define __restrict restrict - #define __tg_complex(fun,x) (__RETCAST_CX(x)( __FLTCX((x)+I) && __IS_FP(x) ? fun ## f (x) : __LDBLCX((x)+I) ? fun ## l (x) : fun(x) )) diff --git a/test-tools/build-wasi-sdk/patches/wasi_sdk.patch b/test-tools/build-wasi-sdk/patches/wasi_sdk.patch deleted file mode 100644 index 0fc4caee..00000000 --- a/test-tools/build-wasi-sdk/patches/wasi_sdk.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/version.sh b/version.sh -index 8e7c44c..ff0d3ba 100755 ---- a/version.sh -+++ b/version.sh -@@ -1,5 +1,6 @@ - #!/usr/bin/env bash --set -e --GIT_DESCR=$(git describe --long --candidates=999 --match='wasi-sdk-*' --dirty='+m' --abbrev=12) --GIT_PACKAGE_VERSION=$(echo $GIT_DESCR | perl -ne 'if(/^wasi-sdk-(\d+)-(\d+)-g([0-9a-f]{7,12})([+]m)?$/) { if($2 == 0) { print "$1.$2$4" } else { print "$1.$2g$3$4" } exit } else { print "could not parse git description"; exit 1 }';) --echo $GIT_PACKAGE_VERSION -+#set -e -+#GIT_DESCR=$(git describe --long --candidates=999 --match='wasi-sdk-*' --dirty='+m' --abbrev=12) -+#GIT_PACKAGE_VERSION=$(echo $GIT_DESCR | perl -ne 'if(/^wasi-sdk-(\d+)-(\d+)-g([0-9a-f]{7,12})([+]m)?$/) { if($2 == 0) { print "$1.$2$4" } else { print "$1.$2g$3$4" } exit } else { print "could not parse git description"; exit 1 }';) -+#echo $GIT_PACKAGE_VERSION -+echo wasi-sdk-13-eng diff --git a/test-tools/build-wasi-sdk/build_wasi_sdk.py b/test-tools/pick-up-emscripten-headers/collect_files.py similarity index 50% rename from test-tools/build-wasi-sdk/build_wasi_sdk.py rename to test-tools/pick-up-emscripten-headers/collect_files.py index a87cdef6..7e832145 100755 --- a/test-tools/build-wasi-sdk/build_wasi_sdk.py +++ b/test-tools/pick-up-emscripten-headers/collect_files.py @@ -9,68 +9,35 @@ The script operates on such directories and files |-- core | `-- deps | |-- emscripten -| `-- wasi-sdk -| `-- src -| |-- llvm-project -| `-- wasi-libc +|-- samples +| `-- workloads +| |-- include `-- test-tools - |-- build-wasi-sdk - | |-- build_wasi_sdk.py - | |-- include - | `-- patches - `-- wasi-sdk - |-- bin - |-- lib - `-- share - `-- wasi-sysroot + |-- pick-up-emscripten_headers + | |-- collect_files.py """ +import argparse import hashlib import logging import os import pathlib -import shlex import shutil -import subprocess import sys import tarfile import tempfile import urllib import urllib.request -logger = logging.getLogger("build_wasi_sdk") +logger = logging.getLogger("pick-up-emscripten-headers") external_repos = { - "config": { - "sha256": "302e5e7f3c4996976c58efde8b2f28f71d51357e784330eeed738e129300dc33", - "store_dir": "core/deps/wasi-sdk/src/config", - "strip_prefix": "config-191bcb948f7191c36eefe634336f5fc5c0c4c2be", - "url": "https://git.savannah.gnu.org/cgit/config.git/snapshot/config-191bcb948f7191c36eefe634336f5fc5c0c4c2be.tar.gz", - }, "emscripten": { - "sha256": "0904a65379aea3ea94087b8c12985b2fee48599b473e3bef914fec2e3941532d", + "sha256": "c5524755b785d8f4b83eb3214fdd3ac4b2e1b1a4644df4c63f06e5968f48f90e", "store_dir": "core/deps/emscripten", - "strip_prefix": "emscripten-2.0.28", - "url": "https://github.com/emscripten-core/emscripten/archive/refs/tags/2.0.28.tar.gz", - }, - "llvm-project": { - "sha256": "dc5169e51919f2817d06615285e9da6a804f0f881dc55d6247baa25aed3cc143", - "store_dir": "core/deps/wasi-sdk/src/llvm-project", - "strip_prefix": "llvm-project-34ff6a75f58377f32a5046a29f55c4c0e58bee9e", - "url": "https://github.com/llvm/llvm-project/archive/34ff6a75f58377f32a5046a29f55c4c0e58bee9e.tar.gz", - }, - "wasi-sdk": { - "sha256": "fc4fdb0e97b915241f32209492a7d0fab42c24216f87c1d5d75f46f7c70a553d", - "store_dir": "core/deps/wasi-sdk", - "strip_prefix": "wasi-sdk-1a953299860bbcc198ad8c12a21d1b2e2f738355", - "url": "https://github.com/WebAssembly/wasi-sdk/archive/1a953299860bbcc198ad8c12a21d1b2e2f738355.tar.gz", - }, - "wasi-libc": { - "sha256": "f6316ca9479d3463eb1c4f6a1d1f659bf15f67cb3c1e2e83d9d11f188dccd864", - "store_dir": "core/deps/wasi-sdk/src/wasi-libc", - "strip_prefix": "wasi-libc-a78cd329aec717f149934d7362f57050c9401f60", - "url": "https://github.com/WebAssembly/wasi-libc/archive/a78cd329aec717f149934d7362f57050c9401f60.tar.gz", - }, + "strip_prefix": "emscripten-3.0.0", + "url": "https://github.com/emscripten-core/emscripten/archive/refs/tags/3.0.0.tar.gz", + } } # TOOD: can we use headers from wasi-libc and clang directly ? @@ -111,25 +78,25 @@ def unpack(tar_file, strip_prefix, dest_dir): with tempfile.TemporaryDirectory() as tmp: with tarfile.open(tar_file) as tar: logger.debug(f"extract to {tmp}") + def is_within_directory(directory, target): - + abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) - + prefix = os.path.commonprefix([abs_directory, abs_target]) - + return prefix == abs_directory - + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): - + for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): raise Exception("Attempted Path Traversal in Tar File") - - tar.extractall(path, members, numeric_owner=numeric_owner) - - + + tar.extractall(path, members, numeric_owner=numeric_owner) + safe_extract(tar, tmp) strip_prefix_dir = ( @@ -160,12 +127,12 @@ def download_repo(name, root): download_flag = store_dir.joinpath("DOWNLOADED") if store_dir.exists() and download_flag.exists(): logger.info( - f"keep using '{store_dir.relative_to(root)}'. Or to remove it and try again" + f"bypass downloading '{store_dir.relative_to(root)}'. Or to remove it and try again if needs a new release" ) return True # download only when the target is neither existed nor broken - download_dir = pathlib.Path("/tmp/build_wasi_sdk/") + download_dir = pathlib.Path("/tmp/pick-up-emscripten-headers/") download_dir.mkdir(exist_ok=True) tar_name = pathlib.Path(external_repos[name]["url"]).name @@ -192,104 +159,67 @@ def download_repo(name, root): download_flag.touch() # leave download files in /tmp + logger.info(f"Has downloaed and stored in {store_dir.relative_to(root)}") return True -def run_patch(patch_file, cwd): - if not patch_file.exists(): - logger.error(f"{patch_file} not found") +def collect_headers(root, install_location): + if not install_location.exists(): + logger.error(f"{install_location} does not found") return False - with open(patch_file, "r") as f: - try: - PATCH_DRY_RUN_CMD = "patch -f -p1 --dry-run" - if subprocess.check_call(shlex.split(PATCH_DRY_RUN_CMD), stdin=f, cwd=cwd): - logger.error(f"patch dry-run {cwd} failed") - return False + install_flag = install_location.joinpath("INSTALLED").resolve() + if install_flag.exists(): + logger.info( + f"bypass downloading '{install_location}'. Or to remove it and try again if needs a new one" + ) + return True - PATCH_CMD = "patch -f -p1" - f.seek(0) - if subprocess.check_call(shlex.split(PATCH_CMD), stdin=f, cwd=cwd): - logger.error(f"patch {cwd} failed") - return False - except subprocess.CalledProcessError: - logger.error(f"patch {cwd} failed") - return False - return True - - -def build_and_install_wasi_sdk(root): - store_dir = root.joinpath(f'{external_repos["wasi-sdk"]["store_dir"]}').resolve() - if not store_dir.exists(): - logger.error(f"{store_dir} does not found") + emscripten_home = root.joinpath( + f'{external_repos["emscripten"]["store_dir"]}' + ).resolve() + if not emscripten_home.exists(): + logger.error(f"{emscripten_home} does not found") return False - # patch wasi-libc and wasi-sdk - patch_flag = store_dir.joinpath("PATCHED") - if not patch_flag.exists(): - if not run_patch( - root.joinpath("test-tools/build-wasi-sdk/patches/wasi_libc.patch"), - store_dir.joinpath("src/wasi-libc"), - ): - return False - - if not run_patch( - root.joinpath("test-tools/build-wasi-sdk/patches/wasi_sdk.patch"), store_dir - ): - return False - - patch_flag.touch() - else: - logger.info("bypass the patch phase") - - # build - build_flag = store_dir.joinpath("BUILDED") - if not build_flag.exists(): - BUILD_CMD = "make build" - if subprocess.check_call(shlex.split(BUILD_CMD), cwd=store_dir): - logger.error(f"build wasi-sdk failed") - return False - - build_flag.touch() - else: - logger.info("bypass the build phase") - - # install - install_flag = store_dir.joinpath("INSTALLED") - binary_path = root.joinpath("test-tools").resolve() - if not install_flag.exists(): - shutil.copytree( - str(store_dir.joinpath("build/install/opt").resolve()), - str(binary_path), - dirs_exist_ok=True, - ) - - # install headers - emscripten_headers = ( - root.joinpath(external_repos["emscripten"]["store_dir"]) - .joinpath("system") - .resolve() - ) - wasi_sysroot_headers = binary_path.joinpath( - "wasi-sdk/share/wasi-sysroot/include" - ).resolve() - for (src, dst) in emscripten_headers_src_dst: - src = emscripten_headers.joinpath(src) - dst = wasi_sysroot_headers.joinpath(dst) - dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(src, dst) - - install_flag.touch() - else: - logger.info("bypass the install phase") + emscripten_headers = emscripten_home.joinpath("system").resolve() + for (src, dst) in emscripten_headers_src_dst: + src = emscripten_headers.joinpath(src) + dst = install_location.joinpath(dst) + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(src, dst) + install_flag.touch() + logger.info(f"Has installed in {install_location}") return True def main(): + parser = argparse.ArgumentParser( + description="collect headers from emscripten for workload compilation" + ) + parser.add_argument( + "--install", + type=str, + required=True, + help="identify installation location", + ) + parser.add_argument( + "--loglevel", + type=str, + default="INFO", + choices=[ + "ERROR", + "WARNING", + "INFO", + ], + help="the logging level", + ) + options = parser.parse_args() + console = logging.StreamHandler() console.setFormatter(logging.Formatter("%(asctime)s - %(message)s")) - logger.setLevel(logging.INFO) + logger.setLevel(getattr(logging, options.loglevel)) logger.addHandler(console) logger.propagate = False @@ -305,12 +235,9 @@ def main(): if not download_repo(repo, root): return False - # build wasi_sdk and install - if not build_and_install_wasi_sdk(root): + if not collect_headers(root, pathlib.Path(options.install)): return False - # TODO install headers from emscripten - return True diff --git a/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile b/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile index f96a2261..6ca21094 100644 --- a/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile +++ b/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile @@ -25,17 +25,17 @@ RUN wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/ # - wamr-sdk ## - download wasi-sdk with wget and set up to /opt/wasi-sdk -RUN wget --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-14/wasi-sdk-14.0-linux.tar.gz \ - && tar -zxvf wasi-sdk-14.0-linux.tar.gz \ - && mv wasi-sdk-14.0 /opt/wasi-sdk/ \ - && rm -f wasi-sdk-14.0-linux.tar.gz +RUN wget --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz \ + && tar -zxvf wasi-sdk-*-linux.tar.gz \ + && mv wasi-sdk-19.0 /opt/wasi-sdk/ \ + && rm -f wasi-sdk-*-linux.tar.gz ## - clone wamr repo -RUN git clone -b main --depth=1 https://github.com/bytecodealliance/wasm-micro-runtime.git +RUN git clone -b main --depth=1 https://github.com/bytecodealliance/wasm-micro-runtime.git WORKDIR /root/wasm-micro-runtime/wamr-compiler RUN ./build_llvm.sh \ - && mkdir build + && mkdir build WORKDIR /root/wasm-micro-runtime/wamr-compiler/build RUN cmake .. \ diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index 00797b31..bc6df908 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -360,16 +360,16 @@ function spec_test() exit 1 ;; esac - if [ ! -f /tmp/wabt-1.0.29-${WABT_PLATFORM}.tar.gz ]; then + if [ ! -f /tmp/wabt-1.0.31-${WABT_PLATFORM}.tar.gz ]; then wget \ - https://github.com/WebAssembly/wabt/releases/download/1.0.29/wabt-1.0.29-${WABT_PLATFORM}.tar.gz \ + https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-${WABT_PLATFORM}.tar.gz \ -P /tmp fi cd /tmp \ - && tar zxf wabt-1.0.29-${WABT_PLATFORM}.tar.gz \ + && tar zxf wabt-1.0.31-${WABT_PLATFORM}.tar.gz \ && mkdir -p ${WORK_DIR}/wabt/out/gcc/Release/ \ - && install wabt-1.0.29/bin/wa* ${WORK_DIR}/wabt/out/gcc/Release/ \ + && install wabt-1.0.31/bin/wa* ${WORK_DIR}/wabt/out/gcc/Release/ \ && cd - fi else @@ -429,7 +429,7 @@ function spec_test() fi # set the current running target - ARGS_FOR_SPEC_TEST+="-m ${TARGET} " + ARGS_FOR_SPEC_TEST+="-m ${TARGET} " # require warmc only in aot mode if [[ $1 == 'aot' ]]; then