CloudEdge/wasm-micro-runtime
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Upgrade toolkits (#1878)

Browse Source 
Upgrade the version of related toolkits:
- upgrade llvm to 15.0
- upgrade wasi-sdk to 19.0
- upgrade emsdk to 3.1.28
- upgrade wabt to 1.0.31
- upgrade binaryen to 111

And upgrade the CI scripts, sample workload build scripts, Dockerfiles, and documents.
This commit is contained in:
Wenyong Huang 2023-02-02 09:42:25 +08:00 committed by GitHub
parent 1614ce12fa
commit 27e7e160af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
43 changed files with 817 additions and 594 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
.devcontainer/Dockerfile
View File

@ -12,7 +12,7 @@ ENV TZ=Asian/Shanghai
# hadolint ignore=DL3008
RUN apt-get update \
&& apt-get install -y apt-transport-https apt-utils build-essential \
ca-certificates curl g++-multilib git gnupg \
ca-certificates ccache curl g++-multilib git gnupg \
libgcc-9-dev lib32gcc-9-dev lsb-release \
ninja-build ocaml ocamlbuild python2.7 \
software-properties-common tree tzdata \
@ -20,6 +20,15 @@ RUN apt-get update \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/*
#
# binaryen
ARG BINARYEN_VER=111
WORKDIR /opt
RUN wget -c --progress=dot:giga https://github.com/WebAssembly/binaryen/releases/download/version_${BINARYEN_VER}/binaryen-version_${BINARYEN_VER}-x86_64-linux.tar.gz \
&& tar xf binaryen-version_${BINARYEN_VER}-x86_64-linux.tar.gz \
&& ln -sf /opt/binaryen-version_111 /opt/binaryen \
&& rm binaryen-version_${BINARYEN_VER}-x86_64-linux.tar.gz
#
# CMAKE (https://apt.kitware.com/)
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@ -38,18 +47,19 @@ RUN wget --progress=dot:giga -O - https://apt.kitware.com/keys/kitware-archive-l
WORKDIR /opt
RUN git clone https://github.com/emscripten-core/emsdk.git
ARG EMSDK_VER=3.0.0
WORKDIR /opt/emsdk
RUN git pull \
&& ./emsdk install 2.0.26 \
&& ./emsdk activate 2.0.26 \
&& ./emsdk install ${EMSDK_VER} \
&& ./emsdk activate ${EMSDK_VER} \
&& echo "source /opt/emsdk/emsdk_env.sh" >> /root/.bashrc
#
# install wasi-sdk
ARG WASI_SDK_VER=16
ARG WASI_SDK_VER=19
RUN wget -c --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VER}/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz -P /opt \
&& tar xf /opt/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz -C /opt \
&& ln -fs /opt/wasi-sdk-${WASI_SDK_VER}.0 /opt/wasi-sdk \
&& ln -sf /opt/wasi-sdk-${WASI_SDK_VER}.0 /opt/wasi-sdk \
&& rm /opt/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz
#
@ -57,7 +67,7 @@ RUN wget -c --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases
ARG WABT_VER=1.0.29
RUN wget -c --progress=dot:giga https://github.com/WebAssembly/wabt/releases/download/${WABT_VER}/wabt-${WABT_VER}-ubuntu.tar.gz -P /opt \
&& tar xf /opt/wabt-${WABT_VER}-ubuntu.tar.gz -C /opt \
&& ln -fs /opt/wabt-${WABT_VER} /opt/wabt \
&& ln -sf /opt/wabt-${WABT_VER} /opt/wabt \
&& rm /opt/wabt-${WABT_VER}-ubuntu.tar.gz
#
@ -70,6 +80,8 @@ RUN mkdir /opt/bazelisk \
#
# install clang+llvm
ARG LLVM_VER=14
RUN apt-get purge -y clang-10 llvm-10 && apt autoremove -y
WORKDIR /etc/apt/apt.conf.d
RUN touch 99verfiy-peer.conf \
&& echo "Acquire { https::Verify-Peer false }" > 99verfiy-peer.conf
@ -77,9 +89,7 @@ RUN touch 99verfiy-peer.conf \
WORKDIR /tmp
RUN wget --progress=dot:giga https://apt.llvm.org/llvm.sh \
&& chmod a+x ./llvm.sh \
&& /tmp/llvm.sh 12 all \
&& ln -sf /usr/bin/clang-format-12 /usr/bin/clang-format \
&& rm -rf /tmp/*
&& ./llvm.sh ${LLVM_VER} all
#
# [Optional]
@ -96,17 +106,28 @@ RUN apt-get update \
# Install required python packages
# hadolint ignore=DL3013
RUN python3 -m pip install --no-cache-dir --upgrade pip \
&& pip3 install --no-cache-dir --user black nose pycparser pylint
&& pip3 install --no-cache-dir black nose pycparser pylint
# set path, PS and clean up
ENV PATH "/opt/bazelisk:/opt/clang-llvm/bin:${PATH}"
RUN echo "export PATH=/opt/bazelisk:/opt/clang-llvm/bin:${PATH}" >> /root/.bashrc \
&& printf "%s\n" "PS1='\n[ \u@wamr-dev-docker \W ]\n$ '" >> /root/.bashrc \
#
# Install github-cli. It doens't work as a feature of devcontainer.json
RUN cd /tmp \
&& wget https://github.com/cli/cli/releases/download/v2.20.2/gh_2.20.2_linux_amd64.deb \
&& dpkg -i gh_2.20.2_linux_amd64.deb
#
# Install NodeJS
RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash -
RUN apt-get install -y nodejs
# set path
ENV PATH="/opt/bazelisk:/usr/lib/llvm-${LLVM_VER}/bin:${PATH}"
ENV CC=/usr/lib/llvm-${LLVM_VER}/bin/clang CXX=/usr/lib/llvm-${LLVM_VER}/bin/clang++
RUN printf "%s\n" "PS1='\n[ \u@wamr-dev-docker \W ]\n$ '" >> /root/.bashrc \
&& apt-get autoremove -y \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /tmp/*
# set workdir when container run
VOLUME /workspace
WORKDIR /workspace
VOLUME /workspaces
WORKDIR /workspaces

12
.devcontainer/devcontainer.json
View File

@ -1,6 +1,5 @@
// Copyright (C) 2019 Intel Corporation. All rights reserved.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at:
// https://github.com/microsoft/vscode-dev-containers/tree/v0.195.0/containers/cpp
{
@ -10,7 +9,12 @@
// Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
// Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon
"args": {
"VARIANT": "ubuntu-20.04"
"BINARYEN_VER": "111",
"EMSDK_VER": "3.0.0",
"LLVM_VER": "15",
"VARIANT": "ubuntu-20.04",
"WASI_SDK_VER": "19",
"WABT_VER": "1.0.31"
}
},
"runArgs": [
@ -27,12 +31,10 @@
// Add the IDs of extensions you want installed when the container is created.
"extensions": [
"dtsvet.vscode-wasm",
"esbenp.prettier-vscode",
"llvm-vs-code-extensions.vscode-clangd",
"ms-python.python",
"ms-python.vscode-pylance",
"ms-vscode.cmake-tools",
"ms-vscode.cpptools",
"twxs.cmake"
]
}
},

70
.github/workflows/build_llvm_libraries.yml vendored
View File

@ -5,23 +5,42 @@ name: Reusable workflow-build_llvm_libraries
on:
workflow_call:
inputs:
runs-on:
os:
required: true
type: string
arch:
required: true
type: string
outputs:
cache_key:
description: "A cached key of LLVM libraries"
value: ${{ jobs.build_llvm_libraries.outputs.key}}
jobs:
build_llvm_libraries:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: ${{ fromJson(inputs.runs-on) }}
runs-on: ${{ inputs.os }}
outputs:
key: ${{ steps.create_lib_cache_key.outputs.key}}
steps:
- name: checkout
uses: actions/checkout@v3
- name: retrive the last commit ID
id: get_last_commit
run: echo "last_commit=$(/usr/bin/env python3 ./build_llvm.py --llvm-ver)" >> $GITHUB_OUTPUT
working-directory: build-scripts
# Bump the prefix number to evict all previous caches and
# enforce a clean build, in the unlikely case that some
# weird build error occur and llvm/build becomes a potential
# suspect.
- name: form the cache key of libraries
id: create_lib_cache_key
run: echo "key=0-llvm-libraries-${{ inputs.os }}-${{ inputs.arch }}-${{ steps.get_last_commit.outputs.last_commit }}" >> $GITHUB_OUTPUT
- name: Cache LLVM libraries
id: cache_llvm
id: retrieve_llvm_libs
uses: actions/cache@v3
with:
path: |
@ -30,10 +49,39 @@ jobs:
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-build-llvm_libraries_ex
key: ${{ steps.create_lib_cache_key.outputs.key}}
- name: Build llvm
id: build_llvm
if: ${{ steps.cache_llvm.outputs.cache-hit != 'true' }}
run: /usr/bin/env python3 ./build_llvm.py --arch X86 WebAssembly
- uses: actions/cache@v3
with:
path: ~/.ccache
key: 0-ccache-${{ inputs.os }}-${{ steps.get_last_commit.outputs.last_commit }}
restore-keys: |
0-ccache-${{ inputs.os }}
if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && inputs.os == 'ubuntu-20.04'
- uses: actions/cache@v3
with:
path: ~/.cache/ccache
key: 0-ccache-${{ inputs.os }}-${{ steps.get_last_commit.outputs.last_commit }}
restore-keys: |
0-ccache-${{ inputs.os }}
if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && inputs.os == 'ubuntu-22.04'
- run: sudo apt install -y ccache ninja-build
if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'ubuntu')
- uses: actions/cache@v3
with:
path: ~/Library/Caches/ccache
key: 0-ccache-${{ inputs.os }}-${{ steps.get_last_commit.outputs.last_commit }}
restore-keys: |
0-ccache-${{ inputs.os }}
if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'macos')
- run: brew install ccache ninja
if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'macos')
- name: Build LLVM libraries
if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
run: /usr/bin/env python3 ./build_llvm.py --arch ${{ inputs.arch }}
working-directory: build-scripts

123
.github/workflows/compilation_on_android_ubuntu.yml vendored
View File

@ -53,8 +53,6 @@ env:
FAST_JIT_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
LLVM_LAZY_JIT_BUILD_OPTIONS: " -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1"
LLVM_EAGER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0"
# LLVM
LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex"
# For Spec Test
DEFAULT_TEST_OPTIONS: "-s spec -b -P"
MULTI_MODULES_TEST_OPTIONS: "-s spec -b -M -P"
@ -64,23 +62,37 @@ env:
WASI_TEST_OPTIONS: "-s wasi_certification"
jobs:
build_llvm_libraries:
build_llvm_libraries_on_ubuntu_2004:
uses: ./.github/workflows/build_llvm_libraries.yml
with:
runs-on: "['ubuntu-20.04', 'ubuntu-22.04']"
os: "ubuntu-20.04"
arch: "X86"
build_llvm_libraries_on_ubuntu_2204:
uses: ./.github/workflows/build_llvm_libraries.yml
with:
os: "ubuntu-22.04"
arch: "X86"
build_wamrc:
needs: [build_llvm_libraries]
needs:
[build_llvm_libraries_on_ubuntu_2004, build_llvm_libraries_on_ubuntu_2204]
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-20.04, ubuntu-22.04]
include:
- os: ubuntu-20.04
llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}
- os: ubuntu-22.04
llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }}
steps:
- name: checkout
uses: actions/checkout@v3
# since jobs.id can't contain the dot character
# it is hard to use `format` to assemble the cache key
- name: Get LLVM libraries
id: cache_llvm
id: retrieve_llvm_libs
uses: actions/cache@v3
with:
path: |
@ -89,10 +101,10 @@ jobs:
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
key: ${{ matrix.llvm_cache_key }}
- name: Quit if cache miss
if: steps.cache_llvm.outputs.cache-hit != 'true'
if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
- name: Build wamrc
@ -103,7 +115,8 @@ jobs:
working-directory: wamr-compiler
build_iwasm:
needs: [build_llvm_libraries]
needs:
[build_llvm_libraries_on_ubuntu_2004, build_llvm_libraries_on_ubuntu_2204]
runs-on: ${{ matrix.os }}
strategy:
matrix:
@ -187,13 +200,21 @@ jobs:
# Fast-JIT mode doesn't support android(X86-32)
- make_options_run_mode: $FAST_JIT_BUILD_OPTIONS
platform: android
# only test andorid on ubuntu latest
- os: ubuntu-20.04
platform: android
include:
- os: ubuntu-20.04
llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}
- os: ubuntu-22.04
llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }}
steps:
- name: checkout
uses: actions/checkout@v3
# only download llvm cache when needed
- name: Get LLVM libraries
id: cache_llvm
id: retrieve_llvm_libs
if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS')
uses: actions/cache@v3
with:
@ -203,10 +224,10 @@ jobs:
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
key: ${{ matrix.llvm_cache_key }}
- name: Quit if cache miss
if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.cache_llvm.outputs.cache-hit != 'true')
if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true')
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
- name: Build iwasm
@ -217,7 +238,13 @@ jobs:
working-directory: product-mini/platforms/${{ matrix.platform }}
build_samples_wasm_c_api:
needs: [build_iwasm, build_llvm_libraries, build_wamrc]
needs:
[
build_iwasm,
build_llvm_libraries_on_ubuntu_2004,
build_llvm_libraries_on_ubuntu_2204,
build_wamrc,
]
runs-on: ${{ matrix.os }}
strategy:
matrix:
@ -233,18 +260,23 @@ jobs:
os: [ubuntu-20.04, ubuntu-22.04]
wasi_sdk_release:
[
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz",
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz",
]
wabt_release:
[
"https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz",
"https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz",
]
include:
- os: ubuntu-20.04
llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}
- os: ubuntu-22.04
llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }}
steps:
- name: checkout
uses: actions/checkout@v3
- name: Get LLVM libraries
id: cache_llvm
id: retrieve_llvm_libs
if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS'))
uses: actions/cache@v3
with:
@ -254,18 +286,18 @@ jobs:
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
key: ${{ matrix.llvm_cache_key }}
- name: Quit if cache miss
if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) && (steps.cache_llvm.outputs.cache-hit != 'true')
if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS')) && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true')
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
- name: download and install wabt
run: |
cd /opt
sudo wget ${{ matrix.wabt_release }}
sudo tar -xzf wabt-1.0.24-*.tar.gz
sudo mv wabt-1.0.24 wabt
sudo tar -xzf wabt-1.0.31-*.tar.gz
sudo mv wabt-1.0.31 wabt
- name: Build wamrc
if: (!endsWith(matrix.make_options, '_INTERP_BUILD_OPTIONS'))
@ -277,19 +309,9 @@ jobs:
- name: Build Sample [wasm-c-api]
run: |
mkdir build && cd build
cmake .. ${{ matrix.make_options }}
cmake --build . --config Release --parallel 4
./callback
./callback_chain
./empty_imports
./global
./hello
./hostref
./memory
./reflect
./table
./trap
cmake -S . -B build ${{ matrix.make_options }}
cmake --build build --config Release --parallel 4
ctest --test-dir build
working-directory: samples/wasm-c-api
build_samples_others:
@ -298,14 +320,13 @@ jobs:
strategy:
matrix:
os: [ubuntu-20.04, ubuntu-22.04]
wasi_sdk_release:
[
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz",
]
wabt_release:
[
"https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz",
]
include:
- os: ubuntu-20.04
wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz"
wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
- os: ubuntu-22.04
wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz"
wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
steps:
- name: checkout
uses: actions/checkout@v3
@ -314,15 +335,15 @@ jobs:
run: |
cd /opt
sudo wget ${{ matrix.wasi_sdk_release }}
sudo tar -xzf wasi-sdk-12.0-*.tar.gz
sudo mv wasi-sdk-12.0 wasi-sdk
sudo tar -xzf wasi-sdk-*.tar.gz
sudo mv wasi-sdk-19.0 wasi-sdk
- name: download and install wabt
run: |
cd /opt
sudo wget ${{ matrix.wabt_release }}
sudo tar -xzf wabt-1.0.24-*.tar.gz
sudo mv wabt-1.0.24 wabt
sudo tar -xzf wabt-1.0.31-*.tar.gz
sudo mv wabt-1.0.31 wabt
- name: Build Sample [basic]
run: |
@ -378,7 +399,7 @@ jobs:
working-directory: ./samples/simple
test:
needs: [build_iwasm, build_llvm_libraries, build_wamrc]
needs: [build_iwasm, build_llvm_libraries_on_ubuntu_2004, build_wamrc]
runs-on: ubuntu-20.04
strategy:
matrix:
@ -392,6 +413,8 @@ jobs:
$THREADS_TEST_OPTIONS,
$WASI_TEST_OPTIONS,
]
llvm_cache_key:
["${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}"]
exclude:
# uncompatiable modes and features
# classic-interp and fast-interp don't support simd
@ -432,7 +455,7 @@ jobs:
#only download llvm libraries in jit and aot mode
- name: Get LLVM libraries
if: env.USE_LLVM == 'true'
id: cache_llvm
id: retrieve_llvm_libs
uses: actions/cache@v3
with:
path: |
@ -441,10 +464,10 @@ jobs:
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
key: ${{ matrix.llvm_cache_key }}
- name: Quit if cache miss
if: env.USE_LLVM == 'true' && steps.cache_llvm.outputs.cache-hit != 'true'
if: env.USE_LLVM == 'true' && steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
- name: run tests

71
.github/workflows/compilation_on_macos.yml vendored
View File

@ -51,26 +51,28 @@ env:
FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
LLVM_LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1"
LLVM_EAGER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0"
LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex"
jobs:
build_llvm_libraries:
uses: ./.github/workflows/build_llvm_libraries.yml
with:
runs-on: "['macos-latest']"
os: "macos-latest"
arch: "X86"
build_wamrc:
needs: [build_llvm_libraries]
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-latest]
include:
- os: macos-latest
llvm_cache_key: ${{ needs.build_llvm_libraries.outputs.cache_key }}
steps:
- name: checkout
uses: actions/checkout@v3
- name: Get LLVM libraries
id: cache_llvm
id: retrieve_llvm_libs
uses: actions/cache@v3
with:
path: |
@ -79,10 +81,10 @@ jobs:
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
key: ${{ matrix.llvm_cache_key }}
- name: Quit if cache miss
if: steps.cache_llvm.outputs.cache-hit != 'true'
if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
- name: Build wamrc
@ -166,13 +168,16 @@ jobs:
make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1"
- make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS
make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1"
include:
- os: macos-latest
llvm_cache_key: ${{ needs.build_llvm_libraries.outputs.cache_key }}
steps:
- name: checkout
uses: actions/checkout@v3
# only download llvm cache when needed
- name: Get LLVM libraries
id: cache_llvm
id: retrieve_llvm_libs
if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS')
uses: actions/cache@v3
with:
@ -182,10 +187,10 @@ jobs:
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
key: ${{ matrix.llvm_cache_key }}
- name: Quit if cache miss
if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.cache_llvm.outputs.cache-hit != 'true')
if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true')
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
- name: Build iwasm
@ -210,8 +215,14 @@ jobs:
#$AOT_BUILD_OPTIONS,
]
os: [macos-latest]
wasi_sdk_release: ["https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-macos.tar.gz"]
wabt_release: ["https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-macos.tar.gz"]
wasi_sdk_release:
[
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-macos.tar.gz",
]
wabt_release:
[
"https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-macos-12.tar.gz",
]
steps:
- name: checkout
uses: actions/checkout@v3
@ -220,24 +231,14 @@ jobs:
run: |
cd /opt
sudo wget ${{ matrix.wabt_release }}
sudo tar -xzf wabt-1.0.24-*.tar.gz
sudo mv wabt-1.0.24 wabt
sudo tar -xzf wabt-1.0.31-*.tar.gz
sudo mv wabt-1.0.31 wabt
- name: Build Sample [wasm-c-api]
run: |
mkdir build && cd build
cmake .. ${{ matrix.make_options }}
cmake --build . --config Release --parallel 4
./callback
./callback_chain
./empty_imports
./global
./hello
./hostref
./memory
./reflect
./table
./trap
cmake -S . -B build ${{ matrix.make_options }}
cmake --build build --config Release --parallel 4
ctest --test-dir build
working-directory: samples/wasm-c-api
build_samples_others:
@ -246,8 +247,14 @@ jobs:
strategy:
matrix:
os: [macos-latest]
wasi_sdk_release: ["https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-macos.tar.gz"]
wabt_release: ["https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-macos.tar.gz"]
wasi_sdk_release:
[
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-macos.tar.gz",
]
wabt_release:
[
"https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-macos-12.tar.gz",
]
steps:
- name: checkout
uses: actions/checkout@v3
@ -256,15 +263,15 @@ jobs:
run: |
cd /opt
sudo wget ${{ matrix.wasi_sdk_release }}
sudo tar -xzf wasi-sdk-12.0-*.tar.gz
sudo mv wasi-sdk-12.0 wasi-sdk
sudo tar -xzf wasi-sdk-*.tar.gz
sudo mv wasi-sdk-19.0 wasi-sdk
- name: download and install wabt
run: |
cd /opt
sudo wget ${{ matrix.wabt_release }}
sudo tar -xzf wabt-1.0.24-*.tar.gz
sudo mv wabt-1.0.24 wabt
sudo tar -xzf wabt-1.0.31-*.tar.gz
sudo mv wabt-1.0.31 wabt
- name: Build Sample [basic]
run: |

57
.github/workflows/compilation_on_sgx.yml vendored
View File

@ -51,13 +51,13 @@ env:
FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
LLVM_LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1"
LLVM_EAGER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0"
LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex"
jobs:
build_llvm_libraries:
uses: ./.github/workflows/build_llvm_libraries.yml
with:
runs-on: "['ubuntu-20.04']"
os: "ubuntu-20.04"
arch: "X86"
build_iwasm:
runs-on: ${{ matrix.os }}
@ -131,7 +131,9 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-20.04]
include:
- os: ubuntu-20.04
llvm_cache_key: ${{ needs.build_llvm_libraries.outputs.cache_key }}
steps:
- name: install SGX SDK and necessary libraries
run: |
@ -150,7 +152,7 @@ jobs:
uses: actions/checkout@v3
- name: Get LLVM libraries
id: cache_llvm
id: retrieve_llvm_libs
uses: actions/cache@v3
with:
path: |
@ -159,10 +161,10 @@ jobs:
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
key: ${{ matrix.llvm_cache_key }}
- name: Quit if cache miss
if: steps.cache_llvm.outputs.cache-hit != 'true'
if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
- name: Build wamrc
@ -189,11 +191,11 @@ jobs:
os: [ubuntu-20.04]
wasi_sdk_release:
[
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz",
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz",
]
wabt_release:
[
"https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz",
"https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz",
]
steps:
- name: checkout
@ -203,8 +205,8 @@ jobs:
run: |
cd /opt
sudo wget ${{ matrix.wabt_release }}
sudo tar -xzf wabt-1.0.24-*.tar.gz
sudo mv wabt-1.0.24 wabt
sudo tar -xzf wabt-1.0.31-*.tar.gz
sudo mv wabt-1.0.31 wabt
- name: install SGX SDK and necessary libraries
run: |
@ -221,19 +223,9 @@ jobs:
- name: Build Sample [wasm-c-api]
run: |
mkdir build && cd build
cmake .. ${{ matrix.make_options }}
cmake --build . --config Release --parallel 4
./callback
./callback_chain
./empty_imports
./global
./hello
./hostref
./memory
./reflect
./table
./trap
cmake -S . -B build ${{ matrix.make_options }}
cmake --build build --config Release --parallel 4
ctest --test-dir build
working-directory: samples/wasm-c-api
build_samples_others:
@ -244,11 +236,11 @@ jobs:
os: [ubuntu-20.04]
wasi_sdk_release:
[
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz",
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz",
]
wabt_release:
[
"https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz",
"https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz",
]
steps:
- name: checkout
@ -258,15 +250,15 @@ jobs:
run: |
cd /opt
sudo wget ${{ matrix.wasi_sdk_release }}
sudo tar -xzf wasi-sdk-12.0-*.tar.gz
sudo mv wasi-sdk-12.0 wasi-sdk
sudo tar -xzf wasi-sdk-*.tar.gz
sudo mv wasi-sdk-19.0 wasi-sdk
- name: download and install wabt
run: |
cd /opt
sudo wget ${{ matrix.wabt_release }}
sudo tar -xzf wabt-1.0.24-*.tar.gz
sudo mv wabt-1.0.24 wabt
sudo tar -xzf wabt-1.0.31-*.tar.gz
sudo mv wabt-1.0.31 wabt
- name: install SGX SDK and necessary libraries
run: |
@ -334,6 +326,7 @@ jobs:
matrix:
running_mode: ["classic-interp", "fast-interp", "aot"]
test_option: ["-x -p -s spec -b -P", "-x -p -s spec -S -b -P"]
llvm_cache_key: ["${{ needs.build_llvm_libraries.outputs.cache_key }}"]
# classic-interp and fast-interp don't support simd
exclude:
- running_mode: "classic-interp"
@ -347,7 +340,7 @@ jobs:
- name: Get LLVM libraries
if: matrix.running_mode == 'aot'
id: cache_llvm
id: retrieve_llvm_libs
uses: actions/cache@v3
with:
path: |
@ -356,10 +349,10 @@ jobs:
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
key: ${{ matrix.llvm_cache_key }}
- name: Quit if cache miss
if: matrix.running_mode == 'aot' && steps.cache_llvm.outputs.cache-hit != 'true'
if: matrix.running_mode == 'aot' && steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
- name: install SGX SDK and necessary libraries

6
.github/workflows/release_process.yml vendored
View File

@ -123,7 +123,7 @@ jobs:
runner: ubuntu-20.04
upload_url: ${{ needs.create_release.outputs.upload_url }}
ver_num: ${{ needs.create_tag.outputs.new_ver}}
wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz
wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz
release_wamr_sdk_on_ubuntu_2204:
needs: [create_tag, create_release]
@ -133,7 +133,7 @@ jobs:
runner: ubuntu-22.04
upload_url: ${{ needs.create_release.outputs.upload_url }}
ver_num: ${{ needs.create_tag.outputs.new_ver}}
wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz
wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz
release_wamr_sdk_on_macos:
needs: [create_tag, create_release]
@ -143,7 +143,7 @@ jobs:
runner: macos-latest
upload_url: ${{ needs.create_release.outputs.upload_url }}
ver_num: ${{ needs.create_tag.outputs.new_ver}}
wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-macos.tar.gz
wasi_sdk_url: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-macos.tar.gz
#
# vscode extension cross-platform

10
.github/workflows/spec_test_on_nuttx.yml vendored
View File

@ -16,7 +16,8 @@ jobs:
build_llvm_libraries:
uses: ./.github/workflows/build_llvm_libraries.yml
with:
runs-on: "['ubuntu-22.04']"
os: "ubuntu-22.04"
arch: "ARM RISCV AArch64"
spec_test_on_qemu:
runs-on: ${{ matrix.os }}
@ -37,6 +38,7 @@ jobs:
"-t aot",
"-t aot -X"
]
llvm_cache_key: [ "${{ needs.build_llvm_libraries.outputs.cache_key }}" ]
steps:
- name: Install Utilities
run: |
@ -72,7 +74,7 @@ jobs:
path: apps/interpreters/wamr/wamr
- name: Get LLVM libraries
id: cache_llvm
id: retrieve_llvm_libs
uses: actions/cache@v3
with:
path: |
@ -81,10 +83,10 @@ jobs:
./core/deps/llvm/build/lib
./core/deps/llvm/build/libexec
./core/deps/llvm/build/share
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
key: ${{ matrix.llvm_cache_key }}
- name: Quit if cache miss
if: steps.cache_llvm.outputs.cache-hit != 'true'
if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
- name: Copy LLVM

3
.gitignore vendored
View File

@ -32,3 +32,6 @@ samples/socket-api/wasm-src/inc/pthread.h
**/__pycache__
tests/benchmarks/coremark/coremark*
samples/workload/include/**
!samples/workload/include/.gitkeep

84
build-scripts/build_llvm.py
View File

@ -21,28 +21,31 @@ def clone_llvm(dst_dir, llvm_repo, llvm_branch):
llvm_dir = dst_dir.joinpath("llvm").resolve()
if not llvm_dir.exists():
print(f"Clone llvm to {llvm_dir} ...")
GIT_CLONE_CMD = f"git clone --depth 1 --branch {llvm_branch} {llvm_repo} llvm"
subprocess.check_output(shlex.split(GIT_CLONE_CMD), cwd=dst_dir)
else:
print(f"There is an LLVM local repo in {llvm_dir}, clean and keep using it")
return llvm_dir
def build_llvm(llvm_dir, platform, backends, projects):
def query_llvm_version(llvm_dir):
GIT_LOG_CMD = f"git log --format=format:'%h' -1"
return subprocess.check_output(
shlex.split(GIT_LOG_CMD), cwd=llvm_dir, universal_newlines=True, text=True
)
def build_llvm(llvm_dir, platform, backends, projects, use_clang=False):
LLVM_COMPILE_OPTIONS = [
'-DCMAKE_BUILD_TYPE:STRING="Release"',
"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
"-DLLVM_APPEND_VC_REV:BOOL=ON",
"-DLLVM_BUILD_BENCHMARKS:BOOL=OFF",
"-DLLVM_BUILD_DOCS:BOOL=OFF",
"-DLLVM_BUILD_EXAMPLES:BOOL=OFF",
"-DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF",
"-DLLVM_BUILD_TESTS:BOOL=OFF",
"-DLLVM_CCACHE_BUILD:BOOL=OFF",
"-DLLVM_CCACHE_BUILD:BOOL=ON",
"-DLLVM_ENABLE_BINDINGS:BOOL=OFF",
"-DLLVM_ENABLE_IDE:BOOL=OFF",
"-DLLVM_ENABLE_LIBEDIT=OFF",
"-DLLVM_ENABLE_TERMINFO:BOOL=OFF",
"-DLLVM_ENABLE_ZLIB:BOOL=OFF",
"-DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF",
@ -54,6 +57,18 @@ def build_llvm(llvm_dir, platform, backends, projects):
"-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON",
]
# use clang/clang++/lld. but macos doesn't support lld
if not sys.platform.startswith("darwin") and use_clang:
if shutil.which("clang") and shutil.which("clang++") and shutil.which("lld"):
os.environ["CC"] = "clang"
os.environ["CXX"] = "clang++"
LLVM_COMPILE_OPTIONS.append('-DLLVM_USE_LINKER:STRING="lld"')
print("Use the clang toolchain")
else:
print("Can not find clang, clang++ and lld, keep using the gcc toolchain")
else:
print("Use the gcc toolchain")
LLVM_EXTRA_COMPILE_OPTIONS = {
"arc": [
'-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="ARC"',
@ -99,8 +114,10 @@ def build_llvm(llvm_dir, platform, backends, projects):
lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve()
if lib_llvm_core_library.exists():
print(f"Please remove {build_dir} manually and try again")
return build_dir
print(
f"It has already been fully compiled. If want to a re-build, please remove {build_dir} manually and try again"
)
return None
compile_options = " ".join(
LLVM_COMPILE_OPTIONS
@ -119,10 +136,11 @@ def build_llvm(llvm_dir, platform, backends, projects):
CONFIG_CMD += " -G'Unix Makefiles'"
else:
CONFIG_CMD += " -A x64"
print(f"{CONFIG_CMD}")
else:
CONFIG_CMD += " -G'Ninja'"
subprocess.check_call(shlex.split(CONFIG_CMD), cwd=build_dir)
BUILD_CMD = f"cmake --build . --target package --parallel {os.cpu_count()}" + (
BUILD_CMD = "cmake --build . --target package" + (
" --config Release" if "windows" == platform else ""
)
subprocess.check_call(shlex.split(BUILD_CMD), cwd=build_dir)
@ -133,23 +151,25 @@ def build_llvm(llvm_dir, platform, backends, projects):
def repackage_llvm(llvm_dir):
build_dir = llvm_dir.joinpath("./build").resolve()
packs = [f for f in build_dir.glob("LLVM-13*.tar.gz")]
packs = [f for f in build_dir.glob("LLVM-*.tar.gz")]
if len(packs) > 1:
raise Exception("Find more than one LLVM-13*.tar.gz")
raise Exception("Find more than one LLVM-*.tar.gz")
if not packs:
return
llvm_package = packs[0].name
# mv build/LLVM-13.0.0*.gz .
# mv build/LLVM-*.gz .
shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir))
# rm -r build
shutil.rmtree(str(build_dir))
# mkdir build
build_dir.mkdir()
# tar xf ./LLVM-13.0.0-*.tar.gz --strip-components=1 --directory=build
# tar xf ./LLVM-*.tar.gz --strip-components=1 --directory=build
CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}"
subprocess.check_call(shlex.split(CMD), cwd=llvm_dir)
# rm ./LLVM-1*.gz
os.remove(llvm_dir.joinpath(llvm_package).resolve())
def main():
@ -184,8 +204,17 @@ def main():
choices=["clang", "lldb"],
help="identify extra LLVM projects, separate by space, like '--project clang lldb'",
)
parser.add_argument(
"--llvm-ver",
action="store_true",
help="return the version info of generated llvm libraries",
)
parser.add_argument(
"--use-clang",
action="store_true",
help="use clang instead of gcc",
)
options = parser.parse_args()
print(f"options={options}")
# if the "platform" is not identified in the command line option,
# detect it
@ -199,12 +228,10 @@ def main():
else:
platform = options.platform
print(f"========== Build LLVM for {platform} ==========\n")
llvm_repo_and_branch = {
"arc": {
"repo": "https://github.com/llvm/llvm-project.git",
"branch": "release/13.x",
"branch": "release/15.x",
},
"xtensa": {
"repo": "https://github.com/espressif/llvm-project.git",
@ -212,7 +239,7 @@ def main():
},
"default": {
"repo": "https://github.com/llvm/llvm-project.git",
"branch": "release/13.x",
"branch": "release/15.x",
},
}
@ -225,19 +252,22 @@ def main():
deps_dir = current_dir.joinpath("../core/deps").resolve()
try:
print(f"==================== CLONE LLVM ====================")
llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"])
llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"])
print()
print(f"==================== BUILD LLVM ====================")
build_llvm(llvm_dir, platform, options.arch, options.project)
if options.llvm_ver:
commit_hash = query_llvm_version(llvm_dir)
print(commit_hash)
return commit_hash is not None
print()
print(f"==================== PACKAGE LLVM ====================")
if (
build_llvm(
llvm_dir, platform, options.arch, options.project, options.use_clang
)
is not None
):
repackage_llvm(llvm_dir)
print()
return True
except subprocess.CalledProcessError:
return False

4
doc/build_wasm_app.md
View File

@ -3,12 +3,12 @@
Prepare WASM building environments
==================================
For C and C++, WASI-SDK version 12.0+ is the major tool supported by WAMR to build WASM applications. Also, we can use [Emscripten SDK (EMSDK)](https://github.com/emscripten-core/emsdk), but it is not recommended. And there are some other compilers such as the standard clang compiler, which might also work [here](./other_wasm_compilers.md).
For C and C++, WASI-SDK version 19.0+ is the major tool supported by WAMR to build WASM applications. Also, we can use [Emscripten SDK (EMSDK)](https://github.com/emscripten-core/emsdk), but it is not recommended. And there are some other compilers such as the standard clang compiler, which might also work [here](./other_wasm_compilers.md).
To install WASI SDK, please download the [wasi-sdk release](https://github.com/CraneStation/wasi-sdk/releases) and extract the archive to default path `/opt/wasi-sdk`.
The official *wasi-sdk release* doesn't fully support *latest 128-bit SIMD spec* yet. WAMR provides a script in [build-wasi-sdk](../test-tools/build-wasi-sdk/) to generate
another wasi-sdk with *llvm-13* from source code and installs it at *../test-tools/wasi-sdk*. If you plan to build WASM applications with *latest 128-bit SIMD*, please use it instead of the official release.
another wasi-sdk with *llvm-15* from source code and installs it at *../test-tools/wasi-sdk*. If you plan to build WASM applications with *latest 128-bit SIMD*, please use it instead of the official release.
And [sample workloads](../samples/workload) are using the self-compiled wasi-sdk.

27
samples/wasm-c-api/CMakeLists.txt
View File

@ -136,26 +136,13 @@ include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake)
set(MM_UTIL src/utils/multi_module_utils.c)
# build executable for each .c
set(EXAMPLES
callback
callback_chain
clone
empty_imports
global
hello
hostref
memory
reflect
table
threads
trap
)
if(WAMR_BUILD_JIT AND WAMR_BUILD_LAZY_JIT)
if((${WAMR_BUILD_JIT} EQUAL 1) AND (${WAMR_BUILD_LAZY_JIT} EQUAL 1))
list(APPEND EXAMPLES serialize)
endif()
endif()
list(APPEND EXAMPLES callback callback_chain empty_imports global hello hostref memory reflect table trap)
# FIXME enable both in the future
#list(APPEND EXAMPLES clone threads)
# FIXME
# if(WAMR_BUILD_JIT EQUAL 1 AND WAMR_BUILD_LAZY_JIT EQUAL 0)
# list(APPEND EXAMPLES serialize)
# endif()
check_pie_supported()

6
samples/wasm-c-api/README.md
View File

@ -4,9 +4,9 @@ Before staring, we need to download and intall [WABT](https://github.com/WebAsse
``` shell
$ cd /opt
$ wget https://github.com/WebAssembly/wabt/releases/download/1.0.19/wabt-1.0.19-ubuntu.tar.gz
$ tar -xzf wabt-1.0.19-ubuntu.tar.gz
$ mv wabt-1.0.19 wabt
$ wget https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz
$ tar -xzf wabt-1.0.31-ubuntu.tar.gz
$ mv wabt-1.0.31 wabt
```
By default, all samples are compiled and run in "interpreter" mode.

116
samples/workload/CMakeLists.txt Normal file
View File

@ -0,0 +1,116 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
cmake_minimum_required (VERSION 3.14)
project(wasm_workloads)
#######################################
add_subdirectory(bwa)
add_subdirectory(meshoptimizer)
add_subdirectory(wasm-av1)
#######################################
include(ExternalProject)
################ iwasm ################
ExternalProject_Add(iwasm
PREFIX
iwasm-build
BUILD_ALWAYS
YES
SOURCE_DIR
${CMAKE_CURRENT_SOURCE_DIR}/../../product-mini/platforms/linux
CONFIGURE_COMMAND
${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/../../product-mini/platforms/linux -B build -DWAMR_BUILD_LIBC_EMCC=1
BUILD_COMMAND
${CMAKE_COMMAND} --build build
INSTALL_COMMAND
# FIXME: replace with --install
${CMAKE_COMMAND} -E copy_if_different
${CMAKE_CURRENT_BINARY_DIR}/iwasm-build/src/iwasm-build/build/iwasm
${CMAKE_CURRENT_BINARY_DIR}/iwasm
)
################ wamrc ################
ExternalProject_Add(wamrc
PREFIX
wamrc-build
BUILD_ALWAYS
YES
SOURCE_DIR
${CMAKE_CURRENT_SOURCE_DIR}/../../wamr-compiler
CONFIGURE_COMMAND
${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/../../wamr-compiler -B build
BUILD_COMMAND
${CMAKE_COMMAND} --build build
INSTALL_COMMAND
# FIXME: replace with --install
${CMAKE_COMMAND} -E copy_if_different
${CMAKE_CURRENT_BINARY_DIR}/wamrc-build/src/wamrc-build/build/wamrc
${CMAKE_CURRENT_BINARY_DIR}/wamrc
)
################ .aot ################
add_custom_target(
bwa_to_aot
ALL
DEPENDS
bwa wamrc
COMMAND
./wamrc -o bwa.aot ./bwa/bwa.wasm
WORKING_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR}
)
add_custom_target(
codecbench_to_aot
ALL
DEPENDS
codecbench wamrc
COMMAND
./wamrc -o codecbench.aot ./meshoptimizer/codecbench.wasm
WORKING_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR}
)
add_custom_target(
av1_to_aot
ALL
DEPENDS
av1 wamrc
COMMAND
./wamrc -o testavx.aot ./wasm-av1/testavx.opt.wasm
WORKING_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR}
)
################ smoking test ################
include(CTest)
add_test(
NAME
run_bwa
COMMAND
./iwasm --dir=. ./bwa.aot index ./bwa/hs38DH-extra.fa
WORKING_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR}
)
add_test(
NAME
run_codecbench
COMMAND
./iwasm codecbench.aot
WORKING_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR}
)
add_test(
NAME
run_av1
COMMAND
./iwasm --dir=. testavx.aot ./wasm-av1/elephants_dream_480p24.ivf
WORKING_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR}
)

29
samples/workload/README.md
View File

@ -1,41 +1,30 @@
All workloads have similar requirment of software dependencies, including
**emsdk**, **wabt** and **binaryen**
All workloads have similar requirment of software dependencies, including **emsdk** and **binaryen**
> There might be slight differences when using MacOS and other Linux distro than Ubuntu. This document only target
Ubuntu 18.04 as example.
> There might be slight differences when using MacOS and other Linux distro than Ubuntu. This document targets
Ubuntu 20.04 as an example.
## Installation instructions
use [preparation.sh](./preparation.sh) to install all dependencies before compiling any workload.
use [preparation.sh](./preparation.sh) to install all dependencies before compiling any workload. Or use [*vscode DevContainer*](../../.devcontainer/)
for details, the script includes below steps:
- **wabt**. Install
[latest release](https://github.com/WebAssembly/wabt/releases/download/1.0.23/wabt-1.0.23-ubuntu.tar.gz)
to */opt/wabt*
``` bash
$ wget https://github.com/WebAssembly/wabt/releases/download/${WABT_VER}/${WABT_FILE}
$ tar zxf ${WABT_FILE} -C /opt
$ ln -sf /opt/wabt-${WABT_VER} /opt/wabt
```
The script installs below software:
- **emsdk**. Refer to [the guide](https://emscripten.org/docs/getting_started/downloads.html). Don't forget to activate
emsdk and set up environment variables. Verify it with `echo ${EMSDK}`. Please be sure to install and activate the building
of 2.0.26
of 3.0.0
``` bash
$ cd /opt
$ git clone https://github.com/emscripten-core/emsdk.git
$ cd emsdk
$ git pull
$ ./emsdk install 2.0.26
$ ./emsdk activate 2.0.26
$ ./emsdk install 3.0.0
$ ./emsdk activate 3.0.0
$ echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc
```
- **binaryen**. Install
[latest release](https://github.com/WebAssembly/binaryen/releases/download/version_101/binaryen-version_101-x86_64-linux.tar.gz)
[latest release](https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-linux.tar.gz)
to */opt/binaryen*
``` bash

62
samples/workload/XNNPACK/CMakeLists.txt
View File

@ -15,8 +15,9 @@ ExternalProject_Add(xnnpack
GIT_PROGRESS ON
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack
UPDATE_COMMAND git checkout .
&& git reset --hard 4d738aef36872669e4bba05a4b259149ba8e62e1
&& cmake -E copy ${CMAKE_CURRENT_SOURCE_DIR}/benchmark.patch ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/third_party
&& git reset --hard 4570a7151aa4f3e57eca14a575eeff6bb13e26be
&& cmake -E copy ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/google3/third_party/XNNPACK/microkernels.bzl
${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/
&& git apply ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack.patch
CONFIGURE_COMMAND ""
# grep xnnpack_benchmark -A 1 BUILD.bazel \
@ -24,70 +25,123 @@ ExternalProject_Add(xnnpack
# | awk '{print $3}' \
# | sed -e 's/\"//g' -e 's/,//g' -e 's/^/\/\/:/g'
BUILD_COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack
&& bazel --output_user_root=build_user_output build -c opt --config=wasm
&& bazel --output_user_root=build-user-output build -c opt --config=wasm
//:qs8_dwconv_bench.wasm
//:qs8_f32_vcvt_bench.wasm
//:qs8_gemm_bench.wasm
//:qs8_requantization_bench.wasm
//:qs8_vadd_bench.wasm
//:qs8_vaddc_bench.wasm
//:qs8_vcvt_bench.wasm
//:qs8_vlrelu_bench.wasm
//:qs8_vmul_bench.wasm
//:qs8_vmulc_bench.wasm
//:qu8_f32_vcvt_bench.wasm
//:qu8_gemm_bench.wasm
//:qu8_requantization_bench.wasm
//:qu8_vadd_bench.wasm
//:qu8_vaddc_bench.wasm
//:qu8_vcvt_bench.wasm
//:qu8_vlrelu_bench.wasm
//:qu8_vmul_bench.wasm
//:qu8_vmulc_bench.wasm
//:bf16_gemm_bench.wasm
//:f16_igemm_bench.wasm
//:f16_gemm_bench.wasm
//:f16_raddstoreexpminusmax_bench.wasm
//:f16_spmm_bench.wasm
//:f16_vrelu_bench.wasm
//:f16_vsigmoid_bench.wasm
//:f16_f32_vcvt_bench.wasm
//:f32_igemm_bench.wasm
//:f32_conv_hwc_bench.wasm
//:f16_conv_hwc2chw_bench.wasm
//:f16_gavgpool_cw_bench.wasm
//:f32_gavgpool_cw_bench.wasm
//:f32_conv_hwc2chw_bench.wasm
//:f16_dwconv_bench.wasm
//:f32_dwconv_bench.wasm
//:f32_dwconv2d_chw_bench.wasm
//:f16_dwconv2d_chw_bench.wasm
//:f32_f16_vcvt_bench.wasm
//:xx_transpose_bench.wasm
//:x8_transpose_bench.wasm
//:x16_transpose_bench.wasm
//:x24_transpose_bench.wasm
//:x32_transpose_bench.wasm
//:x64_transpose_bench.wasm
//:f32_gemm_bench.wasm
//:f32_qs8_vcvt_bench.wasm
//:f32_qu8_vcvt_bench.wasm
//:f32_raddexpminusmax_bench.wasm
//:f32_raddextexp_bench.wasm
//:f32_raddstoreexpminusmax_bench.wasm
//:f32_rmax_bench.wasm
//:f32_spmm_bench.wasm
//:f32_softmax_bench.wasm
//:f16_velu_bench.wasm
//:f32_velu_bench.wasm
//:f32_vhswish_bench.wasm
//:f32_vlrelu_bench.wasm
//:f32_vrelu_bench.wasm
//:f32_vscaleexpminusmax_bench.wasm
//:f32_vscaleextexp_bench.wasm
//:f32_vsigmoid_bench.wasm
//:f16_vsqrt_bench.wasm
//:f32_vsqrt_bench.wasm
//:f32_im2col_gemm_bench.wasm
//:rounding_bench.wasm
//:s16_rmaxabs_bench.wasm
//:s16_window_bench.wasm
//:u32_filterbank_accumulate_bench.wasm
//:u32_filterbank_subtract_bench.wasm
//:u32_vlog_bench.wasm
//:u64_u32_vsqrtshift_bench.wasm
//:i16_vlshift_bench.wasm
//:cs16_vsquareabs_bench.wasm
//:cs16_bfly4_bench.wasm
//:cs16_fftr_bench.wasm
//:x8_lut_bench.wasm
//:abs_bench.wasm
//:average_pooling_bench.wasm
//:bankers_rounding_bench.wasm
//:ceiling_bench.wasm
//:channel_shuffle_bench.wasm
//:convert_bench.wasm
//:convolution_bench.wasm
//:deconvolution_bench.wasm
//:elu_bench.wasm
//:floor_bench.wasm
//:global_average_pooling_bench.wasm
//:hardswish_bench.wasm
//:leaky_relu_bench.wasm
//:max_pooling_bench.wasm
//:negate_bench.wasm
//:sigmoid_bench.wasm
//:prelu_bench.wasm
//:softmax_bench.wasm
//:square_bench.wasm
//:square_root_bench.wasm
//:truncation_bench.wasm
//:f16_gemm_e2e_bench.wasm
//:f32_dwconv_e2e_bench.wasm
//:f32_gemm_e2e_bench.wasm
//:qs8_dwconv_e2e_bench.wasm
//:qs8_gemm_e2e_bench.wasm
//:qu8_gemm_e2e_bench.wasm
//:qu8_dwconv_e2e_bench.wasm
//:end2end_bench.wasm
//:f16_exp_ulp_eval.wasm
//:f16_expminus_ulp_eval.wasm
//:f16_expm1minus_ulp_eval.wasm
//:f16_sigmoid_ulp_eval.wasm
//:f16_sqrt_ulp_eval.wasm
//:f32_exp_ulp_eval.wasm
//:f32_expminus_ulp_eval.wasm
//:f32_expm1minus_ulp_eval.wasm
//:f32_extexp_ulp_eval.wasm
//:f32_sigmoid_ulp_eval.wasm
//:f32_sqrt_ulp_eval.wasm
//:f32_tanh_ulp_eval.wasm
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory
${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/bazel-out/wasm-opt/bin/
${CMAKE_BINARY_DIR}/wasm-opt

4
samples/workload/XNNPACK/README.md
View File

@ -24,7 +24,7 @@ Firstly please build iwasm with simd, libc-emcc and lib-pthread support:
``` bash
$ cd <wamr-dir>/product-mini/platforms/linux/
$ mkdir build && cd build
$ cmake .. -DWAMR_BUILD_SIMD=1 -DWAMR_BUILD_LIBC_EMCC=1 -DWAMR_BUILD_LIB_PTHREAD=1
$ cmake .. -DWAMR_BUILD_LIBC_EMCC=1 -DWAMR_BUILD_LIB_PTHREAD=1
$ make
```
@ -42,7 +42,7 @@ Then compile wasm file to aot file and run:
``` shell
$ cd <wamr-dir>/samples/workload/XNNPACK/xnnpack/bazel-bin
$ wamrc --enable-simd -o average_pooling_bench.aot average_pooling_bench.wasm (or other wasm files)
$ wamrc -o average_pooling_bench.aot average_pooling_bench.wasm (or other wasm files)
$ iwasm average_pooling_bench.aot
```

1
samples/workload/XNNPACK/build_workload.sh
View File

@ -1 +0,0 @@
../docker/build_workload.sh

80
samples/workload/XNNPACK/xnnpack.patch
View File

@ -1,8 +1,8 @@
diff --git a/.bazelrc b/.bazelrc
index ec740f38..29f9d56e 100644
index 688279da1..376996885 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -49,4 +49,9 @@ build:ios_fat --watchos_cpus=armv7k
@@ -53,4 +53,9 @@ build:ios_fat --watchos_cpus=armv7k
build:macos --apple_platform_type=macos
build:macos_arm64 --config=macos
@ -11,42 +11,26 @@ index ec740f38..29f9d56e 100644
+build:macos_arm64 --cpu=darwin_arm64
+
+build:wasm --cpu=wasm
+build:wasm --copt=-msimd128
+build:wasm --features=wasm_simd
+build:wasm --crosstool_top=@emsdk//emscripten_toolchain:everything
+build:wasm --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
diff --git a/BUILD.bazel b/BUILD.bazel
index 3fc8139f..c893356d 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -11988,7 +11988,6 @@ config_setting(
values = {
"crosstool_top": "@emsdk//emscripten_toolchain:everything",
"cpu": "wasm",
- "copt": "-msimd128",
"copt": "-mrelaxed-simd",
},
)
diff --git a/WORKSPACE b/WORKSPACE
index c58e76b6..30934678 100644
index cd8960ffa..5d3e685f4 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -21,6 +21,7 @@ http_archive(
name = "com_google_benchmark",
strip_prefix = "benchmark-master",
urls = ["https://github.com/google/benchmark/archive/master.zip"],
+ patches = ["@//third_party:benchmark.patch"],
)
# FP16 library, used for half-precision conversions
@@ -84,6 +85,19 @@ http_archive(
@@ -92,8 +92,25 @@ http_archive(
],
)
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+ name = "emsdk",
+ strip_prefix = "emsdk-2.0.26/bazel",
+ url = "https://github.com/emscripten-core/emsdk/archive/refs/tags/2.0.26.tar.gz",
+ sha256 = "79e7166aa8eaae6e52cef1363b2d8db795d03684846066bc51f9dcf905dd58ad",
+ # Use emsdk-3.0.0 since the larger version may:
+ # - compress the wasm file into a tar file but not directly generate wasm file
+ # - generate incomplete implementation of libc API, e.g. throw exception in getentropy
+ strip_prefix = "emsdk-3.0.0/bazel",
+ url = "https://github.com/emscripten-core/emsdk/archive/refs/tags/3.0.0.tar.gz",
+ sha256 = "a41dccfd15be9e85f923efaa0ac21943cbab77ec8d39e52f25eca1ec61a9ac9e"
+)
+
+load("@emsdk//:deps.bzl", emsdk_deps = "deps")
@ -56,13 +40,17 @@ index c58e76b6..30934678 100644
+emsdk_emscripten_deps()
+
# Android NDK location and version is auto-detected from $ANDROID_NDK_HOME environment variable
android_ndk_repository(name = "androidndk")
-android_ndk_repository(name = "androidndk")
+#android_ndk_repository(name = "androidndk")
# Android SDK location and API is auto-detected from $ANDROID_HOME environment variable
-android_sdk_repository(name = "androidsdk")
+#android_sdk_repository(name = "androidsdk")
diff --git a/build_defs.bzl b/build_defs.bzl
index fbadb400..e496b78d 100644
index b8217a18d..da232966e 100644
--- a/build_defs.bzl
+++ b/build_defs.bzl
@@ -430,7 +430,7 @@ def xnnpack_benchmark(name, srcs, copts = [], deps = [], tags = []):
@@ -380,7 +380,7 @@ def xnnpack_benchmark(name, srcs, copts = [], deps = [], tags = []):
explicitly specified.
"""
native.cc_binary(
@ -72,7 +60,7 @@ index fbadb400..e496b78d 100644
copts = xnnpack_std_cxxopts() + [
"-Iinclude",
diff --git a/emscripten.bzl b/emscripten.bzl
index 130d5f16..2696ad54 100644
index f1557a7b1..7f964a094 100644
--- a/emscripten.bzl
+++ b/emscripten.bzl
@@ -25,12 +25,19 @@ def xnnpack_emscripten_benchmark_linkopts():
@ -84,7 +72,7 @@ index 130d5f16..2696ad54 100644
- "-s EXIT_RUNTIME=1",
+ "-s ERROR_ON_UNDEFINED_SYMBOLS=0",
"-s ALLOW_MEMORY_GROWTH=1",
"-s TOTAL_MEMORY=445644800", # 425M
"-s TOTAL_MEMORY=536870912", # 512M
- "--pre-js $(location :preamble.js.lds)",
+ "-s USE_PTHREADS=0",
+ "-s STANDALONE_WASM=1",
@ -99,11 +87,33 @@ index 130d5f16..2696ad54 100644
]
def xnnpack_emscripten_deps():
diff --git a/src/log.c b/src/log.c
index 5715f2f85..4b3e4261b 100644
--- a/src/log.c
+++ b/src/log.c
@@ -55,7 +55,7 @@
#endif
#if XNN_LOG_TO_STDIO
-static void xnn_vlog(int output_handle, const char* prefix, size_t prefix_length, const char* format, va_list args) {
+void xnn_vlog(int output_handle, const char* prefix, size_t prefix_length, const char* format, va_list args) {
char stack_buffer[XNN_LOG_STACK_BUFFER_SIZE];
char* heap_buffer = NULL;
char* out_buffer = &stack_buffer[0];
diff --git a/third_party/cpuinfo.BUILD b/third_party/cpuinfo.BUILD
index 128d683e..f6c287c4 100644
index 1997f4e3a..5e03c43af 100644
--- a/third_party/cpuinfo.BUILD
+++ b/third_party/cpuinfo.BUILD
@@ -343,5 +343,5 @@ config_setting(
@@ -150,7 +150,7 @@ cc_library(
"src/arm/midr.h",
],
deps = [
- "@clog",
+ "//deps/clog"
],
)
@@ -352,5 +352,5 @@ config_setting(
config_setting(
name = "emscripten",

15
samples/workload/bwa/CMakeLists.bwa_wasm.txt
View File

@ -1,11 +1,14 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
cmake_minimum_required (VERSION 3.0)
cmake_minimum_required (VERSION 3.14)
project(bwa_wasm C)
include(${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/preparation.cmake)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake)
################ dependencies ################
find_package(Binaryen 111 REQUIRED)
################ LIBZ ################
set(LIBZ_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../libz)
@ -86,12 +89,6 @@ add_executable(${PROJECT_NAME} ${BWA_SOURCE})
set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME bwa.wasm)
target_include_directories(${PROJECT_NAME}
PRIVATE
${WASI_SDK_HOME}/share/wasi-sysroot/include/libc/musl
${WASI_SDK_HOME}/share/wasi-sysroot/include/sse
)
target_compile_definitions(${PROJECT_NAME}
PRIVATE
USE_MALLOC_WRAPPERS
@ -117,7 +114,7 @@ target_link_libraries(${PROJECT_NAME} z_wasm wasi-emulated-process-clocks)
add_custom_target(bwa_wasm_opt ALL
COMMAND
${WASM_OPT} -Oz --enable-simd -o bwa.opt.wasm bwa.wasm
${Binaryen_WASM_OPT} -Oz --enable-simd -o bwa.opt.wasm bwa.wasm
BYPRODUCTS
${CMAKE_CURRENT_BINARY_DIR}/bwa.opt.wasm
WORKING_DIRECTORY

43
samples/workload/bwa/CMakeLists.txt
View File

@ -1,11 +1,19 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
cmake_minimum_required (VERSION 2.8...3.16)
cmake_minimum_required (VERSION 3.14)
project(bwa_wasm)
include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/preparation.cmake)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake)
################ dependencies ################
find_package(Python3 REQUIRED)
find_package(WASISDK 16.0 REQUIRED)
execute_process(
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
)
#######################################
include(ExternalProject)
@ -13,7 +21,7 @@ include(ExternalProject)
################ libz ################
ExternalProject_Add(libz_src
GIT_REPOSITORY https://github.com/madler/zlib.git
GIT_TAG master
GIT_TAG 04f42ceca40f73e2978b50e93806c2a18c1281fc
GIT_PROGRESS ON
GIT_SHALLOW ON
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libz
@ -27,7 +35,7 @@ ExternalProject_Add(libz_src
################ bwa ################
ExternalProject_Add(bwa
GIT_REPOSITORY https://github.com/lh3/bwa.git
GIT_TAG master
GIT_TAG 139f68fc4c3747813783a488aef2adc86626b01b
GIT_PROGRESS ON
GIT_SHALLOW ON
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/bwa
@ -37,10 +45,29 @@ ExternalProject_Add(bwa
&& ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.bwa_wasm.txt CMakeLists.txt
&& git apply ../bwa.patch
CONFIGURE_COMMAND ${CMAKE_COMMAND}
-DWASI_SDK_PREFIX=${WASI_SDK_HOME}
-DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_HOME}/share/cmake/wasi-sdk.cmake
-DCMAKE_SYSROOT=${WASI_SDK_HOME}/share/wasi-sysroot
-DWASI_SDK_PREFIX=${WASISDK_HOME}
-DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN}
-DCMAKE_SYSROOT=${WASISDK_SYSROOT}
-DCMAKE_C_FLAGS=-isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/sse\ -isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/libc/musl
${CMAKE_CURRENT_SOURCE_DIR}/bwa
BUILD_COMMAND make bwa_wasm_opt
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ./bwa.opt.wasm ${CMAKE_BINARY_DIR}/bwa.wasm
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different ./bwa.opt.wasm ${CMAKE_CURRENT_BINARY_DIR}/bwa.wasm
)
################ bwa data ################
ExternalProject_Add(bwa-kit
PREFIX bwa-kit
URL https://sourceforge.net/projects/bio-bwa/files/bwakit/bwakit-0.7.15_x64-linux.tar.bz2/download
URL_HASH SHA256=0a7b11971bc7916b68e9df35a364afe77cb3000df02ffb3a6fbd1aff9be5878c
DOWNLOAD_NAME bwakit-0.7.15_x64-linux.tar.bz2
DOWNLOAD_EXTRACT_TIMESTAMP ON
DOWNLOAD_NO_EXTRACT OFF
DOWNLOAD_NO_PROGRESS ON
UPDATE_COMMAND ""
PATCH_COMMAND ""
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different
${CMAKE_CURRENT_BINARY_DIR}/bwa-kit/src/bwa-kit/resource-GRCh38/hs38DH-extra.fa
${CMAKE_CURRENT_BINARY_DIR}/hs38DH-extra.fa
)

4
samples/workload/bwa/README.md
View File

@ -33,7 +33,7 @@ Firstly please build iwasm with simd support:
``` shell
$ cd <wamr dir>/product-mini/platforms/linux/
$ mkdir build && cd build
$ cmake .. -DWAMR_BUILD_SIMD=1
$ cmake ..
$ make
```
@ -41,6 +41,6 @@ Then compile wasm file to aot file and run:
``` shell
$ cd <wamr dir>/samples/workload/bwa/build
$ <wamr dir>/wamr-compiler/build/wamrc --enable-simd -o bwa.aot bwa.wasm
$ <wamr dir>/wamr-compiler/build/wamrc -o bwa.aot bwa.wasm
$ <wamr dir>/product-mini/platforms/linux/iwasm --dir=. bwa.aot index hs38DH.fa
```

1
samples/workload/bwa/build_workload.sh
View File

@ -1 +0,0 @@
../docker/build_workload.sh

43
samples/workload/cmake/FindBinaryen.cmake Normal file
View File

@ -0,0 +1,43 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# Output below variables:
# - Binaryen_HOME. the installation location
#
include(CMakePrintHelpers)
include(FindPackageHandleStandardArgs)
file(GLOB Binaryen_SEARCH_PATH "/opt/binaryen*")
find_path(Binaryen_HOME
NAMES bin/wasm-opt
PATHS ${Binaryen_SEARCH_PATH}
NO_CMAKE_FIND_ROOT_PATH
NO_SYSTEM_ENVIRONMENT_PATH
REQUIRED
)
execute_process(
COMMAND ${Binaryen_HOME}/bin/wasm-opt --version
OUTPUT_VARIABLE WASM_OPT_OUTPUT
OUTPUT_STRIP_TRAILING_WHITESPACE
)
string(REGEX MATCH version_[0-9]+ Binaryen_VERSION_tmp ${WASM_OPT_OUTPUT})
string(REGEX MATCH [0-9]+ Binaryen_VERSION ${Binaryen_VERSION_tmp})
#cmake_print_variables(Binaryen_VERSION_tmp Binaryen_VERSION)
find_package_handle_standard_args(Binaryen REQUIRED_VARS Binaryen_HOME VERSION_VAR Binaryen_VERSION)
if(Binaryen_FOUND)
mark_as_advanced(Binaryen_SEARCH_PATH)
mark_as_advanced(Binaryen_VERSION_tmp)
mark_as_advanced(Binaryen_VERSION)
mark_as_advanced(WASM_OPT_OUTPUT)
set(Binaryen_WASM_OPT ${Binaryen_HOME}/bin/wasm-opt)
else()
# TODO: install WASISDK
endif()

38
samples/workload/cmake/FindWASISDK.cmake Normal file
View File

@ -0,0 +1,38 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# Output below variables:
# - WASISDK_HOME. the installation location
# - WASISDK_SYSROOT. where wasi-sysroot is
# - WASISDK_TOOLCHAIN. where wasi-sdk.cmake is
#
include(CMakePrintHelpers)
include(FindPackageHandleStandardArgs)
file(GLOB WASISDK_SEARCH_PATH "/opt/wasi-sdk-*")
find_path(WASISDK_HOME
NAMES share/wasi-sysroot
PATHS ${WASISDK_SEARCH_PATH}
NO_CMAKE_FIND_ROOT_PATH
NO_SYSTEM_ENVIRONMENT_PATH
REQUIRED
)
string(REGEX MATCH [0-9]+\.[0-9]+\.*[0-9]* WASISDK_VERSION ${WASISDK_HOME})
#cmake_print_variables(WASISDK_HOME WASISDK_VERSION)
find_package_handle_standard_args(WASISDK REQUIRED_VARS WASISDK_HOME VERSION_VAR WASISDK_VERSION)
if(WASISDK_FOUND)
mark_as_advanced(WASISDK_SEARCH_PATH)
mark_as_advanced(WASISDK_VERSION)
set(WASISDK_CC_COMMAND ${WASISDK_HOME}/bin/clang)
set(WASISDK_CXX_COMMAND ${WASISDK_HOME}/bin/clang++)
set(WASISDK_SYSROOT ${WASISDK_HOME}/share/wasi-sysroot)
set(WASISDK_TOOLCHAIN ${WASISDK_HOME}/share/cmake/wasi-sdk.cmake)
else()
# TODO: install WASISDK
endif()

49
samples/workload/cmake/preparation.cmake
View File

@ -1,49 +0,0 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#######################################
include(ExternalProject)
file(REAL_PATH ../../.. WAMR_ROOT
BASE_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
)
find_path(WASI_SDK_PARENT
name wasi-sdk
PATHS ${WAMR_ROOT}/test-tools/
NO_DEFAULT_PATH
NO_CMAKE_FIND_ROOT_PATH
)
if(NOT WASI_SDK_PARENT)
message(FATAL_ERROR
"can not find 'wasi-sdk' under ${WAMR_ROOT}/test-tools, "
"please run ${WAMR_ROOT}/test-tools/build-wasi-sdk/build_wasi_sdk.py "
"to build wasi-sdk and try again"
)
endif()
set(WASI_SDK_HOME ${WASI_SDK_PARENT}/wasi-sdk)
message(CHECK_START "Detecting WASI-SDK at ${WASI_SDK_HOME}")
if(EXISTS "${WASI_SDK_HOME}/share/cmake/wasi-sdk.cmake")
message(CHECK_PASS "found")
else()
message(CHECK_FAIL "not found")
endif()
################ BINARYEN ################
find_program(WASM_OPT
NAMES wasm-opt
PATHS /opt/binaryen-version_101/bin /opt/binaryen/bin
NO_DEFAULT_PATH
NO_CMAKE_FIND_ROOT_PATH
)
if(NOT WASM_OPT)
message(FATAL_ERROR
"can not find wasm-opt. "
"please download it from "
"https://github.com/WebAssembly/binaryen/releases/download/version_101/binaryen-version_101-x86_64-linux.tar.gz "
"and install it under /opt"
)
endif()

33
samples/workload/docker/build_workload.sh
View File

@ -1,33 +0,0 @@
#!/usr/bin/env bash
#
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
readonly SCRIPT_PATH=$(dirname "$(realpath "$0")")
readonly ROOT=$(realpath "${SCRIPT_PATH}"/../../../)
readonly CURRENT_PATH=$(pwd)
readonly CURRENT_RELATIVE_ROOT=$(realpath --relative-base ${ROOT} ${CURRENT_PATH})
readonly VARIANT=$(lsb_release -c | awk '{print $2}')
docker build \
--build-arg VARIANT=${VARIANT} \
--memory 4G --cpu-quota 50000 \
-t wamr_dev_${VARIANT}:0.1 -f "${ROOT}"/.devcontainer/Dockerfile "${ROOT}"/.devcontainer &&
docker run --rm -it \
--memory 4G \
--cpus ".5" \
--name workload_build_env \
--mount type=bind,source="${ROOT}",target=/workspace \
wamr_dev_${VARIANT}:0.1 \
/bin/bash -c "\
pwd \
&& pushd ${CURRENT_RELATIVE_ROOT} \
&& rm -rf build \
&& mkdir build \
&& pushd build \
&& cmake .. \
&& cmake --build . --config Release \
&& popd \
&& popd \
&& echo 'Go and find out results under ${CURRENT_RELATIVE_ROOT}/build' "

0
samples/workload/include/.gitkeep Normal file
View File

22
samples/workload/meshoptimizer/CMakeLists.txt
View File

@ -1,11 +1,19 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
cmake_minimum_required (VERSION 3.0)
cmake_minimum_required (VERSION 3.14)
project(bench-meshoptimizer)
include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/preparation.cmake)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake)
################ dependencies ################
find_package(Python3 REQUIRED)
find_package(WASISDK 16.0 REQUIRED)
execute_process(
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
)
################ MESHOPTIMIZER ################
include(ExternalProject)
@ -13,7 +21,7 @@ include(ExternalProject)
ExternalProject_Add(codecbench
PREFIX codecbench
GIT_REPOSITORY https://github.com/zeux/meshoptimizer.git
GIT_TAG master
GIT_TAG f926b288264522e1b331a41b07ba40167f396913
GIT_SHALLOW ON
GIT_PROGRESS ON
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/meshoptimizer
@ -21,10 +29,10 @@ ExternalProject_Add(codecbench
&& ${CMAKE_COMMAND} -E echo "Applying patch"
&& git apply ${CMAKE_CURRENT_SOURCE_DIR}/codecbench.patch
CONFIGURE_COMMAND ${CMAKE_COMMAND}
-DWASI_SDK_PREFIX=${WASI_SDK_HOME}
-DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_HOME}/share/cmake/wasi-sdk.cmake
-DCMAKE_SYSROOT=${WASI_SDK_HOME}/share/wasi-sysroot
-DWASI_SDK_PREFIX=${WASISDK_HOME}
-DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN}
-DCMAKE_SYSROOT=${WASISDK_SYSROOT}
${CMAKE_CURRENT_SOURCE_DIR}/meshoptimizer
BUILD_COMMAND make codecbench
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ./codecbench.wasm ${CMAKE_BINARY_DIR}/codecbench.wasm
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different ./codecbench.wasm ${CMAKE_CURRENT_BINARY_DIR}/codecbench.wasm
)

4
samples/workload/meshoptimizer/README.md
View File

@ -44,14 +44,14 @@ Firstly please build iwasm with simd support:
``` shell
$ cd <wamr dir>/product-mini/platforms/linux/
$ mkdir build && cd build
$ cmake .. -DWAMR_BUILD_SIMD=1
$ cmake ..
$ make
```
Then compile wasm file to aot file and run:
``` shell
$ <wamr dir>/wamr-compiler/build/wamrc --enable-simd -o codecbench.aot codecbench.wasm
$ <wamr dir>/wamr-compiler/build/wamrc -o codecbench.aot codecbench.wasm
$ <wamr dir>/product-mini/platforms/linux/build/iwasm codecbench.aot
```

1
samples/workload/meshoptimizer/build_workload.sh
View File

@ -1 +0,0 @@
../docker/build_workload.sh

12
samples/workload/preparation.sh
View File

@ -5,13 +5,13 @@
#
readonly BUILD_CONTENT="/tmp/build_content"
readonly WABT_VER=1.0.23
readonly WABT_VER=1.0.31
readonly WABT_FILE="wabt-${WABT_VER}-ubuntu.tar.gz"
readonly CMAKE_VER=3.16.2
readonly CMAKE_VER=3.25.1
readonly CMAKE_FILE="cmake-${CMAKE_VER}-Linux-x86_64.sh"
readonly BINARYEN_VER=version_101
readonly BINARYEN_VER=version_111
readonly BINARYEN_FILE="binaryen-${BINARYEN_VER}-x86_64-linux.tar.gz"
readonly BAZEL_VER=3.7.0
readonly BAZEL_VER=6.0.0
readonly BAZEL_FILE=bazel-${BAZEL_VER}-installer-linux-x86_64.sh
function DEBUG() {
@ -57,8 +57,8 @@ function install_emsdk() {
git clone https://github.com/emscripten-core/emsdk.git
cd emsdk
git pull
./emsdk install 2.0.26
./emsdk activate 2.0.26
./emsdk install 3.1.28
./emsdk activate 3.1.28
echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc
}

10
samples/workload/tensorflow/build.sh
View File

@ -98,11 +98,11 @@ make
WAMRC_CMD="$(pwd)/wamrc"
cd ${OUT_DIR}
if [[ $1 == '--sgx' ]]; then
${WAMRC_CMD} --enable-simd -sgx -o benchmark_model.aot benchmark_model.wasm
${WAMRC_CMD} -sgx -o benchmark_model.aot benchmark_model.wasm
elif [[ $1 == '--threads' ]]; then
${WAMRC_CMD} --enable-simd --enable-multi-thread -o benchmark_model.aot benchmark_model.wasm
${WAMRC_CMD} --enable-multi-thread -o benchmark_model.aot benchmark_model.wasm
else
${WAMRC_CMD} --enable-simd -o benchmark_model.aot benchmark_model.wasm
${WAMRC_CMD} -o benchmark_model.aot benchmark_model.wasm
fi
# 4. build iwasm with pthread and libc_emcc enable
@ -112,14 +112,14 @@ fi
if [[ $1 == '--sgx' ]]; then
cd ${WAMR_PLATFORM_DIR}/linux-sgx
rm -fr build && mkdir build
cd build && cmake .. -DWAMR_BUILD_SIMD=1 -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1
cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1
make
cd ../enclave-sample
make
else
cd ${WAMR_PLATFORM_DIR}/linux
rm -fr build && mkdir build
cd build && cmake .. -DWAMR_BUILD_SIMD=1 -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1
cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1
make
fi

10
samples/workload/wasm-av1/CMakeLists.avx_wasm.txt
View File

@ -1,14 +1,14 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
cmake_minimum_required (VERSION 2.8...3.16)
cmake_minimum_required (VERSION 3.14)
project(testavx)
include(${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/preparation.cmake)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake)
# a workaround to let aom find our non-public headers
include_directories(${WASI_SDK_HOME}/share/wasi-sysroot/include/libc/musl)
################ dependencies ################
find_package(Binaryen 111 REQUIRED)
################ AOM ################
set(ENABLE_CCACHE ON)
@ -62,7 +62,7 @@ add_dependencies(${PROJECT_NAME} aom)
add_custom_target(${PROJECT_NAME}_opt ALL
COMMAND
${WASM_OPT} -Oz --enable-simd -o ${PROJECT_NAME}.opt.wasm ${PROJECT_NAME}.wasm
${Binaryen_WASM_OPT} -Oz --enable-simd -o ${PROJECT_NAME}.opt.wasm ${PROJECT_NAME}.wasm
BYPRODUCTS
${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.opt.wasm
WORKING_DIRECTORY

24
samples/workload/wasm-av1/CMakeLists.txt
View File

@ -1,11 +1,19 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
cmake_minimum_required (VERSION 2.8...3.16)
cmake_minimum_required (VERSION 3.14)
project(av1_wasm)
include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/preparation.cmake)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake)
################ dependencies ################
find_package(Python3 REQUIRED)
find_package(WASISDK 16.0 REQUIRED)
execute_process(
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
)
#######################################
include(ExternalProject)
@ -23,10 +31,14 @@ ExternalProject_Add(av1
&& ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.avx_wasm.txt CMakeLists.txt
&& git apply ../av1-clang.patch
CONFIGURE_COMMAND ${CMAKE_COMMAND}
-DWASI_SDK_PREFIX=${WASI_SDK_HOME}
-DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_HOME}/share/cmake/wasi-sdk.cmake
-DCMAKE_SYSROOT=${WASI_SDK_HOME}/share/wasi-sysroot
-DWASI_SDK_PREFIX=${WASISDK_HOME}
-DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN}
-DCMAKE_SYSROOT=${WASISDK_SYSROOT}
-DCMAKE_C_FLAGS=-isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/sse\ -isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/libc/musl
${CMAKE_CURRENT_SOURCE_DIR}/av1
BUILD_COMMAND make testavx_opt
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy testavx.opt.wasm ${CMAKE_CURRENT_BINARY_DIR}/testavx.wasm
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different
testavx.opt.wasm
${CMAKE_CURRENT_SOURCE_DIR}/av1/third_party/samples/elephants_dream_480p24.ivf
${CMAKE_CURRENT_BINARY_DIR}
)

4
samples/workload/wasm-av1/README.md
View File

@ -39,7 +39,7 @@ Firstly please build iwasm with simd support:
``` shell
$ cd <wamr dir>/product-mini/platforms/linux/
$ mkdir build && cd build
$ cmake .. -DWAMR_BUILD_SIMD=1 -DWAMR_BUILD_LIBC_EMCC=1
$ cmake .. -DWAMR_BUILD_LIBC_EMCC=1
$ make
```
@ -47,7 +47,7 @@ Then compile wasm file to aot file and run:
``` shell
$ cd <dir of testavx.wasm>
$ <wamr dir>/wamr-compiler/build/wamrc --enable-simd -o testavx.aot testavx.wasm
$ <wamr dir>/wamr-compiler/build/wamrc -o testavx.aot testavx.wasm
# copy sample data like <wamr dir>/samples/workload/wasm-av1/av1/third_party/samples/elephants_dream_480p24.ivf
# make sure you declare the access priority of the directory in which the sample data is
$ <wamr dir>/product-mini/platforms/linux/build/iwasm --dir=. testavx.aot elephants_dream_480p24.ivf

4
samples/workload/wasm-av1/build.sh
View File

@ -85,12 +85,12 @@ cd build && cmake ..
make
# 3.2 compile wasm-av1.wasm to wasm-av1.aot
cd ${OUT_DIR}
${WAMRC_CMD} --enable-simd -o testavx.aot testavx.wasm
${WAMRC_CMD} -o testavx.aot testavx.wasm
# 4. build iwasm with pthread and libc_emcc enable
cd ${WAMR_PLATFORM_DIR}/linux
rm -fr build && mkdir build
cd build && cmake .. -DWAMR_BUILD_SIMD=1 -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1
cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1
make
# 5. run wasm-av1 with iwasm

1
samples/workload/wasm-av1/build_workload.sh
View File

@ -1 +0,0 @@
../docker/build_workload.sh

13
test-tools/build-wasi-sdk/patches/wasi_libc.patch
View File

@ -1,13 +0,0 @@
diff --git a/expected/wasm32-wasi/predefined-macros.txt b/expected/wasm32-wasi/predefined-macros.txt
index c1bb19e..954f3b5 100644
--- a/expected/wasm32-wasi/predefined-macros.txt
+++ b/expected/wasm32-wasi/predefined-macros.txt
@@ -3002,6 +3002,8 @@
#define __alignof_is_defined 1
#define __bitop(x,i,o) ((x)[(i)/8] o (1<<(i)%8))
#define __bool_true_false_are_defined 1
+#define __clang_literal_encoding__ "UTF-8"
+#define __clang_wide_literal_encoding__ "UTF-32"
#define __inline inline
#define __restrict restrict
#define __tg_complex(fun,x) (__RETCAST_CX(x)( __FLTCX((x)+I) && __IS_FP(x) ? fun ## f (x) : __LDBLCX((x)+I) ? fun ## l (x) : fun(x) ))

15
test-tools/build-wasi-sdk/patches/wasi_sdk.patch
View File

@ -1,15 +0,0 @@
diff --git a/version.sh b/version.sh
index 8e7c44c..ff0d3ba 100755
--- a/version.sh
+++ b/version.sh
@@ -1,5 +1,6 @@
#!/usr/bin/env bash
-set -e
-GIT_DESCR=$(git describe --long --candidates=999 --match='wasi-sdk-*' --dirty='+m' --abbrev=12)
-GIT_PACKAGE_VERSION=$(echo $GIT_DESCR | perl -ne 'if(/^wasi-sdk-(\d+)-(\d+)-g([0-9a-f]{7,12})([+]m)?$/) { if($2 == 0) { print "$1.$2$4" } else { print "$1.$2g$3$4" } exit } else { print "could not parse git description"; exit 1 }';)
-echo $GIT_PACKAGE_VERSION
+#set -e
+#GIT_DESCR=$(git describe --long --candidates=999 --match='wasi-sdk-*' --dirty='+m' --abbrev=12)
+#GIT_PACKAGE_VERSION=$(echo $GIT_DESCR | perl -ne 'if(/^wasi-sdk-(\d+)-(\d+)-g([0-9a-f]{7,12})([+]m)?$/) { if($2 == 0) { print "$1.$2$4" } else { print "$1.$2g$3$4" } exit } else { print "could not parse git description"; exit 1 }';)
+#echo $GIT_PACKAGE_VERSION
+echo wasi-sdk-13-eng

185
test-tools/build-wasi-sdk/build_wasi_sdk.py → test-tools/pick-up-emscripten-headers/collect_files.py
View File

@ -9,68 +9,35 @@ The script operates on such directories and files
|-- core
| `-- deps
| |-- emscripten
| `-- wasi-sdk
| `-- src
| |-- llvm-project
| `-- wasi-libc
`-- test-tools
|-- build-wasi-sdk
| |-- build_wasi_sdk.py
|-- samples
| `-- workloads
| |-- include
| `-- patches
`-- wasi-sdk
|-- bin
|-- lib
`-- share
`-- wasi-sysroot
`-- test-tools
|-- pick-up-emscripten_headers
| |-- collect_files.py
"""
import argparse
import hashlib
import logging
import os
import pathlib
import shlex
import shutil
import subprocess
import sys
import tarfile
import tempfile
import urllib
import urllib.request
logger = logging.getLogger("build_wasi_sdk")
logger = logging.getLogger("pick-up-emscripten-headers")
external_repos = {
"config": {
"sha256": "302e5e7f3c4996976c58efde8b2f28f71d51357e784330eeed738e129300dc33",
"store_dir": "core/deps/wasi-sdk/src/config",
"strip_prefix": "config-191bcb948f7191c36eefe634336f5fc5c0c4c2be",
"url": "https://git.savannah.gnu.org/cgit/config.git/snapshot/config-191bcb948f7191c36eefe634336f5fc5c0c4c2be.tar.gz",
},
"emscripten": {
"sha256": "0904a65379aea3ea94087b8c12985b2fee48599b473e3bef914fec2e3941532d",
"sha256": "c5524755b785d8f4b83eb3214fdd3ac4b2e1b1a4644df4c63f06e5968f48f90e",
"store_dir": "core/deps/emscripten",
"strip_prefix": "emscripten-2.0.28",
"url": "https://github.com/emscripten-core/emscripten/archive/refs/tags/2.0.28.tar.gz",
},
"llvm-project": {
"sha256": "dc5169e51919f2817d06615285e9da6a804f0f881dc55d6247baa25aed3cc143",
"store_dir": "core/deps/wasi-sdk/src/llvm-project",
"strip_prefix": "llvm-project-34ff6a75f58377f32a5046a29f55c4c0e58bee9e",
"url": "https://github.com/llvm/llvm-project/archive/34ff6a75f58377f32a5046a29f55c4c0e58bee9e.tar.gz",
},
"wasi-sdk": {
"sha256": "fc4fdb0e97b915241f32209492a7d0fab42c24216f87c1d5d75f46f7c70a553d",
"store_dir": "core/deps/wasi-sdk",
"strip_prefix": "wasi-sdk-1a953299860bbcc198ad8c12a21d1b2e2f738355",
"url": "https://github.com/WebAssembly/wasi-sdk/archive/1a953299860bbcc198ad8c12a21d1b2e2f738355.tar.gz",
},
"wasi-libc": {
"sha256": "f6316ca9479d3463eb1c4f6a1d1f659bf15f67cb3c1e2e83d9d11f188dccd864",
"store_dir": "core/deps/wasi-sdk/src/wasi-libc",
"strip_prefix": "wasi-libc-a78cd329aec717f149934d7362f57050c9401f60",
"url": "https://github.com/WebAssembly/wasi-libc/archive/a78cd329aec717f149934d7362f57050c9401f60.tar.gz",
},
"strip_prefix": "emscripten-3.0.0",
"url": "https://github.com/emscripten-core/emscripten/archive/refs/tags/3.0.0.tar.gz",
}
}
# TOOD: can we use headers from wasi-libc and clang directly ?
@ -111,6 +78,7 @@ def unpack(tar_file, strip_prefix, dest_dir):
with tempfile.TemporaryDirectory() as tmp:
with tarfile.open(tar_file) as tar:
logger.debug(f"extract to {tmp}")
def is_within_directory(directory, target):
abs_directory = os.path.abspath(directory)
@ -129,7 +97,6 @@ def unpack(tar_file, strip_prefix, dest_dir):
tar.extractall(path, members, numeric_owner=numeric_owner)
safe_extract(tar, tmp)
strip_prefix_dir = (
@ -160,12 +127,12 @@ def download_repo(name, root):
download_flag = store_dir.joinpath("DOWNLOADED")
if store_dir.exists() and download_flag.exists():
logger.info(
f"keep using '{store_dir.relative_to(root)}'. Or to remove it and try again"
f"bypass downloading '{store_dir.relative_to(root)}'. Or to remove it and try again if needs a new release"
)
return True
# download only when the target is neither existed nor broken
download_dir = pathlib.Path("/tmp/build_wasi_sdk/")
download_dir = pathlib.Path("/tmp/pick-up-emscripten-headers/")
download_dir.mkdir(exist_ok=True)
tar_name = pathlib.Path(external_repos[name]["url"]).name
@ -192,104 +159,67 @@ def download_repo(name, root):
download_flag.touch()
# leave download files in /tmp
logger.info(f"Has downloaed and stored in {store_dir.relative_to(root)}")
return True
def run_patch(patch_file, cwd):
if not patch_file.exists():
logger.error(f"{patch_file} not found")
def collect_headers(root, install_location):
if not install_location.exists():
logger.error(f"{install_location} does not found")
return False
with open(patch_file, "r") as f:
try:
PATCH_DRY_RUN_CMD = "patch -f -p1 --dry-run"
if subprocess.check_call(shlex.split(PATCH_DRY_RUN_CMD), stdin=f, cwd=cwd):
logger.error(f"patch dry-run {cwd} failed")
return False
PATCH_CMD = "patch -f -p1"
f.seek(0)
if subprocess.check_call(shlex.split(PATCH_CMD), stdin=f, cwd=cwd):
logger.error(f"patch {cwd} failed")
return False
except subprocess.CalledProcessError:
logger.error(f"patch {cwd} failed")
return False
install_flag = install_location.joinpath("INSTALLED").resolve()
if install_flag.exists():
logger.info(
f"bypass downloading '{install_location}'. Or to remove it and try again if needs a new one"
)
return True
def build_and_install_wasi_sdk(root):
store_dir = root.joinpath(f'{external_repos["wasi-sdk"]["store_dir"]}').resolve()
if not store_dir.exists():
logger.error(f"{store_dir} does not found")
return False
# patch wasi-libc and wasi-sdk
patch_flag = store_dir.joinpath("PATCHED")
if not patch_flag.exists():
if not run_patch(
root.joinpath("test-tools/build-wasi-sdk/patches/wasi_libc.patch"),
store_dir.joinpath("src/wasi-libc"),
):
return False
if not run_patch(
root.joinpath("test-tools/build-wasi-sdk/patches/wasi_sdk.patch"), store_dir
):
return False
patch_flag.touch()
else:
logger.info("bypass the patch phase")
# build
build_flag = store_dir.joinpath("BUILDED")
if not build_flag.exists():
BUILD_CMD = "make build"
if subprocess.check_call(shlex.split(BUILD_CMD), cwd=store_dir):
logger.error(f"build wasi-sdk failed")
return False
build_flag.touch()
else:
logger.info("bypass the build phase")
# install
install_flag = store_dir.joinpath("INSTALLED")
binary_path = root.joinpath("test-tools").resolve()
if not install_flag.exists():
shutil.copytree(
str(store_dir.joinpath("build/install/opt").resolve()),
str(binary_path),
dirs_exist_ok=True,
)
# install headers
emscripten_headers = (
root.joinpath(external_repos["emscripten"]["store_dir"])
.joinpath("system")
.resolve()
)
wasi_sysroot_headers = binary_path.joinpath(
"wasi-sdk/share/wasi-sysroot/include"
emscripten_home = root.joinpath(
f'{external_repos["emscripten"]["store_dir"]}'
).resolve()
if not emscripten_home.exists():
logger.error(f"{emscripten_home} does not found")
return False
emscripten_headers = emscripten_home.joinpath("system").resolve()
for (src, dst) in emscripten_headers_src_dst:
src = emscripten_headers.joinpath(src)
dst = wasi_sysroot_headers.joinpath(dst)
dst = install_location.joinpath(dst)
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(src, dst)
install_flag.touch()
else:
logger.info("bypass the install phase")
logger.info(f"Has installed in {install_location}")
return True
def main():
parser = argparse.ArgumentParser(
description="collect headers from emscripten for workload compilation"
)
parser.add_argument(
"--install",
type=str,
required=True,
help="identify installation location",
)
parser.add_argument(
"--loglevel",
type=str,
default="INFO",
choices=[
"ERROR",
"WARNING",
"INFO",
],
help="the logging level",
)
options = parser.parse_args()
console = logging.StreamHandler()
console.setFormatter(logging.Formatter("%(asctime)s - %(message)s"))
logger.setLevel(logging.INFO)
logger.setLevel(getattr(logging, options.loglevel))
logger.addHandler(console)
logger.propagate = False
@ -305,12 +235,9 @@ def main():
if not download_repo(repo, root):
return False
# build wasi_sdk and install
if not build_and_install_wasi_sdk(root):
if not collect_headers(root, pathlib.Path(options.install)):
return False
# TODO install headers from emscripten
return True

8
test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile
View File

@ -25,10 +25,10 @@ RUN wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/
# - wamr-sdk
## - download wasi-sdk with wget and set up to /opt/wasi-sdk
RUN wget --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-14/wasi-sdk-14.0-linux.tar.gz \
&& tar -zxvf wasi-sdk-14.0-linux.tar.gz \
&& mv wasi-sdk-14.0 /opt/wasi-sdk/ \
&& rm -f wasi-sdk-14.0-linux.tar.gz
RUN wget --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz \
&& tar -zxvf wasi-sdk-*-linux.tar.gz \
&& mv wasi-sdk-19.0 /opt/wasi-sdk/ \
&& rm -f wasi-sdk-*-linux.tar.gz
## - clone wamr repo
RUN git clone -b main --depth=1 https://github.com/bytecodealliance/wasm-micro-runtime.git

8
tests/wamr-test-suites/test_wamr.sh
View File

@ -360,16 +360,16 @@ function spec_test()
exit 1
;;
esac
if [ ! -f /tmp/wabt-1.0.29-${WABT_PLATFORM}.tar.gz ]; then
if [ ! -f /tmp/wabt-1.0.31-${WABT_PLATFORM}.tar.gz ]; then
wget \
https://github.com/WebAssembly/wabt/releases/download/1.0.29/wabt-1.0.29-${WABT_PLATFORM}.tar.gz \
https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-${WABT_PLATFORM}.tar.gz \
-P /tmp
fi
cd /tmp \
&& tar zxf wabt-1.0.29-${WABT_PLATFORM}.tar.gz \
&& tar zxf wabt-1.0.31-${WABT_PLATFORM}.tar.gz \
&& mkdir -p ${WORK_DIR}/wabt/out/gcc/Release/ \
&& install wabt-1.0.29/bin/wa* ${WORK_DIR}/wabt/out/gcc/Release/ \
&& install wabt-1.0.31/bin/wa* ${WORK_DIR}/wabt/out/gcc/Release/ \
&& cd -
fi
else