27e7e160af
Upgrade the version of related toolkits: - upgrade llvm to 15.0 - upgrade wasi-sdk to 19.0 - upgrade emsdk to 3.1.28 - upgrade wabt to 1.0.31 - upgrade binaryen to 111 And upgrade the CI scripts, sample workload build scripts, Dockerfiles, and documents.
278 lines
8.4 KiB
Python
Executable File
278 lines
8.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (C) 2019 Intel Corporation. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
#
|
|
|
|
import argparse
|
|
import os
|
|
import pathlib
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sysconfig
|
|
import sys
|
|
|
|
|
|
def clone_llvm(dst_dir, llvm_repo, llvm_branch):
|
|
"""
|
|
any error will raise CallProcessError
|
|
"""
|
|
llvm_dir = dst_dir.joinpath("llvm").resolve()
|
|
|
|
if not llvm_dir.exists():
|
|
GIT_CLONE_CMD = f"git clone --depth 1 --branch {llvm_branch} {llvm_repo} llvm"
|
|
subprocess.check_output(shlex.split(GIT_CLONE_CMD), cwd=dst_dir)
|
|
|
|
return llvm_dir
|
|
|
|
|
|
def query_llvm_version(llvm_dir):
|
|
GIT_LOG_CMD = f"git log --format=format:'%h' -1"
|
|
return subprocess.check_output(
|
|
shlex.split(GIT_LOG_CMD), cwd=llvm_dir, universal_newlines=True, text=True
|
|
)
|
|
|
|
|
|
def build_llvm(llvm_dir, platform, backends, projects, use_clang=False):
|
|
LLVM_COMPILE_OPTIONS = [
|
|
'-DCMAKE_BUILD_TYPE:STRING="Release"',
|
|
"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
|
|
"-DLLVM_APPEND_VC_REV:BOOL=ON",
|
|
"-DLLVM_BUILD_EXAMPLES:BOOL=OFF",
|
|
"-DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF",
|
|
"-DLLVM_BUILD_TESTS:BOOL=OFF",
|
|
"-DLLVM_CCACHE_BUILD:BOOL=ON",
|
|
"-DLLVM_ENABLE_BINDINGS:BOOL=OFF",
|
|
"-DLLVM_ENABLE_IDE:BOOL=OFF",
|
|
"-DLLVM_ENABLE_LIBEDIT=OFF",
|
|
"-DLLVM_ENABLE_TERMINFO:BOOL=OFF",
|
|
"-DLLVM_ENABLE_ZLIB:BOOL=OFF",
|
|
"-DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF",
|
|
"-DLLVM_INCLUDE_DOCS:BOOL=OFF",
|
|
"-DLLVM_INCLUDE_EXAMPLES:BOOL=OFF",
|
|
"-DLLVM_INCLUDE_UTILS:BOOL=OFF",
|
|
"-DLLVM_INCLUDE_TESTS:BOOL=OFF",
|
|
"-DLLVM_BUILD_TESTS:BOOL=OFF",
|
|
"-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON",
|
|
]
|
|
|
|
# use clang/clang++/lld. but macos doesn't support lld
|
|
if not sys.platform.startswith("darwin") and use_clang:
|
|
if shutil.which("clang") and shutil.which("clang++") and shutil.which("lld"):
|
|
os.environ["CC"] = "clang"
|
|
os.environ["CXX"] = "clang++"
|
|
LLVM_COMPILE_OPTIONS.append('-DLLVM_USE_LINKER:STRING="lld"')
|
|
print("Use the clang toolchain")
|
|
else:
|
|
print("Can not find clang, clang++ and lld, keep using the gcc toolchain")
|
|
else:
|
|
print("Use the gcc toolchain")
|
|
|
|
LLVM_EXTRA_COMPILE_OPTIONS = {
|
|
"arc": [
|
|
'-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="ARC"',
|
|
"-DLLVM_ENABLE_LIBICUUC:BOOL=OFF",
|
|
"-DLLVM_ENABLE_LIBICUDATA:BOOL=OFF",
|
|
],
|
|
"xtensa": [
|
|
'-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="Xtensa"',
|
|
],
|
|
"windows": [
|
|
"-DCMAKE_INSTALL_PREFIX=LLVM-install",
|
|
],
|
|
"default": [],
|
|
}
|
|
|
|
LLVM_TARGETS_TO_BUILD = [
|
|
'-DLLVM_TARGETS_TO_BUILD:STRING="' + ";".join(backends) + '"'
|
|
if backends
|
|
else '-DLLVM_TARGETS_TO_BUILD:STRING="AArch64;ARM;Mips;RISCV;X86"'
|
|
]
|
|
|
|
LLVM_PROJECTS_TO_BUILD = [
|
|
'-DLLVM_ENABLE_PROJECTS:STRING="' + ";".join(projects) + '"' if projects else ""
|
|
]
|
|
|
|
# lldb project requires libxml2
|
|
LLVM_LIBXML2_OPTION = [
|
|
"-DLLVM_ENABLE_LIBXML2:BOOL=" + ("ON" if "lldb" in projects else "OFF")
|
|
]
|
|
|
|
# enabling LLVM_INCLUDE_TOOLS will increase ~300M to the final package
|
|
LLVM_INCLUDE_TOOLS_OPTION = [
|
|
"-DLLVM_INCLUDE_TOOLS:BOOL=ON" if projects else "-DLLVM_INCLUDE_TOOLS:BOOL=OFF"
|
|
]
|
|
|
|
if not llvm_dir.exists():
|
|
raise Exception(f"{llvm_dir} doesn't exist")
|
|
|
|
build_dir = llvm_dir.joinpath(
|
|
"win32build" if "windows" == platform else "build"
|
|
).resolve()
|
|
build_dir.mkdir(exist_ok=True)
|
|
|
|
lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve()
|
|
if lib_llvm_core_library.exists():
|
|
print(
|
|
f"It has already been fully compiled. If want to a re-build, please remove {build_dir} manually and try again"
|
|
)
|
|
return None
|
|
|
|
compile_options = " ".join(
|
|
LLVM_COMPILE_OPTIONS
|
|
+ LLVM_LIBXML2_OPTION
|
|
+ LLVM_EXTRA_COMPILE_OPTIONS.get(
|
|
platform, LLVM_EXTRA_COMPILE_OPTIONS["default"]
|
|
)
|
|
+ LLVM_TARGETS_TO_BUILD
|
|
+ LLVM_PROJECTS_TO_BUILD
|
|
+ LLVM_INCLUDE_TOOLS_OPTION
|
|
)
|
|
|
|
CONFIG_CMD = f"cmake {compile_options} ../llvm"
|
|
if "windows" == platform:
|
|
if "mingw" in sysconfig.get_platform().lower():
|
|
CONFIG_CMD += " -G'Unix Makefiles'"
|
|
else:
|
|
CONFIG_CMD += " -A x64"
|
|
else:
|
|
CONFIG_CMD += " -G'Ninja'"
|
|
subprocess.check_call(shlex.split(CONFIG_CMD), cwd=build_dir)
|
|
|
|
BUILD_CMD = "cmake --build . --target package" + (
|
|
" --config Release" if "windows" == platform else ""
|
|
)
|
|
subprocess.check_call(shlex.split(BUILD_CMD), cwd=build_dir)
|
|
|
|
return build_dir
|
|
|
|
|
|
def repackage_llvm(llvm_dir):
|
|
build_dir = llvm_dir.joinpath("./build").resolve()
|
|
|
|
packs = [f for f in build_dir.glob("LLVM-*.tar.gz")]
|
|
if len(packs) > 1:
|
|
raise Exception("Find more than one LLVM-*.tar.gz")
|
|
|
|
if not packs:
|
|
return
|
|
|
|
llvm_package = packs[0].name
|
|
# mv build/LLVM-*.gz .
|
|
shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir))
|
|
# rm -r build
|
|
shutil.rmtree(str(build_dir))
|
|
# mkdir build
|
|
build_dir.mkdir()
|
|
# tar xf ./LLVM-*.tar.gz --strip-components=1 --directory=build
|
|
CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}"
|
|
subprocess.check_call(shlex.split(CMD), cwd=llvm_dir)
|
|
# rm ./LLVM-1*.gz
|
|
os.remove(llvm_dir.joinpath(llvm_package).resolve())
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="build necessary LLVM libraries")
|
|
parser.add_argument(
|
|
"--platform",
|
|
type=str,
|
|
choices=["android", "arc", "darwin", "linux", "windows", "xtensa"],
|
|
help="identify current platform",
|
|
)
|
|
parser.add_argument(
|
|
"--arch",
|
|
nargs="+",
|
|
type=str,
|
|
choices=[
|
|
"AArch64",
|
|
"ARC",
|
|
"ARM",
|
|
"Mips",
|
|
"RISCV",
|
|
"WebAssembly",
|
|
"X86",
|
|
"Xtensa",
|
|
],
|
|
help="identify LLVM supported backends, separate by space, like '--arch ARM Mips X86'",
|
|
)
|
|
parser.add_argument(
|
|
"--project",
|
|
nargs="+",
|
|
type=str,
|
|
default="",
|
|
choices=["clang", "lldb"],
|
|
help="identify extra LLVM projects, separate by space, like '--project clang lldb'",
|
|
)
|
|
parser.add_argument(
|
|
"--llvm-ver",
|
|
action="store_true",
|
|
help="return the version info of generated llvm libraries",
|
|
)
|
|
parser.add_argument(
|
|
"--use-clang",
|
|
action="store_true",
|
|
help="use clang instead of gcc",
|
|
)
|
|
options = parser.parse_args()
|
|
|
|
# if the "platform" is not identified in the command line option,
|
|
# detect it
|
|
if not options.platform:
|
|
if sys.platform.startswith("win32") or sys.platform.startswith("msys"):
|
|
platform = "windows"
|
|
elif sys.platform.startswith("darwin"):
|
|
platform = "darwin"
|
|
else:
|
|
platform = "linux"
|
|
else:
|
|
platform = options.platform
|
|
|
|
llvm_repo_and_branch = {
|
|
"arc": {
|
|
"repo": "https://github.com/llvm/llvm-project.git",
|
|
"branch": "release/15.x",
|
|
},
|
|
"xtensa": {
|
|
"repo": "https://github.com/espressif/llvm-project.git",
|
|
"branch": "xtensa_release_15.x",
|
|
},
|
|
"default": {
|
|
"repo": "https://github.com/llvm/llvm-project.git",
|
|
"branch": "release/15.x",
|
|
},
|
|
}
|
|
|
|
# retrieve the real file
|
|
current_file = pathlib.Path(__file__)
|
|
if current_file.is_symlink():
|
|
current_file = pathlib.Path(os.readlink(current_file))
|
|
|
|
current_dir = current_file.parent.resolve()
|
|
deps_dir = current_dir.joinpath("../core/deps").resolve()
|
|
|
|
try:
|
|
llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"])
|
|
llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"])
|
|
|
|
if options.llvm_ver:
|
|
commit_hash = query_llvm_version(llvm_dir)
|
|
print(commit_hash)
|
|
return commit_hash is not None
|
|
|
|
if (
|
|
build_llvm(
|
|
llvm_dir, platform, options.arch, options.project, options.use_clang
|
|
)
|
|
is not None
|
|
):
|
|
repackage_llvm(llvm_dir)
|
|
|
|
return True
|
|
except subprocess.CalledProcessError:
|
|
return False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(0 if main() else 1)
|