From ed94b7dcc4355b02f445b88f4ba115d77fdefe83 Mon Sep 17 00:00:00 2001 From: YaoLe Date: Thu, 29 Oct 2020 11:34:34 +0800 Subject: [PATCH] Implement Inclavare Containers PAL interface in WAMR Linux-SGX (#429) * Implement the PAL interface for rune Work in progress Signed-off-by: Le Yao * Support PAL for one runtime with multi-instances Load runtime into enclave and run multi-instances Signed-off-by: Le Yao --- .../linux-sgx/enclave-sample/App/App.cpp | 167 ++++++++++++++ .../linux-sgx/enclave-sample/App/README.md | 206 ++++++++++++++++++ .../linux-sgx/enclave-sample/App/pal_api.h | 146 +++++++++++++ .../enclave-sample/App/wamr-bundle.md | 65 ++++++ 4 files changed, 584 insertions(+) create mode 100644 product-mini/platforms/linux-sgx/enclave-sample/App/README.md create mode 100644 product-mini/platforms/linux-sgx/enclave-sample/App/pal_api.h create mode 100644 product-mini/platforms/linux-sgx/enclave-sample/App/wamr-bundle.md diff --git a/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp b/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp index 35096619..e3c37a1f 100644 --- a/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp +++ b/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp @@ -16,6 +16,7 @@ #include "Enclave_u.h" #include "sgx_urts.h" +#include "pal_api.h" #ifndef TRUE #define TRUE 1 @@ -33,6 +34,12 @@ static sgx_enclave_id_t g_eid = 0; +sgx_enclave_id_t +pal_get_enclave_id(void) +{ + return g_eid; +} + void ocall_print(const char* str) { @@ -734,3 +741,163 @@ fail1: return 0; } +int +wamr_pal_get_version(void) +{ + return WAMR_PAL_VERSION; +} + +int +wamr_pal_init(const struct wamr_pal_attr *args) +{ + sgx_enclave_id_t *p_eid = &g_eid; + + if (enclave_init(&g_eid) < 0) { + std::cout << "Fail to initialize enclave." << std::endl; + return 1; + } +} + +int +wamr_pal_create_process(struct wamr_pal_create_process_args *args) +{ + uint32_t stack_size = 16 * 1024, heap_size = 16 * 1024; + int log_verbose_level = 2; + bool is_repl_mode = false, alloc_with_pool = false; + const char *dir_list[8] = { NULL }; + uint32_t dir_list_size = 0; + const char *env_list[8] = { NULL }; + uint32_t env_list_size = 0; + uint32_t max_thread_num = 4; + char *wasm_files[16]; + void *wasm_module_inst[16]; + + int argc = 2; + char *argv[argc] = { (char*)"./iwasm", (char *)args->argv[0] }; + + uint8_t *wasm_files_buf = NULL; + void *wasm_modules = NULL; + int len = 0, i; + + char *temp = (char *)args->argv[0]; + while(temp) { + len++; + temp=(char *)args->argv[len]; + } + + if (len > sizeof(wasm_files)/sizeof(char *)) { + printf("Number of input files is out of range\n"); + return -1; + } + + for (i = 0; i < len; ++i) { + wasm_files[i] = (char *)args->argv[i]; + } + + /* Init runtime */ + if (!init_runtime(alloc_with_pool, max_thread_num)) { + printf("Failed to init runtime\n"); + return -1; + } + + /* Set log verbose level */ + if (!set_log_verbose_level(log_verbose_level)) { + printf("Failed to set log level\n"); + destroy_runtime(); + return -1; + } + + for (i = 0; i < len; ++i) { + uint8_t *wasm_file_buf = NULL; + uint32_t wasm_file_size; + void *wasm_module = NULL; + char error_buf[128] = { 0 }; + + /* Load WASM byte buffer from WASM bin file */ + if (!(wasm_file_buf = (uint8_t *)read_file_to_buffer + (wasm_files[i], &wasm_file_size))) { + printf("Failed to read file to buffer\n"); + destroy_runtime(); + return -1; + } + + /* Load module */ + if (!(wasm_module = load_module(wasm_file_buf, wasm_file_size, + error_buf, sizeof(error_buf)))) { + printf("%s\n", error_buf); + free(wasm_file_buf); + destroy_runtime(); + return -1; + } + + /* Set wasi arguments */ + if (!set_wasi_args(wasm_module, dir_list, dir_list_size, + env_list, env_list_size, argv, argc)) { + printf("%s\n", "set wasi arguments failed.\n"); + unload_module(wasm_module); + free(wasm_file_buf); + destroy_runtime(); + return -1; + } + + /* Instantiate module */ + if (!(wasm_module_inst[i] = instantiate_module(wasm_module, + stack_size, heap_size, + error_buf, + sizeof(error_buf)))) { + printf("%s\n", error_buf); + unload_module(wasm_module); + free(wasm_file_buf); + destroy_runtime(); + return -1; + } + + app_instance_main(wasm_module_inst[i], argc, argv); + + /* Deinstantiate module */ + deinstantiate_module(wasm_module_inst[i]); + unload_module(wasm_module); + free(wasm_file_buf); + } + + destroy_runtime(); + return 0; +} + +int +wamr_pal_destroy(void) +{ + //sgx_destroy_enclave(g_eid); + return 0; +} + +int +wamr_pal_exec(struct wamr_pal_exec_args *args) +{ + //app_instance_main(wasm_module_inst[i], argc, argv); + return 0; +} + +int +wamr_pal_kill(int pid, int sig) +{ + //deinstantiate_module(wasm_module_inst[i]); + //unload_module(wasm_module); + //free(wasm_file_buf); + return 0; +} + +int pal_get_version(void) __attribute__((weak, alias ("wamr_pal_get_version"))); + +int pal_init(const struct wamr_pal_attr *attr)\ +__attribute__ ((weak, alias ("wamr_pal_init"))); + +int pal_create_process(struct wamr_pal_create_process_args *args)\ +__attribute__ ((weak, alias ("wamr_pal_create_process"))); + +int pal_exec(struct wamr_pal_exec_args *args)\ +__attribute__ ((weak, alias ("wamr_pal_exec"))); + +int pal_kill(int pid, int sig) __attribute__ ((weak, alias ("wamr_pal_kill"))); + +int pal_destroy(void) __attribute__ ((weak, alias ("wamr_pal_destroy"))); diff --git a/product-mini/platforms/linux-sgx/enclave-sample/App/README.md b/product-mini/platforms/linux-sgx/enclave-sample/App/README.md new file mode 100644 index 00000000..49ec39a2 --- /dev/null +++ b/product-mini/platforms/linux-sgx/enclave-sample/App/README.md @@ -0,0 +1,206 @@ +# WAMR as an Enclave Runtime for Rune + +## Build WAMR vmcore (iwasm) for Linux-SGX + +### SIM Mode + +The default SGX mode in WAMR is the SIM mode. Build the source code and enclave example, please refer to [this guild](https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/linux_sgx.md#build-wamr-vmcore-iwasm-for-linux-sgx). + +### HW Mode + +Please do the following changes before execute [this guild](https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/linux_sgx.md#build-wamr-vmcore-iwasm-for-linux-sgx). + +```shell +diff --git a/product-mini/platforms/linux-sgx/enclave-sample/Makefile b/product-mini/platforms/linux-sgx/enclave-sample/Makefile +index f06b5b8..f247f3e 100644 +--- a/product-mini/platforms/linux-sgx/enclave-sample/Makefile ++++ b/product-mini/platforms/linux-sgx/enclave-sample/Makefile +@@ -4,7 +4,7 @@ + ######## SGX SDK Settings ######## + + SGX_SDK ?= /opt/intel/sgxsdk +-SGX_MODE ?= SIM ++SGX_MODE ?= HW + SGX_ARCH ?= x64 + SGX_DEBUG ?= 0 + SPEC_TEST ?= 0 +``` + +```shell +diff --git a/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal b/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal +index a64d577..747d995 100644 +--- a/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal ++++ b/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal +@@ -4,7 +4,7 @@ + ######## SGX SDK Settings ######## + + SGX_SDK ?= /opt/intel/sgxsdk +-SGX_MODE ?= SIM ++SGX_MODE ?= HW + SGX_ARCH ?= x64 + SGX_DEBUG ?= 0 + SPEC_TEST ?= 0 + +``` + +```shell +diff --git a/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp b/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp +index c321575..3b41c30 100644 +--- a/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp ++++ b/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp +@@ -31,6 +31,7 @@ + #define MAX_PATH 1024 + + #define TEST_OCALL_API 0 ++#define SGX_DEBUG_FLAG 1 + +``` + +After building, please sign enclave.so to generate enclave.signed.so which is needed in PAL + +```shell +/opt/intel/sgxsdk/bin/x64/sgx_sign sign -key Enclave/Enclave_private.pem -enclave enclave.so -out enclave.signed.so -config Enclave/Enclave.config.xml +``` + +--- + +## Build PAL dynamically linked shared object + +To build WAMR as an Enclave Runtime for [Inclavare Containers](https://github.com/alibaba/inclavare-containers), we should implement the [PAL interface](https://github.com/alibaba/inclavare-containers/blob/master/rune/libenclave/internal/runtime/pal/spec_v2.md) in WAMR for rune to call the PAL to create the enclave with WAMR and run applications. + +```shell +g++ -shared -fPIC -o libwamr-pal.so App/*.o libvmlib_untrusted.a -L/opt/intel/sgxsdk/lib64 -lsgx_urts -lpthread -lssl -lcrypto +cp ./libwamr-pal.so /usr/lib/libwamr-pal.so +``` + +Note: `/opt/intel/sgxsdk/` is where you installed the SGX SDK + +--- + +## Build WAMR application + +To Build a WAMR application, please refer to [this guide](https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/build_wasm_app.md#build-wasm-applications) + +To run a WAMR application with Intel SGX enclave by `rune`, please compile the `.wasm` file to `.aot` file, refer to [this guide](https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/build_wasm_app.md#compile-wasm-to-aot-module) + +--- + +## Build WAMR container image + +Under the `enclave-sample` directory, to create the WAMR docker images to load the `enclave.signed.so` and target application wasm files, please type the following commands to create a `Dockerfile`: + +For centos: + +```shell +cat >Dockerfile < Dockerfile < 0, then success; otherwise, it is an invalid version. + */ +int wamr_pal_get_version(void); + +/* + * WAMR PAL attributes + */ +typedef struct wamr_pal_attr { + // WAMR instance directory. + // + // The default value is "."; that is, the current working directory + const char *instance_dir; + // Log level. + // + // Specifies the log verbose level (0 to 5, default is 2) + // large level with more log + // + const char *log_level; +} wamr_pal_attr_t; + +#define WAMR_PAL_ATTR_INITVAL { \ + .instance_dir = ".", \ + .log_level = 2 \ +} + +/* + * The struct which consists of file descriptors of standard I/O + */ +typedef struct wamr_stdio_fds { + int stdin_fd; + int stdout_fd; + int stderr_fd; +} wamr_stdio_fds_t; + +/* + * The struct which consists of arguments needed by wamr_pal_create_process + */ +struct wamr_pal_create_process_args { + + // Path to new process. + // + // The path of the command which will be created as a new process. + // + // Mandatory field. Must not be NULL. + const char *path; + + // Argments array pass to new process. + // + // The arguments to the command. By convention, the argv[0] should be the program name. + // And the last element of the array must be NULL to indicate the length of array. + // + // Mandatory field. Must not be NULL. + const char **argv; + + // Untrusted environment variable array pass to new process. + // + // The untrusted env vars to the command. And the last element of the array must be + // NULL to indicate the length of array. + // + // Optional field. + const char **env; + + // File descriptors of the redirected standard I/O (i.e., stdin, stdout, stderr) + // + // If set to NULL, will use the original standard I/O file descriptors. + // + // Optional field. + const struct wamr_stdio_fds *stdio; + + // Output. Pid of new process in libos. + // + // If wamr_pal_create_process returns success, pid of the new process will + // be updated. + // + // Mandatory field. Must not be NULL. + int *pid; +}; + +/* + * The struct which consists of arguments needed by wamr_pal_exec + */ +struct wamr_pal_exec_args { + // Pid of new process created with wamr_pal_create_process. + // + // Mandatory field. + int pid; + + // Output. The exit status of the command. The semantic of + // this value follows the one described in wait(2) man + // page. For example, if the program terminated normally, + // then WEXITSTATUS(exit_status) gives the value returned + // from a main function. + // + // Mandatory field. Must not be NULL. + int *exit_value; +}; + +int wamr_pal_init(const struct wamr_pal_attr *args); + +int wamr_pal_create_process(struct wamr_pal_create_process_args *args); + +int wamr_pal_destroy(void); + +int wamr_pal_exec(struct wamr_pal_exec_args *args); + +int wamr_pal_kill(int pid, int sig); + +int pal_get_version(void); + +int pal_init(const struct wamr_pal_attr *attr); + +int pal_create_process(struct wamr_pal_create_process_args *args); + +int pal_exec(struct wamr_pal_exec_args *args); + +int pal_kill(int pid, int sig); + +int pal_destroy(void); + + +#ifdef __cplusplus +} +#endif + +#endif /* __WAMR_PAL_API_H__ */ + diff --git a/product-mini/platforms/linux-sgx/enclave-sample/App/wamr-bundle.md b/product-mini/platforms/linux-sgx/enclave-sample/App/wamr-bundle.md new file mode 100644 index 00000000..0c64f2ee --- /dev/null +++ b/product-mini/platforms/linux-sgx/enclave-sample/App/wamr-bundle.md @@ -0,0 +1,65 @@ +# Run WAMR bundle for Rune + +## Create WAMR Application bundle + +In order to use `rune` you must have your container image in the format of an OCI bundle. If you have Docker installed you can use its `export` method to acquire a root filesystem from an existing WAMR application container image. + +```shell +# create the top most bundle directory +mkdir -p "$HOME/rune_workdir" +cd "$HOME/rune_workdir" +mkdir rune-container +cd rune-container + +# create the rootfs directory +mkdir rootfs + +# export wamr application image via Docker into the rootfs directory +docker export $(docker create ${wamr_application_image}) | sudo tar -C rootfs -xvf - +``` + +After a root filesystem is populated you just generate a spec in the format of a config.json file inside your bundle. `rune` provides a spec command which is similar to `runc` to generate a template file that you are then able to edit. + +```shell +rune spec +``` + +To find features and documentation for fields in the spec please refer to the [specs](https://github.com/opencontainers/runtime-spec) repository. + +In order to run the target applications in WAMR with `rune`, you need to change the entrypoint from `sh` to `/run/rune/${wasm_app1.wasm}`, and in order to run multi-applications in one runtime with enclave, change it to `/run/rune/${wasm_app1.aot}`, `/run/rune/${wasm_app2.aot}` ... + +```yaml + "process": { + "args": [ + "/run/rune/demo.aot" + ], + } +``` + +and then configure enclave runtime as following: + +```yaml + "annotations": { + "enclave.type": "intelSgx", + "enclave.runtime.path": "/usr/lib/libwamr-pal.so", + "enclave.runtime.args": "./" + } +``` + +where: + +- @enclave.type: specify the type of enclave hardware to use, such as `intelSgx`. +- @enclave.runtime.path: specify the path to enclave runtime to launch. For an WAMR application, you need to specify the path to `libwamr-pal.so`. +- @enclave.runtime.args: specify the specific arguments to enclave runtime, separated by the comma. + +--- + +## Run WAMR Application + +Assuming you have an OCI bundle from the previous step you can execute the container in this way. + +```shell +cd "$HOME/rune_workdir/rune-container" +sudo rune run ${wamr_application_container_name} +``` +