使用switchless特性
switchless特性介绍
技术定义： switchless是一种通过共享内存减少REE与TEE上下文切换及数据拷贝次数，优化REE与TEE交互性能的技术。
典型应用场景： 传统应用做机密计算改造拆分成非安全侧CA与安全侧TA后
当CA业务逻辑中存在频繁调用TA接口时，调用中间过程耗时占比较大，严重影响业务性能。
当CA与TA存在频繁大块数据交换时，普通ECALL调用底层会有多次内存拷贝，导致性能低下。
针对以上两种典型场景，可以通过switchless优化交互性能，降低机密计算拆分带来的性能损耗，最佳效果可达到与拆分前同等数量级。
支持硬件平台：
- Intel SGX
- ARM TrustZone 鲲鹏920
约束限制
虽然开启switchless节省了一定时间，但它们需要额外的线程来为调用提供服务。如果工作线程忙于等待消息，将会消耗大量CPU，另外更多的工作线程通常意味着更多的CPU资源竞争和更多的线程上下文切换，反而可能损害性能，所以switchless的最佳配置是经过实际业务模型与性能测试，在资源占用与性能要求中选出平衡点。
特性配置项规格
用户调用cc_enclave_create创建Enclave时，需在feature参数中传入switchless的特性配置，配置项如下：
typedef struct _cc_sl_config_t {
uint32_t num_uworkers;
uint32_t num_tworkers;
uint32_t switchless_calls_pool_size;
uint32_t retries_before_fallback;
uint32_t retries_before_sleep;
uint32_t parameter_num;
} cc_sl_config_t;
各配置项规格如下表：
| 配置项 | 说明 |
|---|---|
| num_uworkers | 非安全侧代理工作线程数，用于执行switchless OCALL，当前该字段仅在SGX平台生效，ARM平台可以配置，但是因ARM平台暂不支持OCALL，所以配置后不会生效。 规格： ARM：最大值：512；最小值：1；默认值：8（当num_uworkers配置为0时，会使用默认参数） SGX：最大值：4294967295；最小值：1 |
| num_tworkers | 安全侧代理工作线程数，用于执行switchless ECALL。 规格： ARM：最大值：512；最小值：1；默认值：8（当num_tworkers配置为0时，会使用默认参数） SGX：最大值：4294967295；最小值：1 |
| switchless_calls_pool_size | switchless调用任务池的大小，实际可容纳switchless_calls_pool_size * 64个switchless调用任务（例：switchless_calls_pool_size=1，可容纳64个switchless调用任务）。 规格： ARM：最大值：8；最小值：1；默认值：1（配置为0时） SGX：最大值：8；最小值：1；默认值：1（当switchless_calls_pool_size配置为0时，会使用默认参数） |
| retries_before_fallback | 执行retries_before_fallback次汇编pause指令后，若switchless调用仍没有被另一侧的代理工作线程执行，就回退到switch调用模式，该字段仅在SGX平台生效。 规格： SGX：最大值：4294967295；最小值：1；默认值：20000（当retries_before_fallback配置为0时，会使用默认参数） |
| retries_before_sleep | 执行retries_before_sleep次汇编pause指令后，若代理工作线程一直没有等到有任务来，则进入休眠状态，该字段仅在SGX平台生效。 规格： SGX：最大值：4294967295；最小值：1；默认值：20000（当retries_before_sleep配置为0时，会使用默认参数） |
| parameter_num | switchless函数支持的最大参数个数，该字段仅在ARM平台生效，SGX平台无此限制。 规格： ARM：最大值：16；最小值：0 |
switchless开发流程
这里给出使用 switchless 特性开发一个 C 语言程序 switchless 的例子，方便用户理解使用 switchless 开发应用程序。
基于secGear API开发应用的具体流程请参考开发secGear应用程序.md
编写 EDL（Enclave Definition Language）文件
switchless 函数需添加'transition_using_threads'标识。
enclave { include "secgear_urts.h" from "secgear_tstdc.edl" import *; from "secgear_tswitchless.edl" import *; trusted { public int get_string_switchless([out, size=32]char *buf) transition_using_threads; }; };编写顶层文件 CMakeLists.txt
编写顶层文件 CMakeLists.txt，置于 switchless 工作目录下，用于配置编译时的处理器架构、所需的 EDL 文件等信息。
其中，EDL_FILE 是 EDL 文件，需用户指定，例子中为 switchless.edl。DPATH 是安全侧加载动态库，配置如例子中所示。
cmake_minimum_required(VERSION 3.10 FATAL_ERROR) project(switchless C) set(CMAKE_C_STANDARD 99) if (NOT DEFINED ENCLAVE) set(ENCLAVE "SGX") endif() set(SGX_SDK_DEFAULT_PATH /opt/intel/sgxsdk) set(GP_SDK_DEFAULT_PATH /opt/itrustee_sdk) set(PL_SDK_DEFAULT_PATH /root/dev/sdk) set(SGX_SSL_DEFAULT_PATH /opt/intel/sgxssl) set(GP_SSL_DEFAULT_PATH /opt/itrustee_sdk/itrustee_sdk_ssl) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) if(${ENCLAVE} STREQUAL "GP") if (NOT DEFINED SDK_PATH) set(iTrusteeSDK ${GP_SDK_DEFAULT_PATH}) else() set(iTrusteeSDK ${SDK_PATH}) endif() message("Current Platform: ARM Trustzone, iTrustee SDK PATH:${iTrusteeSDK}") if(NOT IS_DIRECTORY ${iTrusteeSDK}) message(FATAL_ERROR "Please provide the correct iTrusteeSDK path") endif() set(CC_GP ON) endif() if(${ENCLAVE} STREQUAL "SGX") if (NOT DEFINED SDK_PATH) set(SGXSDK ${SGX_SDK_DEFAULT_PATH}) else() set(SGXSDK ${SDK_PATH}) endif() message("Current Platform: Intel SGX, SGX SDK PATH:${SGXSDK}") if(NOT IS_DIRECTORY ${SGXSDK}) message(FATAL_ERROR "Please provide the correct SGXSDK path") endif() set(CC_SGX ON) endif() if(${ENCLAVE} STREQUAL "PL") if (NOT DEFINED SDK_PATH) set(PLSDK ${PL_SDK_DEFAULT_PATH}) else() set(PLSDK ${SDK_PATH}) endif() message("Current Platform: RISC-V, Penglai SDK PATH:${PLSDK}") if(NOT IS_DIRECTORY ${PLSDK}) message(FATAL_ERROR "Please provide the correct Penglai SDK path") endif() set(CC_PL ON) endif() set(CURRENT_ROOT_PATH ${CMAKE_CURRENT_SOURCE_DIR}) #set edl name set(EDL_FILE switchless.edl) set(CODEGEN codegen) if(CC_GP) set(CODETYPE trustzone) set(UUID ebc87fc2-05dc-41b3-85b9-f9f0ef481bad) add_definitions(-DPATH="${LOCAL_ROOT_PATH_INSTALL}/data/${UUID}.sec") endif() if(CC_SGX) set(CODETYPE sgx) add_definitions(-DPATH="${CMAKE_CURRENT_BINARY_DIR}/enclave/enclave.signed.so") endif() add_subdirectory(${CURRENT_ROOT_PATH}/enclave) add_subdirectory(${CURRENT_ROOT_PATH}/host)编写非安全侧代码和 CMakeLists.txt
3.1 编写 main.c
使用 cc_enclave_create 创建安全区 enclave 上下文时，需在 features 参数中传入 switchless 特性配置。使能 switchless 特性的主要工作有：创建任务池，大小由配置中的 switchless_call_pool_size 决定；根据 num_uworkers / num_tworkers 创建 Untrust / Trust 工作线程池等。
#include <stdio.h> #include <unistd.h> #include <linux/limits.h> #include <sys/time.h> #include <string.h> #include "enclave.h" #include "secgear_uswitchless.h" #include "secgear_shared_memory.h" #include "switchless_u.h" #define BUF_LEN 32 int main() { int retval = 0; char *path = PATH; char buf[BUF_LEN]; cc_enclave_t *context = NULL; context = (cc_enclave_t *)malloc(sizeof(cc_enclave_t)); cc_enclave_result_t res = CC_FAIL; char real_p[PATH_MAX]; /* switchless configuration */ cc_sl_config_t sl_cfg = CC_USWITCHLESS_CONFIG_INITIALIZER; sl_cfg.num_tworkers = 2; /* 2 tworkers */ sl_cfg.sl_call_pool_size_qwords = 2; /* 2 * 64 tasks */ enclave_features_t features = {ENCLAVE_FEATURE_SWITCHLESS, (void *)&sl_cfg}; res = cc_enclave_create(real_p, AUTO_ENCLAVE_TYPE, 0, SECGEAR_DEBUG_FLAG, &features, 1, context); ... char *shared_buf = (char *)cc_malloc_shared_memory(context, BUF_LEN); ... /* switchless ecall */ res = get_string_switchless(context, &retval, shared_buf); if (res != CC_SUCCESS || retval != (int)CC_SUCCESS) { printf("Switchless ecall error\n"); } else { printf("shared_buf: %s\n", shared_buf); } res = cc_free_shared_memory(context, shared_buf); ... res = cc_enclave_destroy(context); ... return res; }3.2 编写非安全侧 CMakeLists.txt
# 设置编译环境变量 #set auto code prefix set(PREFIX switchless) #set host exec name set(OUTPUT secgear_switchless) #set host src code set(SOURCE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/main.c) # 使用代码生成工具生成辅助代码。CODEGEN 和 CODETYPE 变量也在顶层 CMakeLists.txt 中定义。--search-path 用于指定 switchless.edl 中导入依赖的其他 EDL 文件路径 # set auto code if(CC_GP) set(AUTO_FILES ${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}_u.h ${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}_u.c ${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}_args.h) add_custom_command(OUTPUT ${AUTO_FILES} DEPENDS ${CURRENT_ROOT_PATH}/${EDL_FILE} COMMAND ${CODEGEN} --${CODETYPE} --untrusted ${CURRENT_ROOT_PATH}/${EDL_FILE} --search-path /usr/include/secGear) endif() if(CC_SGX) set(AUTO_FILES ${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}_u.h ${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}_u.c) add_custom_command(OUTPUT ${AUTO_FILES} DEPENDS ${CURRENT_ROOT_PATH}/${EDL_FILE} COMMAND ${CODEGEN} --${CODETYPE} --untrusted ${CURRENT_ROOT_PATH}/${EDL_FILE} --search-path /usr/include/secGear --search-path ${SGXSDK}/include) endif() # 设置编译选项和链接选项 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIE -L/usr/lib64") set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS} -s") # 编译链接引用目录 if(CC_GP) if(${CMAKE_VERSION} VERSION_LESS "3.13.0") link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) endif() add_executable(${OUTPUT} ${SOURCE_FILE} ${AUTO_FILES}) target_include_directories(${OUTPUT} PRIVATE ${CMAKE_BINARY_DIR}/host /usr/include/secGear ${CMAKE_CURRENT_BINARY_DIR}) if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0") target_link_directories(${OUTPUT} PRIVATE /usr/lib64 ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) endif() endif() if(CC_SGX) if(${CMAKE_VERSION} VERSION_LESS "3.13.0") link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) endif() add_executable(${OUTPUT} ${SOURCE_FILE} ${AUTO_FILES}) target_include_directories(${OUTPUT} PRIVATE /usr/include/secGear /opt/intel/sgxsdk/include ${CMAKE_CURRENT_BINARY_DIR}) if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0") target_link_directories(${OUTPUT} PRIVATE ${CMAKE_LIBRARY_OUTPUT_DIRECTORY} ${SGXSDK}/lib64) endif() endif() if(CC_SIM) target_link_libraries(${OUTPUT} secgearsim pthread) else() if(CC_GP) target_link_libraries(${OUTPUT} secgear pthread) endif() if(CC_SGX) target_link_libraries(${OUTPUT} secgear pthread -Wl,--whole-archive -lsgx_uswitchless -Wl,--no-whole-archive -lsgx_urts) endif() endif() # 指定二进制安装目录 set_target_properties(${OUTPUT} PROPERTIES SKIP_BUILD_RPATH TRUE) if(CC_GP) install(TARGETS ${OUTPUT} RUNTIME DESTINATION ${LOCAL_ROOT_PATH_INSTALL}/vendor/bin/ PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ) endif() if(CC_SGX) install(TARGETS ${OUTPUT} RUNTIME DESTINATION ${CMAKE_BINARY_DIR}/bin/ PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ) endif()编写安全侧代码、CMakeLists.txt 和配置文件，放在 enclave 目录
4.1 编写安全侧代码 enclave.c
#include <stdio.h> #include <string.h> #include "switchless_t.h" #define TA_HELLO_WORLD "secgear hello world!" #define BUF_MAX 32 int get_string_switchless(char *shared_buf) { strncpy(shared_buf, TA_HELLO_WORLD, strlen(TA_HELLO_WORLD) + 1); return 0; }4.2 编写安全侧 CMakeLists.txt
#set auto code prefix set(PREFIX switchless) #set sign key set(PEM Enclave_private.pem) #set sign tool set(SIGN_TOOL sign_tool.sh) #set enclave src code set(SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/enclave.c) #set log level set(PRINT_LEVEL 3) add_definitions(-DPRINT_LEVEL=${PRINT_LEVEL}) # WHITE_LIST_x：为设置iTrustee的二进制白名单，用来设置哪些非安全侧的二进制可以调用安全侧的动态库。 # WHITE_LIST_OWNER：设置运行二进制所属的用户权限，只有该用户才可以调用安全侧动态库。 if(CC_GP) #set signed output set(OUTPUT ${UUID}.sec) set(WHITE_LIST_0 ${LOCAL_ROOT_PATH_INSTALL}/vendor/bin/secgear_switchless) set(WHITE_LIST_OWNER root) set(WHITELIST WHITE_LIST_0) set(AUTO_FILES ${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}_t.h ${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}_t.c ${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}_args.h) add_custom_command(OUTPUT ${AUTO_FILES} DEPENDS ${CURRENT_ROOT_PATH}/${EDL_FILE} COMMAND ${CODEGEN} --${CODETYPE} --trusted ${CURRENT_ROOT_PATH}/${EDL_FILE} --search-path /usr/include/secGear) endif() # SGX 安全侧动态库签名 if(CC_SGX) set(OUTPUT enclave.signed.so) set(AUTO_FILES ${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}_t.h ${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}_t.c) add_custom_command(OUTPUT ${AUTO_FILES} DEPENDS ${CURRENT_ROOT_PATH}/${EDL_FILE} COMMAND ${CODEGEN} --${CODETYPE} --trusted ${CURRENT_ROOT_PATH}/${EDL_FILE} --search-path /usr/include/secGear --search-path ${SGXSDK}/include) endif() # 设置编译选项 set(COMMON_C_FLAGS "-W -Wall -Werror -fno-short-enums -fno-omit-frame-pointer -fstack-protector-strong \ -Wstack-protector --param ssp-buffer-size=4 -frecord-gcc-switches -Wextra -nostdinc -nodefaultlibs \ -fno-peephole -fno-peephole2 -Wno-main -Wno-error=unused-parameter -D_FORTIFY_SOURCE=2 -O2 \ -Wno-error=unused-but-set-variable -L/usr/lib64 -Wno-error=format-truncation=") set(COMMON_C_LINK_FLAGS "-Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack -Wl,-nostdlib -nodefaultlibs -nostartfiles") # itrustee 需生成 manifest.txt。指定 itrustee 编译选项和头文件、链接文件的搜索路径 if(CC_GP) set(CMAKE_C_FLAGS "${COMMON_C_FLAGS} -march=armv8-a") set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS} -s -fPIC") set(CMAKE_SHARED_LINKER_FLAGS "${COMMON_C_LINK_FLAGS} -Wl,-s") set(ITRUSTEE_TEEDIR ${iTrusteeSDK}/) set(ITRUSTEE_LIBC ${iTrusteeSDK}/thirdparty/open_source/musl/libc) if(${CMAKE_VERSION} VERSION_LESS "3.13.0") link_directories(${CMAKE_BINARY_DIR}/lib/) endif() add_library(${PREFIX} SHARED ${SOURCE_FILES} ${AUTO_FILES}) target_include_directories( ${PREFIX} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}/enclave /usr/include/secGear ${ITRUSTEE_TEEDIR}/include/TA ${ITRUSTEE_TEEDIR}/include/TA/huawei_ext ${ITRUSTEE_LIBC}/arch/aarch64 ${ITRUSTEE_LIBC}/ ${ITRUSTEE_LIBC}/arch/arm/bits ${ITRUSTEE_LIBC}/arch/generic ${ITRUSTEE_LIBC}/arch/arm) if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0") target_link_directories(${PREFIX} PUBLIC /usr/lib64) endif() foreach(WHITE_LIST ${WHITELIST}) add_definitions(-D${WHITE_LIST}="${${WHITE_LIST}}") endforeach(WHITE_LIST) add_definitions(-DWHITE_LIST_OWNER="${WHITE_LIST_OWNER}") target_link_libraries(${PREFIX} secgear_tee) #for trustzone compiling, you should contact us to get config and private_key.pem for test, so we will not sign and install binary in this example # add_custom_command(TARGET ${PREFIX} POST_BUILD COMMAND bash ${SIGN_TOOL} -d sign -x trustzone -i ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/lib${PREFIX}.so -c ${CMAKE_CURRENT_SOURCE_DIR}/manifest.txt -m ${CMAKE_CURRENT_SOURCE_DIR}/config_cloud.ini -o ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${OUTPUT}) install(FILES ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${OUTPUT} #DESTINATION /data DESTINATION ${LOCAL_ROOT_PATH_INSTALL}/data PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) endif() if(CC_SGX) set(SGX_DIR ${SGXSDK}) set(CMAKE_C_FLAGS "${COMMON_C_FLAGS} -m64 -fvisibility=hidden") set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS} -s") set(LINK_LIBRARY_PATH ${SGX_DIR}/lib64) if(CC_SIM) set(Trts_Library_Name sgx_trts_sim) set(Service_Library_Name sgx_tservice_sim) else() set(Trts_Library_Name sgx_trts) set(Service_Library_Name sgx_tservice) endif() set(Crypto_Library_Name sgx_tcrypto) set(CMAKE_SHARED_LINKER_FLAGS "${COMMON_C_LINK_FLAGS} -Wl,-z,defs -Wl,-pie -Bstatic -Bsymbolic -eenclave_entry \ -Wl,--export-dynamic -Wl,--defsym,__ImageBase=0 -Wl,--gc-sections \ -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/Enclave.lds") if(${CMAKE_VERSION} VERSION_LESS "3.13.0") link_directories(${LINK_LIBRARY_PATH}) endif() add_library(${PREFIX} SHARED ${SOURCE_FILES} ${AUTO_FILES}) target_include_directories(${PREFIX} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} /usr/include/secGear ${SGX_DIR}/include/tlibc ${SGX_DIR}/include/libcxx ${SGX_DIR}/include) if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0") target_link_directories(${PREFIX} PRIVATE ${LINK_LIBRARY_PATH}) endif() target_link_libraries(${PREFIX} -Wl,--whole-archive -lsgx_tswitchless ${Trts_Library_Name} -Wl,--no-whole-archive -Wl,--start-group -lsgx_tstdc -lsgx_tcxx -l${Crypto_Library_Name} -l${Service_Library_Name} -Wl,--end-group) add_custom_command(TARGET ${PREFIX} POST_BUILD COMMAND umask 0177 COMMAND openssl genrsa -3 -out ${PEM} 3072 COMMAND bash ${SIGN_TOOL} -d sign -x sgx -i ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/lib${PREFIX}.so -k ${PEM} -o ${OUTPUT} -c ${CMAKE_CURRENT_SOURCE_DIR}/Enclave.config.xml) endif() if(NOT DEFINED CC_PL) set_target_properties(${PREFIX} PROPERTIES SKIP_BUILD_RPATH TRUE) endif()
常见问题
sgx环境下开启switchless特性创建enclave后，直接销毁enclave，再使用enclave会产生core dump
sgx开启switchless需有以下两步：
- cc_enclave_create时传入switchless feature参数
- 在第一次ecall调用中初始化switchless线程调度
如果没有调用ecall函数，就直接调用cc_enclave_destroy，会在sgx库中销毁switchless调度线程时异常。
由于switchless的实际应用场景是存在频繁ecall调用的，所以初始化switchless特性后，通常会有ecall调用，不会存在问题。
文档捉虫
