urma_init_attr_t init_attr = {
    .token = 0,
    .uasid = 0
};
urma_init(&init_attr);

int dev_num = 1;
urma_device_t **device_list = urma_get_device_list(&dev_num);
// 用户从设备列表中选取设备urma_dev
urma_device_attr_t dev_attr = {0};
urma_query_device(urma_dev, &dev_attr);
uint32_t eid_cnt = 1;
urma_eid_info_t *eid_list = urma_get_eid_list(urma_dev, &eid_cnt);
// 用户从eid列表中选取eid对应的eid_index，下面以eid_list[0]为例
uint32_t eid_index = eid_list[0].eid_index;

urma_jfc_cfg_t jfc_cfg = {
    .depth = 64,
    .flag = {.value = 0},
    .ceqn = 0,
    .jfce = nullptr,
    .user_ctx = 0,
};
urma_jfc_t *jfc = urma_create_jfc(urma_ctx, &jfc_cfg);

static urma_token_t test_token = {
    .token = 0xABCDEF, // 由用户确定
};
urma_jfr_cfg_t jfr_cfg = {
    .depth = 64,
    .flag.bs.tag_matching = URMA_NO_TAG_MATCHING,
    .flag.bs.order_type = 0,
    .trans_mode = URMA_TM_RM,
    .min_rnr_timer = URMA_TYPICAL_MIN_RNR_TIMER,
    .jfc = jfc,
    .token_value = test_token,
    .id = 0,
    .max_sge = 1
};
urma_jfr_t *jfr = urma_create_jfr(urma_ctx, &jfr_cfg);

urma_jfs_cfg_t jfs_cfg = {
    .depth = 256,
    .flag.bs.order_type = 0,
    .flag.bs.multi_path = 0,
    .trans_mode = URMA_TM_RM,
    .priority = URMA_MAX_PRIORITY, /* Highest priority */
    .max_sge = 1,
    .max_inline_data = 0,
    .rnr_retry = URMA_TYPICAL_RNR_RETRY,
    .err_timeout = URMA_TYPICAL_ERR_TIMEOUT,
    .jfc = jfc,
    .user_ctx = (uint64_t)NULL
};
urma_jetty_cfg_t jetty_cfg = {
    .flag.bs.share_jfr = 1,
    .jfs_cfg = jfs_cfg,
    .shared.jfr = jfr
};
urma_jetty_t *jetty = urma_create_jetty(urma_ctx, &jetty_cfg);

// 以分配4KB对齐的1GB buffer为例
#define PAGE_SIZE (0x1 << PAGE_SHIFT) // 4KB
#define MEM_SIZE 0x40000000 // 1GB
void *va = memalign(PAGE_SIZE, MEM_SIZE);
(void)memset(va, 0, MEM_SIZE);
urma_reg_seg_flag_t flag = {
    .bs.token_policy = URMA_TOKEN_NONE,
    .bs.cacheable = URMA_NON_CACHEABLE,
    .bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE | URMA_ACCESS_ATOMIC,
    .bs.token_id_valid = 0,
    .bs.reserved = 0
};
urma_seg_cfg_t seg_cfg = {
    .va = (uint64_t)va,
    .len = MEM_SIZE,
    .token_id = NULL,
    .token_value = test_token,
    .flag = flag,
    .user_ctx = (uintptr_t)NULL,
    .iova = 0
};
urma_target_seg_t *local_tseg = urma_register_seg(urma_ctx, &seg_cfg);

// 获取对端info
urma_seg_t remote_seg = {
    .ubva.eid = info-\>eid;
    .ubva.uasid = info-\>uasid;
    .ubva.va = info-\>seg_va;
    .len = info-\>seg_len;
    .attr.value = info-\>seg_flag;
    .token_id = info-\>seg_token_id;
};
urma_jetty_id_t remote_jetty_id = info-\>jetty_id;

urma_rjetty_t remote_jetty = {
    .jetty_id = remote_jetty_id,
    .trans_mode = URMA_TM_RM,
    .type = URMA_JETTY,
    .tp_type = URMA_RTP,
    .flag.bs.order_type = 0,
    .flag.bs.share_tp = 0
};
urma_target_jetty_t *t_jetty = urma_import_jetty(urma_ctx, &remote_jetty, &test_token);
urma_import_seg_flag_t flag = {
    .bs.cacheable = URMA_NON_CACHEABLE,
    .bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE | URMA_ACCESS_ATOMIC,
    .bs.mapping = URMA_SEG_NOMAP,
    .bs.reserved = 0
};
urma_target_seg_t *import_tseg = urma_import_seg(urma_ctx, &remote_seg, &test_token, 0, flag);

// read/write共同参数准备
urma_sge_t src_sge = {
    .addr = (uint64_t)va,
    .len = MSG_SIZE,
    .tseg = local_tseg
};
urma_sge_t dst_sge = {
    .addr = remote_seg.ubva.va,
    .len = MSG_SIZE,
    .tseg = import_tseg
};
urma_sg_t src_sg = {
    .sge = &src_sge,
    .num_sge = 1
};
urma_sg_t dst_sg = {
    .sge = &dst_sge,
    .num_sge = 1
};
/* 先示例write */
// write是把本端src_sg的数据写到对端dst_sg
urma_rw_wr_t rw = {
    .src = src_sg,
    .dst = dst_sg
};
urma_jfs_wr_t wr = {
    .opcode = URMA_OPC_WRITE,
    .flag.bs.complete_enable = 1,
    .flag.bs.inline_flag = 0,
    .tjetty = t_jetty,
    .rw = rw,
    .next = NULL
};
urma_jfs_wr_t *bad_wr = NULL;
urma_post_jetty_send_wr(jetty, &wr, &bad_wr);
urma_cr_t cr = {0};
urma_poll_jfc(jfc, 1, &cr);
/* read示例 */
// read是把对端dst_sg数据读到本端src_sg
rw.src = dst_sg;
rw.dst = src_sg;
wr.rw = rw;
wr.opcode = URMA_OPC_READ;
urma_post_jetty_send_wr(ctx-\>jetty, &wr, &bad_wr);
urma_poll_jfc(jfc, 1, &cr);

/* 轮询示例 */
for (int i = 0; i < MAX_POLL_JFC_CNT; i++) {
    cnt = urma_poll_jfc(jfc, 1, &cr);
    if (cnt < 0) {
        fprintf(stderr, "Failed to poll jfc, return_value of urma_poll_jfc is %d\n", cnt);
        return -1;
    } else if (cnt \> 0) {
        if (cr.status == URMA_CR_SUCCESS) {
            return 0;
        } else {
            fprintf(stderr, "Failed to poll jfc, cr_status:%d\n", cr.status);
            return -1;
        }
    }
    usleep(SLEEP_TIME);
}
/* 中断示例 */
cnt = urma_wait_jfc(jfce, 1, TIMEOUT, &ev_jfc);
if (cnt < 0 || (cnt == 1 && jfc != ev_jfc)) {
    fprintf(stderr, "Failed to wait jfc\n");
    return -1;
}
cnt = urma_poll_jfc(jfc, 1, &cr);
if (cnt <= 0 || cr.status != URMA_CR_SUCCESS) {
    return -1;
}
uint32_t ack_cnt = 1;
urma_ack_jfc((urma_jfc_t **)&ev_jfc, &ack_cnt, 1);
if (urma_rearm_jfc(jfc, false) != URMA_SUCCESS) {
    return -1;
}

// 预先post一批recv wr
uint64_t offset = MSG_SIZE;
for (int i = 0; i < RECV_BATCH_CNT; i++) {
    if (offset + MSG_SIZE \> MEM_SIZE) {
        return NULL;
    }
    src_sge.addr = (uint64_t)va + offset;
    src_sge.len = MSG_SIZE;
    src_sge.tseg = local_tseg;
    src_sg.sge = &src_sge;
    src_sg.num_sge = 1;
    wr.src = src_sg;
    wr.user_ctx = offset;
    wr.next = NULL;
    if (urma_post_jetty_recv_wr(jetty, &wr, &bad_wr) != URMA_SUCCESS) {
        fprintf(stderr, "Failed to recv %i in server jfr thread\n", i);
        return NULL;
    }
    offset += MSG_SIZE;
}
// poll到成功的cr后，补充消耗的recv wr
if (cr.opcode == URMA_CR_OPC_SEND) {
    if (urma_post_jetty_recv_wr(jetty, &wr, &bad_wr) != URMA_SUCCESS) {
        fprintf(stderr, "Failed to recv in server jetty thread\n");
        return NULL;
    }
}

urma_sge_t src_sge = {
    .addr = (uint64_t)va
    .len = MSG_SIZE,
    .tseg = local_tseg
};
urma_sg_t src_sg = {
    .sge = &src_sge,
    .num_sge = 1
};
urma_send_wr_t send_wr = {
    .src = src_sg,
    .tseg = local_tseg
};
urma_jfs_wr_t jfs_wr = {
    .opcode = URMA_OPC_SEND,
    .flag.bs.complete_enable = 1,
    .tjetty = t_jetty,
    .send = send_wr,
    .next = NULL
};
urma_jfs_wr_t *bad_jfs_wr = NULL;
urma_post_jetty_send_wr(jetty, &jfs_wr, &bad_jfs_wr);

typedef struct urma_init_attr {
    uint64_t token; /* [Optional] security token */
    uint32_t uasid; /* [Optional] uasid to set and reserve. If the parameter is 0, the system will randomly assign a non-0 value. */
} urma_init_attr_t;

typedef int urma_status_t;

#define URMA_SUCCESS 0
#define URMA_EAGAIN EAGAIN // Resource temporarily unavailable
#define URMA_ENOMEM ENOMEM // Failed to allocate memory
#define URMA_ENOPERM EPERM // Operation not permitted
#define URMA_ETIMEOUT ETIMEDOUT // Operation time out
#define URMA_EINVAL EINVAL // Invalid argument
#define URMA_EEXIST EEXIST // Exist
#define URMA_EINPROGRESS EINPROGRESS
#define URMA_FAIL 0x1000 /* 0x1000 */

typedef struct urma_device {
    char name[URMA_MAX_NAME]; /* [Public] urma device's name, the names of devices
    in different transport modes are different. */
    char path[URMA_MAX_PATH]; /* [Public] urma device's path in sysfs. */
    urma_transport_type_t type; /* [Public] urma device's transport type. */
    struct urma_provider_ops_t *ops; /* [Private] urma device driver's ops. */
    struct urma_sysfs_dev_t *sysfs_dev; /* [Private] internal device corresponding to the urma device */
} urma_device_t;

typedef enum urma_transport_type {
    URMA_TRANSPORT_INVALID = -1,
    URMA_TRANSPORT_UB = 0,
    URMA_TRANSPORT_MAX
} urma_transport_type_t;

typedef struct urma_provider_ops {
    const char *name;
    urma_provider_attr_t attr;
    urma_match_entry_t *match_table;
    urma_status_t (*init)(urma_init_attr_t *conf);
    urma_status_t (*uninit)(void);
    /* Device OPs */
    urma_status_t (*query_device)(urma_device_t *dev, urma_device_attr_t *dev_attr);
    urma_context_t *(*create_context)(urma_device_t *dev, uint32_t eid_index, int dev_fd);
    urma_status_t (*delete_context)(urma_context_t *ctx);
    urma_status_t (*get_uasid)(uint32_t *uasid); /* obsolete */
    /* Log Ops */
    urma_status_t (*register_log_func)(urma_log_cb_t func);
    urma_status_t (*unregister_log_func)(void);
} urma_provider_ops_t;

typedef struct urma_match_entry {
    uint16_t vendor_id;
    uint16_t device_id;
} urma_match_entry_t;

typedef struct urma_sysfs_dev {
    char dev_name[URMA_MAX_NAME];
    char sysfs_path[URMA_MAX_SYSFS_PATH];
    char driver_name[URMA_MAX_NAME];
    urma_transport_type_t transport_type; /* transport type */
    urma_driver_t *driver;
    urma_device_t *urma_device;
    urma_device_attr_t dev_attr;
    uint16_t device_id;
    uint16_t vendor_id;
    struct ub_list node; /* Add to device list */
    uint32_t flag;
    struct timespec time_created;
} urma_sysfs_dev_t;

typedef struct urma_driver {
    struct urma_provider_ops *ops;
    struct ub_list node; /* Add to driver list */
} urma_driver_t;

struct ub_list {
    struct ub_list *prev, *next;
};

typedef struct urma_device_attr {
    urma_guid_t guid; /* [Public] */
    urma_device_cap_t dev_cap; /* [Public] capabilities of device. */
    uint8_t port_cnt; /* [Public] port number of device. */
    struct urma_port_attr_t port_attr[MAX_PORT_CNT];
    uint32_t reserved_jetty_id_min;
    uint32_t reserved_jetty_id_max;
} urma_device_attr_t;

typedef struct urma_guid {
    uint8_t raw[URMA_GUID_SIZE];
} urma_guid_t;

typedef struct urma_device_cap {
    urma_device_feature_t feature; /* [Public] support feature of device, such as OOO, LS etc. */
    uint32_t max_jfc; /* [Public] max number of jfc supported by the device. */
    uint32_t max_jfs; /* [Public] max number of jfs supported by the device. */
    uint32_t max_jfr; /* [Public] max number of jfr supported by the device. */
    uint32_t max_jetty; /* [Public] max number of jetty supported by the device. */
    uint32_t max_jetty_grp; /* [Public] max number of jetty group supported by the device. */
    uint32_t max_jetty_in_jetty_grp; /* [Public] max number of jetty per jetty group supported by the device. */
    uint32_t max_jfc_depth; /* [Public] max depth of jfc supported by the device. */
    uint32_t max_jfs_depth; /* [Public] max depth of jfs supported by the device. */
    uint32_t max_jfr_depth; /* [Public] max depth of jfr supported by the device. */
    uint32_t max_jfs_inline_len; /* [Public] max inline length(byte) supported by the jfs. */
    uint32_t max_jfs_sge; /* [Public] max number of sge supported by the jfs. */
    uint32_t max_jfs_rsge; /* [Public] max number of remote sge supported by the jfs. */
    uint32_t max_jfr_sge; /* [Public] max number of sge supported by the jfr. */
    uint64_t max_msg_size; /* [Public] max message size supported by the device. */
    uint32_t max_read_size;
    uint32_t max_write_size;
    uint32_t max_cas_size;
    uint32_t max_swap_size;
    uint32_t max_fetch_and_add_size;
    uint32_t max_fetch_and_sub_size;
    uint32_t max_fetch_and_and_size;
    uint32_t max_fetch_and_or_size;
    uint32_t max_fetch_and_xor_size;
    urma_atomic_feature_t atomic_feat; /* [Public] support atomic feature of device */
    uint16_t trans_mode; /* [Public] bit OR of supported transport modes */
    uint16_t sub_trans_mode_cap; /* [Public] bit OR of supported transport modes cap, urma_sub_trans_mode_cap_t */
    uint16_t congestion_ctrl_alg; /* [Public] one or more mode from urma_congestion_ctrl_alg_t */
    uint32_t ceq_cnt; /* [Public] ceq_cnt */
    uint32_t max_tp_in_tpg; /* [Public] max tp in tpg */
    uint32_t max_eid_cnt; /* [Public] max eid count */
    uint64_t page_size_cap; /* [Public] page size capability, must include PAGE_SIZE(4k) */
    uint32_t max_oor_cnt; /* [Public] max OOR window size by packet, only for user tp */
    uint32_t mn; /* [Public] only for user tp */
    uint32_t max_netaddr_cnt; /* [Public] only for user tp */
    urma_order_type_cap_t rm_order_cap;
    urma_order_type_cap_t rc_order_cap;
    urma_tp_type_cap_t rm_tp_cap;
    urma_tp_type_cap_t rc_tp_cap;
    urma_tp_type_cap_t um_tp_cap;
    urma_tp_feature_t tp_feature;
} urma_device_cap_t;

typedef union urma_device_feature {
    struct {
        uint32_t oor : 1; /* [Public] URMA_OUT_OF_ORDER_RECEIVING. */
        uint32_t jfc_per_wr : 1; /* [Public] URMA_JFC_PER_WR. */
        uint32_t stride_op : 1; /* [Public] URMA_STRIDE_OP. */
        uint32_t load_store_op : 1; /* [Public] URMA_LOAD_STORE_OP. */
        uint32_t non_pin : 1; /* [Public] URMA_NON_PIN. */
        uint32_t pmem : 1; /* [Public] URMA_PERSISTENCE_MEM. */
        uint32_t jfc_inline : 1; /* [Public] URMA_JFC_INLINE. */
        uint32_t spray_en : 1; /* [Public] URMA_SPRAY_ENABLE for UDP port. */
        uint32_t selective_retrans : 1; /* [Public] URMA_SELECTIVE_RETRANS. */
        uint32_t live_migrate : 1; /* [Public] support live migration. */
        uint32_t dca : 1; /* [Public] for user tp */
        uint32_t jetty_grp : 1; /* [Public] support jetty group. */
        uint32_t error_suspend : 1; /* [Public] support suspend jetty or jfs on error. */
        uint32_t outorder_comp : 1; /* [Public] support out-of-order completion. */
        uint32_t mn : 1; /* [Public] for user tp */
        uint32_t clan : 1; /* [Public] for user tp */
        uint32_t muti_seg_per_token_id : 1;
        uint32_t ipourma_en : 1;
        uint32_t ctp_en : 1;
        uint32_t uboe : 1;
        uint32_t reserved : 12;
    } bs;
    uint32_t value;
} urma_device_feature_t;

typedef union urma_atomic_feature {
    struct {
        uint32_t cas : 1;
        uint32_t swap : 1;
        uint32_t fetch_and_add : 1;
        uint32_t fetch_and_sub : 1;
        uint32_t fetch_and_and : 1;
        uint32_t fetch_and_or : 1;
        uint32_t fetch_and_xor : 1;
        uint32_t reserved : 25;
    } bs;
    uint32_t value;
} urma_atomic_feature_t;

typedef enum urma_sub_trans_mode_cap {
    URMA_RC_TP_DST_ORDERING = 0x1, /* rc mode with tp dst ordering */
    URMA_RC_TA_DST_ORDERING = 0x1 << 1, /* rc mode with ta dst ordering */
    URMA_RC_USER_TP = 0x1 << 2, /* rc mode with user tp */
} urma_sub_trans_mode_cap_t;

typedef enum urma_congestion_ctrl_alg {
    URMA_CC_NONE = 0x1 << URMA_TP_CC_NONE,
    URMA_CC_DCQCN = 0x1 << URMA_TP_CC_DCQCN,
    URMA_CC_DCQCN_AND_NETWORK_CC = 0x1 << URMA_TP_CC_DCQCN_AND_NETWORK_CC,
    URMA_CC_LDCP = 0x1 << URMA_TP_CC_LDCP,
    URMA_CC_LDCP_AND_CAQM = 0x1 << URMA_TP_CC_LDCP_AND_CAQM,
    URMA_CC_LDCP_AND_OPEN_CC = 0x1 << URMA_TP_CC_LDCP_AND_OPEN_CC,
    URMA_CC_HC3 = 0x1 << URMA_TP_CC_HC3,
    URMA_CC_DIP = 0x1 << URMA_TP_CC_DIP,
    URMA_CC_ACC = 0x1 << URMA_TP_CC_ACC
} urma_congestion_ctrl_alg_t;

typedef union urma_order_type_cap {
    struct {
        uint32_t ot : 1;
        uint32_t oi : 1;
        uint32_t ol : 1;
        uint32_t no : 1;
        uint32_t reserved : 28;
    } bs;
    uint32_t value;
} urma_order_type_cap_t;

typedef union urma_tp_type_cap {
    struct {
        uint32_t rtp : 1;
        uint32_t ctp : 1;
        uint32_t utp : 1;
        uint32_t reserved : 29;
    } bs;
    uint32_t value;
} urma_tp_type_cap_t;

typedef union urma_tp_feature {
    struct {
        uint32_t rm_multi_path : 1;
        uint32_t rc_multi_path : 1;
        uint32_t reserved : 30;
    } bs;
    uint32_t value;
} urma_tp_feature_t;

typedef struct urma_port_attr {
    urma_mtu_t max_mtu; /* [Public] MTU_256, MTU_512, MTU_1024 etc. */
    urma_port_state_t state; /* [Public] PORT_DOWN, PORT_INIT, PORT_ACTIVE */
    urma_link_width_t active_width; /* [Public] link width: X1, X2, X4. */
    urma_speed_t active_speed; /* [Public] bandwidth. */
    urma_mtu_t active_mtu; /* [Public] current effective mtu. */
} urma_port_attr_t;

typedef enum urma_mtu {
    URMA_MTU_256 = 1,
    URMA_MTU_512,
    URMA_MTU_1024,
    URMA_MTU_2048,
    URMA_MTU_4096,
    URMA_MTU_8192,
} urma_mtu_t;

typedef enum urma_port_state {
    URMA_PORT_NOP = 0,
    URMA_PORT_DOWN,
    URMA_PORT_INIT,
    URMA_PORT_ARMED,
    URMA_PORT_ACTIVE,
    URMA_PORT_ACTIVE_DEFER,
} urma_port_state_t;

typedef enum urma_link_width {
    URMA_LINK_X1 = 0x1,
    URMA_LINK_X2 = 0x1 << 1,
    URMA_LINK_X4 = 0x1 << 2,
    URMA_LINK_X8 = 0x1 << 3,
    URMA_LINK_X16 = 0x1 << 4,
    URMA_LINK_X32 = 0x1 << 5,
} urma_link_width_t;

typedef enum urma_speed {
    URMA_SP_10M = 0,
    URMA_SP_100M,
    URMA_SP_1G,
    URMA_SP_2_5G,
    URMA_SP_5G,
    URMA_SP_10G,
    URMA_SP_14G,
    URMA_SP_25G,
    URMA_SP_40G,
    URMA_SP_50G,
    URMA_SP_100G,
    URMA_SP_200G,
    URMA_SP_400G,
    URMA_SP_800G,
} urma_speed_t;

typedef struct urma_eid_info {
    urma_eid_t eid;
    uint32_t eid_index; /* 0\~UBCORE_MAX_EID_CNT -1 */
} urma_eid_info_t;

typedef union urma_eid {
    uint8_t raw[URMA_EID_SIZE]; /* Network Order */
    struct {
        uint64_t reserved; /* If IPv4 mapped to IPv6, == 0 */
        uint32_t prefix; /* If IPv4 mapped to IPv6, == 0x0000ffff */
        uint32_t addr; /* If IPv4 mapped to IPv6, == IPv4 addr */
    } in4;
    struct {
        uint64_t subnet_prefix;
        uint64_t interface_id;
    } in6;
} urma_eid_t;

typedef struct urma_context {
    struct urma_device *dev; /* [Private] point to the corresponding urma device. */
    struct urma_ops *ops; /* [Private] operation of urma device. */
    int dev_fd; /* [Private] fd of urma device's sysfs file. */
    int async_fd; /* [Private] fd of urma device's async event file. */
    pthread_mutex_t mutex; /* [Private] mutex of urma context. */
    urma_eid_t eid; /* [Public] eid of urma device. */
    uint32_t eid_index;
    uint32_t uasid; /* [Public] uasid of current process. */
    struct urma_ref ref; /* [Private] reference count of urma context. */
    urma_context_aggr_mode_t aggr_mode; /* [Public] aggregated mode of urma context. */
} urma_context_t;

typedef struct urma_ops {
    /* OPs name */
    const char *name;
    /* Jetty OPs */
    urma_jfc_t *(*create_jfc)(urma_context_t *ctx, urma_jfc_cfg_t *jfc_cfg);
    urma_status_t (*modify_jfc)(urma_jfc_t *jfc, urma_jfc_attr_t *attr);
    urma_status_t (*delete_jfc)(urma_jfc_t *jfc);
    urma_status_t (*delete_jfc_batch)(urma_jfc_t **jfc, int jfc_num, urma_jfc_t **bad_jfc);
    urma_jfs_t *(*create_jfs)(urma_context_t *ctx, urma_jfs_cfg_t *jfs);
    urma_status_t (*modify_jfs)(urma_jfs_t *jfs, urma_jfs_attr_t *attr);
    urma_status_t (*query_jfs)(urma_jfs_t *jfs, urma_jfs_cfg_t *cfg, urma_jfs_attr_t *attr);
    int (*flush_jfs)(urma_jfs_t *jfs, int cr_cnt, urma_cr_t *cr);
    urma_status_t (*delete_jfs)(urma_jfs_t *jfs);
    urma_status_t (*delete_jfs_batch)(urma_jfs_t **jfs_arr, int jfs_num, urma_jfs_t **bad_jfs);
    urma_jfr_t *(*create_jfr)(urma_context_t *ctx, urma_jfr_cfg_t *jfr);
    urma_status_t (*modify_jfr)(urma_jfr_t *jfr, urma_jfr_attr_t *attr);
    urma_status_t (*query_jfr)(urma_jfr_t *jfr, urma_jfr_cfg_t *cfg, urma_jfr_attr_t *attr);
    urma_status_t (*delete_jfr)(urma_jfr_t *jfr);
    urma_status_t (*delete_jfr_batch)(urma_jfr_t **jfr_arr, int jfr_num, urma_jfr_t **bad_jfr);
    urma_target_jetty_t *(*import_jfr)(urma_context_t *ctx, urma_rjfr_t *rjfr, urma_token_t *token);
    urma_status_t (*unimport_jfr)(urma_target_jetty_t *target_jfr);
    urma_status_t (*advise_jfr)(urma_jfs_t *jfs, urma_target_jetty_t *tjfr);
    urma_status_t (*unadvise_jfr)(urma_jfs_t *jfs, urma_target_jetty_t *tjfr);
    urma_status_t (*advise_jfr_async)(urma_jfs_t *jfs, urma_target_jetty_t *tjfr, urma_advise_async_cb_func cb_fun,
    void *cb_arg);
    urma_jetty_t *(*create_jetty)(urma_context_t *ctx, urma_jetty_cfg_t *jetty_cfg);
    urma_status_t (*modify_jetty)(urma_jetty_t *jetty, urma_jetty_attr_t *jetty_attr);
    urma_status_t (*query_jetty)(urma_jetty_t *jetty, urma_jetty_cfg_t *cfg, urma_jetty_attr_t *attr);
    int (*flush_jetty)(urma_jetty_t *jetty, int cr_cnt, urma_cr_t *cr);
    urma_status_t (*delete_jetty)(urma_jetty_t *jetty);
    urma_status_t (*delete_jetty_batch)(urma_jetty_t **jetty_arr, int jetty_num, urma_jetty_t **bad_jetty);
    urma_target_jetty_t *(*import_jetty)(urma_context_t *ctx, urma_rjetty_t *rjetty, urma_token_t *rjetty_token);
    urma_status_t (*unimport_jetty)(urma_target_jetty_t *target_jetty);
    urma_status_t (*advise_jetty)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty);
    urma_status_t (*unadvise_jetty)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty);
    urma_status_t (*advise_jetty_async)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty,
    urma_advise_async_cb_func cb_fun, void *cb_arg);
    urma_status_t (*bind_jetty)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty);
    urma_status_t (*unbind_jetty)(urma_jetty_t *jetty);
    urma_jetty_grp_t *(*create_jetty_grp)(urma_context_t *ctx, urma_jetty_grp_cfg_t *cfg);
    urma_status_t (*delete_jetty_grp)(urma_jetty_grp_t *jetty_grp);
    urma_jfce_t *(*create_jfce)(urma_context_t *ctx);
    urma_status_t (*delete_jfce)(urma_jfce_t *jfce);
    /**
    * Get tpn of current jetty
    * @param[in] jetty: the jetty pointer created before
    * Return: 0 or positive as correct tpn; negative as get tpn failure
    */
    int (*get_tpn)(urma_jetty_t *jetty);
    int (*modify_tp)(urma_context_t *ctx, uint32_t tpn, urma_tp_cfg_t *cfg, urma_tp_attr_t *attr,
    urma_tp_attr_mask_t mask);
    /* Control plane OPs */
    urma_status_t (*get_tp_list)(urma_context_t *ctx, urma_get_tp_cfg_t *cfg, uint32_t *tp_cnt,
    urma_tp_info_t *tp_list);
    urma_status_t (*set_tp_attr)(const urma_context_t *ctx, const uint64_t tp_handle, const uint8_t tp_attr_cnt,
    const uint32_t tp_attr_bitmap, const urma_tp_attr_value_t *tp_attr);
    urma_status_t (*get_tp_attr)(const urma_context_t *ctx, const uint64_t tp_handle, uint8_t *tp_attr_cnt,
    uint32_t *tp_attr_bitmap, urma_tp_attr_value_t *tp_attr);
    urma_target_jetty_t *(*import_jetty_ex)(urma_context_t *ctx, urma_rjetty_t *rjetty, urma_token_t *token_value,
    urma_active_tp_cfg_t *active_tp_cfg);
    urma_target_jetty_t *(*import_jfr_ex)(urma_context_t *ctx, urma_rjfr_t *rjfr, urma_token_t *token_value,
    urma_active_tp_cfg_t *active_tp_cfg);
    urma_status_t (*bind_jetty_ex)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty,
    urma_active_tp_cfg_t *active_tp_cfg);
    /* Segment OPs */
    urma_token_id_t *(*alloc_token_id)(urma_context_t *ctx);
    urma_token_id_t *(*alloc_token_id_ex)(urma_context_t *ctx, urma_token_id_flag_t flag);
    urma_status_t (*free_token_id)(urma_token_id_t *token_id);
    urma_target_seg_t *(*register_seg)(urma_context_t *ctx, urma_seg_cfg_t *seg_cfg);
    urma_status_t (*unregister_seg)(urma_target_seg_t *target_seg);
    urma_target_seg_t *(*import_seg)(urma_context_t *ctx, urma_seg_t *seg, urma_token_t *token, uint64_t addr,
    urma_import_seg_flag_t flag);
    urma_status_t (*unimport_seg)(urma_target_seg_t *target_seg);
    /* Events OPs */
    urma_status_t (*get_async_event)(urma_context_t *ctx, urma_async_event_t *event);
    void (*ack_async_event)(urma_async_event_t *event);
    /* Other OPs */
    int (*user_ctl)(urma_context_t *ctx, urma_user_ctl_in_t *in, urma_user_ctl_out_t *out);
    /* Dataplane OPs */
    urma_status_t (*post_jfs_wr)(urma_jfs_t *jfs, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr);
    urma_status_t (*post_jfr_wr)(urma_jfr_t *jfr, urma_jfr_wr_t *wr, urma_jfr_wr_t **bad_wr);
    urma_status_t (*post_jetty_send_wr)(urma_jetty_t *jetty, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr);
    urma_status_t (*post_jetty_recv_wr)(urma_jetty_t *jetty, urma_jfr_wr_t *wr, urma_jfr_wr_t **bad_wr);
    int (*poll_jfc)(urma_jfc_t *jfc, int cr_cnt, urma_cr_t *cr);
    urma_status_t (*rearm_jfc)(urma_jfc_t *jfc, bool solicited_only);
    int (*wait_jfc)(urma_jfce_t *jfce, uint32_t jfc_cnt, int time_out, urma_jfc_t *jfc[]);
    void (*ack_jfc)(urma_jfc_t *jfc[], uint32_t nevents[], uint32_t jfc_cnt);
    /* Jetty async OPs */
    urma_target_jetty_t *(*import_jetty_async)(urma_notifier_t *notifier, const urma_rjetty_t *rjetty,
    const urma_token_t *token_value, uint64_t user_ctx, int timeout);
    urma_status_t (*unimport_jetty_async)(urma_target_jetty_t *target_jetty);
    urma_status_t (*bind_jetty_async)(urma_notifier_t *notifier, urma_jetty_t *jetty, urma_target_jetty_t *tjetty,
    uint64_t user_ctx, int timeout);
    urma_status_t (*unbind_jetty_async)(urma_jetty_t *jetty);
    urma_notifier_t *(*create_notifier)(urma_context_t *ctx);
    urma_status_t (*delete_notifier)(urma_notifier_t *notifier);
    int (*wait_notify)(urma_notifier_t *notifier, uint32_t cnt, urma_notify_t *notify, int timeout);
    void (*ack_notify)(uint32_t cnt, urma_notify_t *notify);
    urma_status_t (*get_eid_by_ip)(const urma_context_t *ctx, const urma_net_addr_t *net_addr, urma_eid_t *eid);
    urma_status_t (*get_ip_by_eid)(const urma_context_t *ctx, const urma_eid_t *eid, urma_net_addr_t *net_addr);
    urma_status_t (*get_smac)(const urma_context_t *ctx, uint8_t *mac);
    urma_status_t (*get_dmac)(const urma_context_t *ctx, const urma_net_addr_t *net_addr, uint8_t *mac);
} urma_ops_t;

typedef struct urma_ref {
#ifndef \_\_cplusplus
    atomic_ulong atomic_cnt;
#else
    std::atomic_ulong atomic_cnt;
#endif
} urma_ref_t;

typedef enum urma_context_aggr_mode {
    URMA_AGGR_MODE_STANDALONE,
    URMA_AGGR_MODE_ACTIVE_BACKUP,
    URMA_AGGR_MODE_BALANCE,
} urma_context_aggr_mode_t;

typedef enum urma_context_opt_name {
    URMA_OPT_AGGR_MODE,
} urma_opt_name_t;

typedef struct urma_net_addr_info {
    urma_net_addr_t netaddr;
    uint32_t index;
} urma_net_addr_info_t;

typedef struct urma_net_addr {
    sa_family_t sin_family; /* AF_INET/AF_INET6 */
    union {
        struct in_addr in4;
        struct in6_addr in6;
    };
    uint64_t vlan;
    uint8_t mac[URMA_MAC_BYTES];
    uint32_t prefix_len;
} urma_net_addr_t;

typedef struct urma_token_id {
    urma_context_t *urma_ctx;
    uint32_t token_id;
    uint64_t handle;
    urma_ref_t ref;
} urma_token_id_t;

typedef union urma_token_id_flag {
    struct {
        uint32_t multi_seg : 1;
        uint32_t reserved : 31;
    } bs;
    uint32_t value;
} urma_token_id_flag_t;

typedef struct urma_jfc_cfg {
    uint32_t depth; /* [Required] the depth of jfc, no greater than urma_device_cap_t-\>jfc_depth */
    urma_jfc_flag_t flag; /* [Optional] see urma_jfc_flag_t, set flag.value to be 0 by default */
    uint32_t ceqn; /* [Optional] event queue id, less than urma_device_cap_t-\>ceq_cnt
    set to 0 by default */
    urma_jfce_t *jfce; /* [Required] the event of jfc */
    uint64_t user_ctx; /* [Optional] private data of jfc, set to NULL by default */
} urma_jfc_cfg_t;

typedef union urma_jfc_flag {
    struct {
        uint32_t lock_free : 1;
        uint32_t jfc_inline : 1;
        uint32_t non_blocking : 1;
        uint32_t has_drv_ext : 1;
        uint32_t reserved : 28;
    } bs;
    uint32_t value;
} urma_jfc_flag_t;

typedef struct urma_jfce {
    urma_context_t *urma_ctx; /* [Private] point to urma context. */
    int fd; /* [Private] fd of completed event. */
    struct urma_ref_t ref; /* [Private] reference count of urma context. */
} urma_jfce_t;

typedef struct urma_jfc {
    urma_context_t *urma_ctx; /* [Private] point to urma context. */
    urma_jfc_id_t jfc_id; /* [Public] see urma_jetty_id. */
    urma_jfc_cfg_t jfc_cfg; /* [Public] storage jfc config. */
    uint64_t handle;
    pthread_mutex_t event_mutex;
    pthread_cond_t event_cond;
    uint32_t comp_events_acked;
    uint32_t async_events_acked;
} urma_jfc_t;

typedef struct urma_jfc_attr {
    uint32_t mask; /* mask value, refer to urma_jfc_attr_mask_t */
    uint16_t moderate_count;
    uint16_t moderate_period; /* in micro seconds */
} urma_jfc_attr_t;

typedef enum urma_jfc_attr_mask {
    JFC_MODERATE_COUNT = 0x1,
    JFC_MODERATE_PERIOD = 0x1 << 1
} urma_jfc_attr_mask_t;

typedef struct urma_async_event {
    /* may be SW queue error, may be HW port error */
    const urma_context_t *urma_ctx;
    union {
        urma_jfc_t *jfc;
        urma_jfs_t *jfs;
        urma_jfr_t *jfr;
        urma_jetty_t *jetty;
        urma_jetty_grp_t *jetty_grp;
        uint32_t port_id;
        uint32_t eid_idx;
    } element;
    urma_async_event_type_t event_type;
    void *priv;
} urma_async_event_t;

typedef enum urma_async_event_type {
    URMA_EVENT_JFC_ERR,
    URMA_EVENT_JFS_ERR,
    URMA_EVENT_JFR_ERR,
    URMA_EVENT_JFR_LIMIT,
    URMA_EVENT_JETTY_ERR,
    URMA_EVENT_JETTY_LIMIT,
    URMA_EVENT_JETTY_GRP_ERR,
    URMA_EVENT_PORT_ACTIVE,
    URMA_EVENT_PORT_DOWN,
    URMA_EVENT_DEV_FATAL,
    URMA_EVENT_EID_CHANGE, // eid change, HNM and other management roles will be modified.
    URMA_EVENT_ELR_ERR, /* Entity level error */
    URMA_EVENT_ELR_DONE /* Entity flush done */
} urma_async_event_type_t;

typedef struct urma_jfs_cfg {
    uint32_t depth; /* [Required] the depth of jfs, defaut urma_device_cap_t-\>jfs_depth */
    urma_jfs_flag_t flag; /* [Optional] see urma_jfs_flag_t definition */
    urma_transport_mode_t trans_mode; /* [Required] transport mode, must be supported by the device */
    uint8_t priority; /* [Optional] set the priority of JFS, ranging from [0, 15]
    Services with low delay need to set high priority. */
    uint8_t max_sge; /* [Optional] max sge count in one wr, defaut urma_device_cap_t-\>max_jfs_sge */
    uint8_t max_rsge; /* [Optional] max remote sge count in one wr, defaut urma_device_cap_t-\>max_jfs_sge */
    uint32_t max_inline_data; /* [Optional] the max inline data size of JFS. if the parameter is 0,
    the system will assign device's max inline data length. */
    uint8_t rnr_retry; /* [Optional] number of times that jfs will resend packets before report error,
    when the remote side is not ready to receive (RNR), ranging from [0, 7],
    the value 0 means never retry and,
    the value 7 means retry infinite number of times for RDMA devices */
    uint8_t err_timeout; /* [Optional] the timeout before report error, ranging from [0, 31],
    the actual timeout in usec is caculated by: 4.096*(2^err_timeout) */
    urma_jfc_t *jfc; /* [Required] need to specify jfc */
    uint64_t user_ctx; /* [Optional] private data of jfs */
} urma_jfs_cfg_t;

typedef union urma_jfs_flag {
    struct {
        uint32_t lock_free : 1; /* default as 0, lock protected */
        uint32_t error_suspend : 1; /* 0: error continue; 1: error suspend */
        uint32_t outorder_comp : 1; /* 0: not support; 1: support out-of-order completion */
        uint32_t order_type : 8; /* (0x0): default, auto config by driver */
        /* (0x1): OT, target ordering */
        /* (0x2): OI, initiator ordering */
        /* (0x3): OL, low layer ordering */
        /* (0x4): UNO, unreliable non ordering */
        uint32_t multi_path : 1; /* 1: multi-path, 0: single path, for ubagg only. */
        uint32_t ctp_rc_mul_path_mode : 1; /* 1: ctp rc mode multi-path */
        uint32_t reserved : 19;
    } bs;
    uint32_t value;
} urma_jfs_flag_t;

typedef enum urma_order_type {
    URMA_DEF_ORDER,
    URMA_OT, // target ordering
    URMA_OI, // initiator ordering
    URMA_OL, // low layer ordering
    URMA_NO // unreliable non ordering
} urma_order_type_t;

typedef struct urma_jfs {
    urma_context_t *urma_ctx; /* [Private] point to urma context. */
    urma_jfs_id_t jfs_id; /* [Public] see urma_jetty_id. */
    urma_jfs_cfg_t jfs_cfg; /* [Public] storage jfs config. */
    uint64_t handle;
    pthread_mutex_t event_mutex;
    pthread_cond_t event_cond;
    uint32_t async_events_acked;
} urma_jfs_t;

typedef struct urma_jfs_attr {
    uint32_t mask; /* mask value refer to urma_jfs_attr_mask_t */
    urma_jfs_state_t state;
} urma_jfs_attr_t;

typedef enum urma_jfs_attr_mask {
    JFS_STATE = 0x1
} urma_jfs_attr_mask_t;

typedef struct urma_jfr_cfg {
    uint32_t id; /* [Optional] specify jfr id. If the parameter is 0,
    the system will randomly assign a non-0 value. */
    uint32_t depth; /* [Required] total depth, include berth, default urma_device_cap_t-\>jfr_depth. */
    urma_jfr_flag_t flag; /* [Optional] whether is in TAG_matching, whether is in DC/IDC mode. */
    urma_transport_mode_t trans_mode; /* [Required] transport mode, must be supported by the device */
    uint8_t max_sge; /* [Optional] max sge count in one wr, default urma_device_cap_t-\>max_jfr_sge. */
    uint8_t min_rnr_timer; /* [Optional] the minimum RNR NACK timer, ranging from [0, 31], i.e.
    the time before jfr sends NACK to the sender for the reason of "ready to receive" */
    urma_jfc_t *jfc; /* [Required] need to specify jfc. */
    urma_token_t token_value; /* [Required] specify token_value for jfr. */
    uint64_t user_ctx; /* [Optional] private data of jfr */
} urma_jfr_cfg_t;

typedef union urma_jfr_flag {
    struct {
        uint32_t token_policy : 3; /* 0: URMA_TOKEN_NONE
        1: URMA_TOKEN_PLAIN_TEXT
        2: URMA_TOKEN_SIGNED
        3: URMA_TOKEN_ALL_ENCRYPTED
        4: URMA_TOKEN_RESERVED */
        uint32_t tag_matching : 1; /* 0: URMA_NO_TAG_MATCHING.
        1: URMA_WITH_TAG_MATCHING. */
        uint32_t lock_free : 1;
        uint32_t order_type : 8; /* (0x0): default, auto config by driver */
        /* (0x1): OT, target ordering */
        /* (0x2): OI, initiator ordering */
        /* (0x3): OL, low layer ordering */
        /* (0x4): UNO, unreliable non ordering */
        uint32_t reserved : 19;
    } bs;
    uint32_t value;
} urma_jfr_flag_t;

typedef enum urma_transport_mode {
    URMA_TM_RM = 0x1, /* Reliable message */
    URMA_TM_RC = 0x1 << 1, /* Reliable connection */
    URMA_TM_UM = 0x1 << 2, /* Unreliable message */
} urma_transport_mode_t;

typedef struct urma_jfr {
    urma_context_t *urma_ctx; /* [Private] point to urma context. */
    urma_jfr_id_t jfr_id; /* [Public] see urma_jetty_id. */
    urma_jfr_cfg_t jfr_cfg; /* [Public] storage jfr config. */
    uint64_t handle;
    pthread_mutex_t event_mutex;
    pthread_cond_t event_cond;
    uint32_t async_events_acked;
    urma_jfr_opt_t urma_jfr_opt;
} urma_jfr_t;

typedef struct urma_jfr_attr {
    uint32_t mask; // mask value refer to urma_jfr_attr_mask_t
    uint32_t rx_threshold;
    urma_jfr_state_t state;
} urma_jfr_attr_t;

typedef enum urma_jfr_attr_mask {
    JFR_RX_THRESHOLD = 0x1,
    JFR_STATE = 0x1 << 1
} urma_jfr_attr_mask_t;

typedef enum urma_jfr_state {
    URMA_JFR_STATE_RESET = 0,
    URMA_JFR_STATE_READY,
    URMA_JFR_STATE_ERROR
} urma_jfr_state_t;

typedef struct urma_rjfr {
    urma_jfr_id_t jfr_id; /* see urma_jetty_id */
    urma_transport_mode_t trans_mode;
    urma_import_jetty_flag_t flag;
    urma_tp_type_t tp_type;
} urma_rjfr_t;

typedef union urma_import_jetty_flag {
    struct {
        uint32_t token_policy : 3;
        uint32_t order_type : 8; /* (0x0): default, auto config by driver */
        /* (0x1): OT, target ordering */
        /* (0x2): OI, initiator ordering */
        /* (0x3): OL, low layer ordering */
        /* (0x4): UNO, unreliable non ordering */
        uint32_t share_tp : 1; /* 1: shared tp; 0: non-shared tp. When rc mode is not ta dst ordering,
        this flag can only be set to 0. */
        uint32_t has_drv_ext : 1; /* Driver-defined behavior for import_jetty, such as affinity control. */
        uint32_t has_user_info : 1; /* 0: no extension data.
                                       1: extension data is appended after urma_rjetty_t. */
        uint32_t reserved : 18;
    } bs;
    uint32_t value;
} urma_import_jetty_flag_t;

typedef enum urma_tp_type {
    URMA_RTP,
    URMA_CTP,
    URMA_UTP
} urma_tp_type_t;

typedef struct urma_target_jetty {
    urma_context_t *urma_ctx; /* [Private] point to urma context. */
    urma_jetty_id_t id; /* [Private] see urma_jetty_id. */
    uint64_t handle;
    urma_transport_mode_t trans_mode;
    urma_tp_t tp;
    urma_target_type_t type; // todo supplementary target type
    urma_import_jetty_flag_t flag;
    urma_jetty_grp_policy_t policy;
    urma_tp_type_t tp_type;
} urma_target_jetty_t;

typedef struct urma_jetty_id {
    urma_eid_t eid;
    uint32_t uasid; /* maybe zero(stand for kernel) or non-zero(stand for app) */
    uint32_t id;
} urma_jetty_id_t;

typedef struct urma_tp {
    uint32_t tpn; /* vtpn */
} urma_tp_t;

typedef enum urma_target_type {
    URMA_JFR = 0,
    URMA_JETTY,
    URMA_JETTY_GROUP
} urma_target_type_t;

typedef enum urma_jetty_grp_policy {
    URMA_JETTY_GRP_POLICY_RR = 0,
    URMA_JETTY_GRP_POLICY_HASH_HINT = 1
} urma_jetty_grp_policy_t;

typedef struct urma_active_tp_cfg {
    uint64_t tp_handle;
    uint64_t peer_tp_handle;
    uint64_t tag;
    urma_active_tp_attr_t tp_attr;
} urma_active_tp_cfg_t;

typedef struct urma_active_tp_attr {
    uint32_t tx_psn;
    uint32_t rx_psn;
    uint64_t reserved;
} urma_active_tp_attr_t;