io_uring 是 linux 内核 5.10 引入的异步 io 接口。相比起用户态的DPDK、SPDK，io_uring 作为内核的一部分，通过 mmap 的方式实现用户和内核共享内存，并基于 memory barrier 在这块内存上实现了两个无锁环形队列：
submission queue ring(sq) 和
completion queue ring(cq)。 sq 用于用户程序向内核提交 IO 任务，内核执行完成的任务会放入cq，用户程序从 cq 获取结果。在提交任务和返回任务结果时，用户程序和内核共用环形队列中的数据，不再需要额外的数据拷贝。此外，io_uring 还提供了两种轮询 Polling 模式，可以避免提交任务时的系统调用，以及io完成后的中断通知。
1、性能测试
1.1、FIO
iops 是指单位时间内系统能处理的I/O请求数量，用于存储设备性能测试。这里我们使用硬盘性能辅助测试工具 FIO，来直观感受异步 io: io_uring 的性能优势。
# 安装 fio
sudo apt install fio
# 运行方式
fio job_file
- 1
- 2
- 3
- 4
需要通过编写一个配置文件来预定义 FIO 将要以什么样的模式来执行任务。
FIO 的基本参数：
rw readwrite：定义 IO 类型。随机读 randread、随机写 randwrite、顺序读 read、顺序写 write、顺序读写 rw readwrite ，随机混合读写 randrwbs, blocksize：IO 的块大小。默认4ksize: IO 传输的数据大小ioengine：IO 引擎。同步模式psync、异步模式io_uringiodepth：I/O 引擎若使用异步模式，保持队列深度direct: 是否使用非缓冲 io ，默认 false 缓冲 io
编写的 posix.fio 配置文件如下
[global]
thread=1
group_reporting=1
direct=1
verify=0
time_based=1
runtime=10
bs=16K
size=16384
iodepth=64
rw=randwrite
filename=Cat
ioengine=io_uring
[test]
stonewall
description="variable bs"
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
实验结果：iops：psync 8k, io_uring 19.0k，由此可以看出异步 io 的性能优势。
1.2、rust_echo_benc
服务器性能测试方法
- 连接数
- 每个请求连接的大小
- 持续时间
epoll 与 io_uring 事件的区别
- epoll 设置完后，不更改。
- io_uring 设置一次，触发一次。
接下来，进行同步 epoll 与异步 io_uring 服务器的测试对比，代码见 liburing 测试代码
# 安装 rust_echo_benc
git clone https://github.com/haraldh/rust_echo_bench.git
cargo run --release
测试
cargo run --release -- --address "127.0.0.1:9999" --number 1000 --duration 60 --length 512
- 1
- 2
- 3
- 4
- 5
- 6
实验结果：在网络 io 方面，io_uring并不明显。在磁盘 io 方面，io_uring 具有一定的优势。
2、io_uring
io_uring 提供了三个系统调用接口 io_uring_setup、io_uring_enter、io_uring_register
2.1、io_uring_setup
在 kernel 中创建：
- 提交队列 SQ：里面每一项是 sqe(submission queue event)，描述1个任务
- 完成队列 CQ：里面每一项是 cqe(completion queue event)，描述1个任务返回结果
- 提交队列项 SQEs 数组（Submission Queue Entries）
SQ 和 CQ 采用 Ringbuffer 的结构，有 head 和 tail 两个成员，head = tail 时队列为空。每个节点保存的是 SQEs 数组的偏移量，实际的请求保存在 SQEs 数组中，这样就可以批量提交一组 SQEs 上不连续的请求。SQ 和 CQ 本身没有提供锁等同步机制，向 SQ中放入 sqe，从 CQ 中取出 cqe，都需要通过 memory barrier 来实现。
函数返回1个 fd 用于 io_uring 管理。用户将 fd 以 mmap 的方式映射到内存，实现了用户态和内核态的共享内存。
/*
- 参数1 entries：期望的 sq 长度。默认cq长度是sq的两倍
- 参数2 params: 配置io_uring，内核返回的 sq/cq 配置信息也通过它带回来
*/
int io_uring_setup(unsigned entries, struct io_uring_params *params)
struct io_uring_params {
__u32 sq_entries;
__u32 cq_entries;
__u32 flags;
__u32 sq_thread_cpu;
__u32 sq_thread_idle;
__u32 resv[5];
struct io_sqring_offsets sq_off;
struct io_cqring_offsets cq_off;
};
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
2.2、io_uring_enter
调用时，执行两个操作
- 提交 IO 请求：把 sqe 的索引尾插到 SQ 中，调用
io_uring_enter提交到内核 - 等待 IO 完成：内核将完成的 IO 放到 CQ 中，用户轮询 CQ 来等待结果
/*
- 参数1 fd：io_uring_setup返回的fd
- 参数2 to_submit: 一次提交多少个 sqe 到内核
- 参数3 min_complete: 要求内核至少等待min_complete个任务完成再返回
- 参数4 flags：接口控制行为，IORING_ENTER_GETEVENTS
*/
int io_uring_enter(unsigned int fd, u32 to_submit, u32 min_complete, u32 flags);
- 1
- 2
- 3
- 4
- 5
- 6
- 7
2.3、io_uring_register
注册用于异步 I/O 的文件或用户缓冲区
对于文件， 保持内核长时间持有该文件的索引。每次通过 sqe 向内核传递一个 fd，内核都需要通过 fd 找到对应的文件索引，完成该sqe 处理后，则将该索引释放。对于高 iops 的场景，这个开销会拖慢请求的速度。通过预先注册一组已经打开的文件。
对于缓冲区，保持内存的长期映射。内核在读写前进行page map，读写完成后，执行unmap。类似的，通过预注册，来避免多次的 map 和 unmap。
/*
- 参数1 fd：io_uring_setup返回的fd
- 参数2 opcode: 注册类型。
文件类型: IORING_REGISTER_FILES；
用户缓冲类型 buffer: IORING_REGISTER_BUFFERS
- 参数3 arg:
文件类型: 指向一个fd数组；
用户缓冲类型：指向一个struct iovec的数组。
- 参数4 nr_args：arg数组的长度
*/
int io_uring_register(unsigned int fd, unsigned int opcode,
void *arg, unsigned int nr_args);
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
2.4、使用方法：cat 程序为例
接下来，基于 io_uring 的系统调用接口进行封装，实现自定义的 uring_cat 程序
// gcc -o uring_cat uring_cat.c
// ./uring_cat filename
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/syscall.h>
#include <sys/mman.h>
#include <sys/uio.h>
#include <linux/fs.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <linux/io_uring.h>
#define URING_QUEUE_DEPTH 1024
#define BLOCK_SZ 1024
// sqring
struct app_io_sq_ring {
<span class="token keyword">unsigned</span> <span class="token operator">*</span>head<span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> <span class="token operator">*</span>tail<span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> <span class="token operator">*</span>ring_mask<span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> <span class="token operator">*</span>ring_entries<span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> <span class="token operator">*</span>flags<span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> <span class="token operator">*</span>array<span class="token punctuation">;</span>
};
// cqring
struct app_io_cq_ring {
<span class="token keyword">unsigned</span> <span class="token operator">*</span>head<span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> <span class="token operator">*</span>tail<span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> <span class="token operator">*</span>ring_mask<span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> <span class="token operator">*</span>ring_entries<span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">io_uring_cqe</span> <span class="token operator">*</span>cqes<span class="token punctuation">;</span>
};
// 提交器: cq, sq, sqe
struct submitter {
<span class="token keyword">int</span> ring_fd<span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">app_io_sq_ring</span> sq_ring<span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">app_io_cq_ring</span> cq_ring<span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">io_uring_sqe</span> <span class="token operator">*</span>sqes<span class="token punctuation">;</span>
};
-------------------
struct file_info {
off_t file_sz;
struct iovec iovecs[];
};
-------------------
// 利用系统调用执行 io_uring_setup 流程
// 1、int 0x80 中断信号
// 2、mv arg1, eax
// 3、mv arg2, ebx
// 4、call sys_call_table: sys_call_table[__NR_io_uring_setup]
int io_uring_setup(unsigned entries, struct io_uring_params *p)
{
return (int) syscall(__NR_io_uring_setup, entries, p);
}
int io_uring_enter(int ring_fd, unsigned int to_submit,
unsigned int min_complete, unsigned int flags)
{
return (int) syscall(__NR_io_uring_enter, ring_fd, to_submit, min_complete,
flags, NULL, 0);
}
int app_setup_uring(struct submitter *s) {
<span class="token keyword">struct</span> <span class="token class-name">io_uring_params</span> p<span class="token punctuation">;</span>
<span class="token function">memset</span><span class="token punctuation">(</span><span class="token operator">&</span>p<span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">,</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span>p<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// 创建sq, cq, sqes</span>
s<span class="token operator">-></span>ring_fd <span class="token operator">=</span> <span class="token function">io_uring_setup</span><span class="token punctuation">(</span>URING_QUEUE_DEPTH<span class="token punctuation">,</span> <span class="token operator">&</span>p<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>s<span class="token operator">-></span>ring_fd <span class="token operator"><</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token keyword">return</span> <span class="token operator">-</span><span class="token number">1</span><span class="token punctuation">;</span>
<span class="token comment">// 获取初始的sq,cq的大小，sq_off, cq_off起始偏移地址</span>
<span class="token keyword">int</span> sring_sz <span class="token operator">=</span> p<span class="token punctuation">.</span>sq_off<span class="token punctuation">.</span>array <span class="token operator">+</span> p<span class="token punctuation">.</span>sq_entries <span class="token operator">*</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span><span class="token keyword">unsigned</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">int</span> cring_sz <span class="token operator">=</span> p<span class="token punctuation">.</span>cq_off<span class="token punctuation">.</span>cqes <span class="token operator">+</span> p<span class="token punctuation">.</span>cq_entries <span class="token operator">*</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">io_uring_cqe</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// io_uring特性：IORING_FEAT_SINGLE_MMAP：内核通过一次mmap完成sq, cq的映射</span>
<span class="token comment">// 即sq，cq共用1块内存，则两者大小必须设置相同</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>p<span class="token punctuation">.</span>features <span class="token operator">&</span> IORING_FEAT_SINGLE_MMAP<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>cring_sz <span class="token operator">></span> sring_sz<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
sring_sz <span class="token operator">=</span> cring_sz<span class="token punctuation">;</span>
<span class="token punctuation">}</span>
cring_sz <span class="token operator">=</span> sring_sz<span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token comment">// 1、将 sq 的映射到用户空间，sq_ptr 指向sq首地址</span>
<span class="token keyword">void</span> <span class="token operator">*</span>sq_ptr <span class="token operator">=</span> <span class="token function">mmap</span><span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">,</span> sring_sz<span class="token punctuation">,</span> PROT_READ<span class="token operator">|</span>PROT_WRITE<span class="token punctuation">,</span> MAP_SHARED<span class="token operator">|</span>MAP_POPULATE<span class="token punctuation">,</span>
s<span class="token operator">-></span>ring_fd<span class="token punctuation">,</span> IORING_OFF_SQ_RING<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>sq_ptr <span class="token operator">==</span> MAP_FAILED<span class="token punctuation">)</span> <span class="token keyword">return</span> <span class="token operator">-</span><span class="token number">1</span><span class="token punctuation">;</span>
<span class="token comment">// 2、将 cq 的映射到用户空间，cq_ptr 指向cq首地址</span>
<span class="token keyword">void</span> <span class="token operator">*</span>cq_ptr<span class="token punctuation">;</span>
<span class="token comment">// 若共用一块内存，则两个指针指向相同</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>p<span class="token punctuation">.</span>features <span class="token operator">&</span> IORING_FEAT_SINGLE_MMAP<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
cq_ptr <span class="token operator">=</span> sq_ptr<span class="token punctuation">;</span>
<span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token punctuation">{<!-- --></span>
<span class="token comment">// 若使用两块内存，则重新对cq进行mmap，</span>
cq_ptr <span class="token operator">=</span> <span class="token function">mmap</span><span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">,</span> sring_sz<span class="token punctuation">,</span> PROT_READ<span class="token operator">|</span>PROT_WRITE<span class="token punctuation">,</span> MAP_SHARED<span class="token operator">|</span>MAP_POPULATE<span class="token punctuation">,</span>
s<span class="token operator">-></span>ring_fd<span class="token punctuation">,</span> IORING_OFF_CQ_RING<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>cq_ptr <span class="token operator">==</span> MAP_FAILED<span class="token punctuation">)</span> <span class="token keyword">return</span> <span class="token operator">-</span><span class="token number">1</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token keyword">struct</span> <span class="token class-name">app_io_sq_ring</span> <span class="token operator">*</span>sring <span class="token operator">=</span> <span class="token operator">&</span>s<span class="token operator">-></span>sq_ring<span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">app_io_cq_ring</span> <span class="token operator">*</span>cring <span class="token operator">=</span> <span class="token operator">&</span>s<span class="token operator">-></span>cq_ring<span class="token punctuation">;</span>
sring<span class="token operator">-></span>head <span class="token operator">=</span> sq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>sq_off<span class="token punctuation">.</span>head<span class="token punctuation">;</span>
sring<span class="token operator">-></span>tail <span class="token operator">=</span> sq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>sq_off<span class="token punctuation">.</span>tail<span class="token punctuation">;</span>
sring<span class="token operator">-></span>ring_mask <span class="token operator">=</span> sq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>sq_off<span class="token punctuation">.</span>ring_mask<span class="token punctuation">;</span>
sring<span class="token operator">-></span>ring_entries <span class="token operator">=</span> sq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>sq_off<span class="token punctuation">.</span>ring_entries<span class="token punctuation">;</span>
sring<span class="token operator">-></span>flags <span class="token operator">=</span> sq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>sq_off<span class="token punctuation">.</span>flags<span class="token punctuation">;</span>
sring<span class="token operator">-></span>array <span class="token operator">=</span> sq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>sq_off<span class="token punctuation">.</span>array<span class="token punctuation">;</span>
<span class="token comment">// 3、将 seqs 映射到用户空间</span>
s<span class="token operator">-></span>sqes <span class="token operator">=</span> <span class="token function">mmap</span><span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">,</span> p<span class="token punctuation">.</span>sq_entries <span class="token operator">*</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">io_uring_sqe</span><span class="token punctuation">)</span><span class="token punctuation">,</span>
PROT_READ <span class="token operator">|</span> PROT_WRITE<span class="token punctuation">,</span> MAP_SHARED <span class="token operator">|</span> MAP_POPULATE<span class="token punctuation">,</span> s<span class="token operator">-></span>ring_fd<span class="token punctuation">,</span> IORING_OFF_SQES<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>s<span class="token operator">-></span>sqes <span class="token operator">==</span> MAP_FAILED<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token keyword">return</span> <span class="token number">1</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
cring<span class="token operator">-></span>head <span class="token operator">=</span> cq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>cq_off<span class="token punctuation">.</span>head<span class="token punctuation">;</span>
cring<span class="token operator">-></span>tail <span class="token operator">=</span> cq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>cq_off<span class="token punctuation">.</span>tail<span class="token punctuation">;</span>
cring<span class="token operator">-></span>ring_mask <span class="token operator">=</span> cq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>cq_off<span class="token punctuation">.</span>ring_mask<span class="token punctuation">;</span>
cring<span class="token operator">-></span>ring_entries <span class="token operator">=</span> cq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>cq_off<span class="token punctuation">.</span>ring_entries<span class="token punctuation">;</span>
cring<span class="token operator">-></span>cqes <span class="token operator">=</span> cq_ptr <span class="token operator">+</span> p<span class="token punctuation">.</span>cq_off<span class="token punctuation">.</span>cqes<span class="token punctuation">;</span>
<span class="token keyword">return</span> <span class="token number">0</span><span class="token punctuation">;</span>
}
off_t get_file_size(int fd) {
struct stat st;
if(fstat(fd, &st) < 0) {
perror("fstat");
return -1;
}
if (S_ISBLK(st.st_mode)) {
unsigned long long bytes;
if (ioctl(fd, BLKGETSIZE64, &bytes) != 0) {
perror("ioctl");
return -1;
}
return bytes;
} else if (S_ISREG(st.st_mode))
return st.st_size;
return -1;
}
void output_to_console(char buf, int len) {
while (len--) {
fputc(buf++, stdout);
}
}
void read_from_cq(struct submitter *s) {
<span class="token keyword">struct</span> <span class="token class-name">file_info</span> <span class="token operator">*</span>fi<span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">app_io_cq_ring</span> <span class="token operator">*</span>cring <span class="token operator">=</span> <span class="token operator">&</span>s<span class="token operator">-></span>cq_ring<span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">io_uring_cqe</span> <span class="token operator">*</span>cqe<span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> head <span class="token operator">=</span> <span class="token operator">*</span>cring<span class="token operator">-></span>head<span class="token punctuation">;</span>
<span class="token keyword">while</span> <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token comment">//read_barrier();</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>head <span class="token operator">==</span> <span class="token operator">*</span>cring<span class="token operator">-></span>tail<span class="token punctuation">)</span> <span class="token keyword">break</span><span class="token punctuation">;</span>
cqe <span class="token operator">=</span> <span class="token operator">&</span>cring<span class="token operator">-></span>cqes<span class="token punctuation">[</span>head <span class="token operator">&</span> <span class="token operator">*</span>s<span class="token operator">-></span>cq_ring<span class="token punctuation">.</span>ring_mask<span class="token punctuation">]</span><span class="token punctuation">;</span>
fi <span class="token operator">=</span> <span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">file_info</span><span class="token operator">*</span><span class="token punctuation">)</span>cqe<span class="token operator">-></span>user_data<span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>cqe<span class="token operator">-></span>res <span class="token operator"><</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token function">fprintf</span><span class="token punctuation">(</span><span class="token constant">stderr</span><span class="token punctuation">,</span> <span class="token string">"Error: %d\n"</span><span class="token punctuation">,</span> cqe<span class="token operator">-></span>res<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token keyword">int</span> blocks <span class="token operator">=</span> fi<span class="token operator">-></span>file_sz <span class="token operator">/</span> BLOCK_SZ<span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>fi<span class="token operator">-></span>file_sz <span class="token operator">%</span> BLOCK_SZ<span class="token punctuation">)</span> blocks <span class="token operator">++</span><span class="token punctuation">;</span>
<span class="token keyword">int</span> i <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
<span class="token keyword">while</span> <span class="token punctuation">(</span><span class="token operator">++</span>i <span class="token operator"><</span> blocks<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token function">output_to_console</span><span class="token punctuation">(</span>fi<span class="token operator">-></span>iovecs<span class="token punctuation">[</span>i<span class="token punctuation">]</span><span class="token punctuation">.</span>iov_base<span class="token punctuation">,</span> fi<span class="token operator">-></span>iovecs<span class="token punctuation">[</span>i<span class="token punctuation">]</span><span class="token punctuation">.</span>iov_len<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token function">printf</span><span class="token punctuation">(</span><span class="token string">"------------------------i : %d, blocks: %d\n"</span><span class="token punctuation">,</span> i<span class="token punctuation">,</span> blocks<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
head <span class="token operator">++</span><span class="token punctuation">;</span>
<span class="token function">printf</span><span class="token punctuation">(</span><span class="token string">"head: %d, tail: %d, blocks: %d\n"</span><span class="token punctuation">,</span>
head<span class="token punctuation">,</span> <span class="token operator">*</span>cring<span class="token operator">-></span>tail<span class="token punctuation">,</span> blocks<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token operator">*</span>cring<span class="token operator">-></span>head <span class="token operator">=</span> head<span class="token punctuation">;</span>
<span class="token function">printf</span><span class="token punctuation">(</span><span class="token string">"exit read_from_cq\n"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">//write_barrier();</span>
}
int submit_to_sq(char file_path, struct submitter s) {
<span class="token keyword">int</span> filefd <span class="token operator">=</span> <span class="token function">open</span><span class="token punctuation">(</span>file_path<span class="token punctuation">,</span> O_RDONLY<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>filefd <span class="token operator"><</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token keyword">return</span> <span class="token operator">-</span><span class="token number">1</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token keyword">struct</span> <span class="token class-name">app_io_sq_ring</span> <span class="token operator">*</span>sring <span class="token operator">=</span> <span class="token operator">&</span>s<span class="token operator">-></span>sq_ring<span class="token punctuation">;</span>
<span class="token class-name">off_t</span> filesz <span class="token operator">=</span> <span class="token function">get_file_size</span><span class="token punctuation">(</span>filefd<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>filesz <span class="token operator"><</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token keyword">return</span> <span class="token operator">-</span><span class="token number">1</span><span class="token punctuation">;</span>
<span class="token class-name">off_t</span> bytes_remaining <span class="token operator">=</span> filesz<span class="token punctuation">;</span>
<span class="token keyword">int</span> blocks <span class="token operator">=</span> filesz <span class="token operator">/</span> BLOCK_SZ<span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>filesz <span class="token operator">%</span> BLOCK_SZ<span class="token punctuation">)</span> blocks <span class="token operator">++</span><span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">file_info</span> <span class="token operator">*</span>fi <span class="token operator">=</span> <span class="token function">malloc</span><span class="token punctuation">(</span><span class="token keyword">sizeof</span><span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">file_info</span><span class="token punctuation">)</span> <span class="token operator">+</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">iovec</span><span class="token punctuation">)</span> <span class="token operator">*</span> blocks<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">!</span>fi<span class="token punctuation">)</span> <span class="token keyword">return</span> <span class="token operator">-</span><span class="token number">2</span><span class="token punctuation">;</span>
fi<span class="token operator">-></span>file_sz <span class="token operator">=</span> filesz<span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> current_block<span class="token punctuation">;</span>
<span class="token keyword">while</span> <span class="token punctuation">(</span>bytes_remaining<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token class-name">off_t</span> bytes_to_read <span class="token operator">=</span> bytes_remaining<span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>bytes_to_read <span class="token operator">></span> BLOCK_SZ<span class="token punctuation">)</span> bytes_to_read <span class="token operator">=</span> BLOCK_SZ<span class="token punctuation">;</span>
fi<span class="token operator">-></span>iovecs<span class="token punctuation">[</span>current_block<span class="token punctuation">]</span><span class="token punctuation">.</span>iov_len <span class="token operator">=</span> bytes_to_read<span class="token punctuation">;</span>
<span class="token keyword">void</span> <span class="token operator">*</span>buf<span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token function">posix_memalign</span><span class="token punctuation">(</span><span class="token operator">&</span>buf<span class="token punctuation">,</span> BLOCK_SZ<span class="token punctuation">,</span> BLOCK_SZ<span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token keyword">return</span> <span class="token number">1</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
fi<span class="token operator">-></span>iovecs<span class="token punctuation">[</span>current_block<span class="token punctuation">]</span><span class="token punctuation">.</span>iov_base <span class="token operator">=</span> buf<span class="token punctuation">;</span>
current_block <span class="token operator">++</span><span class="token punctuation">;</span>
bytes_remaining <span class="token operator">-=</span> bytes_to_read<span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token keyword">unsigned</span> next_tail <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">,</span> tail <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">,</span> index <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
next_tail <span class="token operator">=</span> tail <span class="token operator">=</span> <span class="token operator">*</span>sring<span class="token operator">-></span>tail<span class="token punctuation">;</span>
next_tail <span class="token operator">++</span><span class="token punctuation">;</span>
index <span class="token operator">=</span> tail <span class="token operator">&</span> <span class="token operator">*</span>s<span class="token operator">-></span>sq_ring<span class="token punctuation">.</span>ring_mask<span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">io_uring_sqe</span> <span class="token operator">*</span>sqe <span class="token operator">=</span> <span class="token operator">&</span>s<span class="token operator">-></span>sqes<span class="token punctuation">[</span>index<span class="token punctuation">]</span><span class="token punctuation">;</span>
sqe<span class="token operator">-></span>fd <span class="token operator">=</span> filefd<span class="token punctuation">;</span>
sqe<span class="token operator">-></span>flags <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
sqe<span class="token operator">-></span>opcode <span class="token operator">=</span> IORING_OP_READV<span class="token punctuation">;</span>
sqe<span class="token operator">-></span>addr <span class="token operator">=</span> <span class="token punctuation">(</span><span class="token keyword">unsigned</span> <span class="token keyword">long</span><span class="token punctuation">)</span>fi<span class="token operator">-></span>iovecs<span class="token punctuation">;</span>
sqe<span class="token operator">-></span>len <span class="token operator">=</span> blocks<span class="token punctuation">;</span>
sqe<span class="token operator">-></span>off <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
sqe<span class="token operator">-></span>user_data <span class="token operator">=</span> <span class="token punctuation">(</span><span class="token keyword">unsigned</span> <span class="token keyword">long</span> <span class="token keyword">long</span><span class="token punctuation">)</span>fi<span class="token punctuation">;</span>
sring<span class="token operator">-></span>array<span class="token punctuation">[</span>index<span class="token punctuation">]</span> <span class="token operator">=</span> index<span class="token punctuation">;</span>
tail <span class="token operator">=</span> next_tail<span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">*</span>sring<span class="token operator">-></span>tail <span class="token operator">!=</span> tail<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token operator">*</span>sring<span class="token operator">-></span>tail <span class="token operator">=</span> tail<span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token keyword">int</span> ret <span class="token operator">=</span> <span class="token function">io_uring_enter</span><span class="token punctuation">(</span>s<span class="token operator">-></span>ring_fd<span class="token punctuation">,</span> <span class="token number">1</span><span class="token punctuation">,</span> <span class="token number">1</span><span class="token punctuation">,</span> IORING_ENTER_GETEVENTS<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>ret <span class="token operator"><</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token keyword">return</span> <span class="token number">1</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token keyword">return</span> <span class="token number">0</span><span class="token punctuation">;</span>
}
int main(int argc, char *argv[]) {
<span class="token keyword">struct</span> <span class="token class-name">submitter</span> <span class="token operator">*</span>s <span class="token operator">=</span> <span class="token function">malloc</span><span class="token punctuation">(</span><span class="token keyword">sizeof</span><span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">submitter</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">!</span>s<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token function">perror</span><span class="token punctuation">(</span><span class="token string">"malloc"</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">return</span> <span class="token operator">-</span><span class="token number">1</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token function">memset</span><span class="token punctuation">(</span>s<span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">,</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">submitter</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// 1、setup</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token function">app_setup_uring</span><span class="token punctuation">(</span>s<span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token keyword">return</span> <span class="token number">1</span><span class="token punctuation">;</span>
<span class="token keyword">int</span> i <span class="token operator">=</span> <span class="token number">1</span><span class="token punctuation">;</span>
<span class="token keyword">for</span> <span class="token punctuation">(</span>i <span class="token operator">=</span> <span class="token number">1</span><span class="token punctuation">;</span>i <span class="token operator"><</span> argc<span class="token punctuation">;</span>i <span class="token operator">++</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token comment">// 2、submit</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token function">submit_to_sq</span><span class="token punctuation">(</span>argv<span class="token punctuation">[</span>i<span class="token punctuation">]</span><span class="token punctuation">,</span> s<span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token comment">//fprintf(stderr, "Error reading file\n");</span>
<span class="token keyword">return</span> <span class="token number">1</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token function">read_from_cq</span><span class="token punctuation">(</span>s<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token keyword">return</span> <span class="token number">0</span><span class="token punctuation">;</span>
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
- 111
- 112
- 113
- 114
- 115
- 116
- 117
- 118
- 119
- 120
- 121
- 122
- 123
- 124
- 125
- 126
- 127
- 128
- 129
- 130
- 131
- 132
- 133
- 134
- 135
- 136
- 137
- 138
- 139
- 140
- 141
- 142
- 143
- 144
- 145
- 146
- 147
- 148
- 149
- 150
- 151
- 152
- 153
- 154
- 155
- 156
- 157
- 158
- 159
- 160
- 161
- 162
- 163
- 164
- 165
- 166
- 167
- 168
- 169
- 170
- 171
- 172
- 173
- 174
- 175
- 176
- 177
- 178
- 179
- 180
- 181
- 182
- 183
- 184
- 185
- 186
- 187
- 188
- 189
- 190
- 191
- 192
- 193
- 194
- 195
- 196
- 197
- 198
- 199
- 200
- 201
- 202
- 203
- 204
- 205
- 206
- 207
- 208
- 209
- 210
- 211
- 212
- 213
- 214
- 215
- 216
- 217
- 218
- 219
- 220
- 221
- 222
- 223
- 224
- 225
- 226
- 227
- 228
- 229
- 230
- 231
- 232
- 233
- 234
- 235
- 236
- 237
- 238
- 239
- 240
- 241
- 242
- 243
- 244
- 245
- 246
- 247
- 248
- 249
- 250
- 251
- 252
- 253
- 254
- 255
- 256
- 257
- 258
- 259
- 260
- 261
- 262
- 263
- 264
- 265
- 266
- 267
- 268
- 269
- 270
- 271
- 272
- 273
- 274
- 275
- 276
- 277
- 278
- 279
- 280
- 281
- 282
- 283
- 284
- 285
- 286
- 287
- 288
- 289
- 290
- 291
- 292
- 293
- 294
- 295
- 296
- 297
- 298
- 299
- 300
- 301
- 302
- 303
- 304
- 305
- 306
- 307
- 308
- 309
- 310
- 311
- 312
- 313
- 314
- 315
- 316
- 317
- 318
- 319
- 320
- 321
- 322
- 323
- 324
- 325
3、liburing
由于 io_uring 使用起来比较麻烦，作者封装了 io_uring 接口，创作了 liburing 库。
# 安装 liburing
git clone https://github.com/axboe/liburing.git
./configure
make && make install
- 1
- 2
- 3
- 4
3.1、liburing api
// 初始化io_uring，内部调用io_uring_setup
int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
struct io_uring_params *p);
// 提交 sq 到内核，内核完成后移动到 cq，内部调用 io_uring_enter
// 1、提交io请求：将sqe的偏移信息加入到sq，提交sq到内核，不阻塞等待其完成
// 2、等待io完成：内核在io完成后，自动将sqe的偏移信息加入到cq
int io_uring_submit(struct io_uring *ring);
// 等待io完成，获取cqe
// 阻塞等待
unsigned io_uring_peek_batch_cqe(struct io_uring ring,
struct io_uring_cqe cqes, unsigned count);
// 不阻塞等待
int io_uring_wait_cqes(struct io_uring ring, struct io_uring_cqe cqe_ptr,
unsigned wait_nr, struct __kernel_timespec ts,
sigset_t sigmask);
// 轮询 cq 队列，将 cq 队首后移动 nr 个
static inline void io_uring_cq_advance(struct io_uring *ring, unsigned nr)
// 和libaio封装的io_prep_writev一样
static inline void io_uring_prep_writev(struct io_uring_sqe sqe, int fd,const struct iovec iovecs, unsigned nr_vecs, off_t offset)
// 和libaio封装的io_prep_readv一样
static inline void io_uring_prep_readv(struct io_uring_sqe sqe, int fd, const struct iovec iovecs, unsigned nr_vecs, off_t offset)
// 销毁 io
void io_uring_queue_exit(struct io_uring *ring);
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
3.2、测试代码
利用 liburing 编写的简单测试 iouring_server
// gcc -o iouring_server iouring_server.c -luring
#include <liburing.h>
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <unistd.h>
#define ENTRIES_LENGTH 4096
#define MAX_CONNECTIONS 1024
#define BUFFER_LENGTH 1024
char buf_table[MAX_CONNECTIONS][BUFFER_LENGTH] = {0};
// 传递的事件
enum {
READ,
WRITE,
ACCEPT,
};
// 连接信息
struct conninfo {
int connfd; // fd
int type; // 事件类型
};
void set_read_event(struct io_uring ring, int fd, void buf, size_t len, int flags) {
<span class="token keyword">struct</span> <span class="token class-name">io_uring_sqe</span> <span class="token operator">*</span>sqe <span class="token operator">=</span> <span class="token function">io_uring_get_sqe</span><span class="token punctuation">(</span>ring<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// io_uring 读事件</span>
<span class="token function">io_uring_prep_recv</span><span class="token punctuation">(</span>sqe<span class="token punctuation">,</span> fd<span class="token punctuation">,</span> buf<span class="token punctuation">,</span> len<span class="token punctuation">,</span> flags<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">conninfo</span> ci <span class="token operator">=</span> <span class="token punctuation">{<!-- --></span>
<span class="token punctuation">.</span>connfd <span class="token operator">=</span> fd<span class="token punctuation">,</span>
<span class="token punctuation">.</span>type <span class="token operator">=</span> READ
<span class="token punctuation">}</span><span class="token punctuation">;</span>
<span class="token function">memcpy</span><span class="token punctuation">(</span><span class="token operator">&</span>sqe<span class="token operator">-></span>user_data<span class="token punctuation">,</span> <span class="token operator">&</span>ci<span class="token punctuation">,</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">conninfo</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
}
void set_write_event(struct io_uring ring, int fd, const void buf, size_t len, int flags) {
<span class="token keyword">struct</span> <span class="token class-name">io_uring_sqe</span> <span class="token operator">*</span>sqe <span class="token operator">=</span> <span class="token function">io_uring_get_sqe</span><span class="token punctuation">(</span>ring<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// io_uring 写事件</span>
<span class="token function">io_uring_prep_send</span><span class="token punctuation">(</span>sqe<span class="token punctuation">,</span> fd<span class="token punctuation">,</span> buf<span class="token punctuation">,</span> len<span class="token punctuation">,</span> flags<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">conninfo</span> ci <span class="token operator">=</span> <span class="token punctuation">{<!-- --></span>
<span class="token punctuation">.</span>connfd <span class="token operator">=</span> fd<span class="token punctuation">,</span>
<span class="token punctuation">.</span>type <span class="token operator">=</span> WRITE
<span class="token punctuation">}</span><span class="token punctuation">;</span>
<span class="token function">memcpy</span><span class="token punctuation">(</span><span class="token operator">&</span>sqe<span class="token operator">-></span>user_data<span class="token punctuation">,</span> <span class="token operator">&</span>ci<span class="token punctuation">,</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">conninfo</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
}
void set_accept_event(struct io_uring ring, int fd,
struct sockaddr cliaddr, socklen_t *clilen, unsigned flags) {
<span class="token comment">// 获取 sq 队列的空 sqe</span>
<span class="token keyword">struct</span> <span class="token class-name">io_uring_sqe</span> <span class="token operator">*</span>sqe <span class="token operator">=</span> <span class="token function">io_uring_get_sqe</span><span class="token punctuation">(</span>ring<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// io_uring的accept事件：将fd放入到sqe里</span>
<span class="token function">io_uring_prep_accept</span><span class="token punctuation">(</span>sqe<span class="token punctuation">,</span> fd<span class="token punctuation">,</span> cliaddr<span class="token punctuation">,</span> clilen<span class="token punctuation">,</span> flags<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// 用于回调函数</span>
<span class="token keyword">struct</span> <span class="token class-name">conninfo</span> ci <span class="token operator">=</span> <span class="token punctuation">{<!-- --></span>
<span class="token punctuation">.</span>connfd <span class="token operator">=</span> fd<span class="token punctuation">,</span>
<span class="token punctuation">.</span>type <span class="token operator">=</span> ACCEPT
<span class="token punctuation">}</span><span class="token punctuation">;</span>
<span class="token function">memcpy</span><span class="token punctuation">(</span><span class="token operator">&</span>sqe<span class="token operator">-></span>user_data<span class="token punctuation">,</span> <span class="token operator">&</span>ci<span class="token punctuation">,</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">conninfo</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
}
int main() {
<span class="token keyword">int</span> listenfd <span class="token operator">=</span> <span class="token function">socket</span><span class="token punctuation">(</span>AF_INET<span class="token punctuation">,</span> SOCK_STREAM<span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>listenfd <span class="token operator">==</span> <span class="token operator">-</span><span class="token number">1</span><span class="token punctuation">)</span> <span class="token keyword">return</span> <span class="token operator">-</span><span class="token number">1</span><span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">sockaddr_in</span> servaddr<span class="token punctuation">,</span> clientaddr<span class="token punctuation">;</span>
servaddr<span class="token punctuation">.</span>sin_family <span class="token operator">=</span> AF_INET<span class="token punctuation">;</span>
servaddr<span class="token punctuation">.</span>sin_addr<span class="token punctuation">.</span>s_addr <span class="token operator">=</span> <span class="token function">htonl</span><span class="token punctuation">(</span>INADDR_ANY<span class="token punctuation">)</span><span class="token punctuation">;</span>
servaddr<span class="token punctuation">.</span>sin_port <span class="token operator">=</span> <span class="token function">htons</span><span class="token punctuation">(</span><span class="token number">9999</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">-</span><span class="token number">1</span> <span class="token operator">==</span> <span class="token function">bind</span><span class="token punctuation">(</span>listenfd<span class="token punctuation">,</span> <span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">sockaddr</span><span class="token operator">*</span><span class="token punctuation">)</span><span class="token operator">&</span>servaddr<span class="token punctuation">,</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span>servaddr<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token keyword">return</span> <span class="token operator">-</span><span class="token number">2</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token function">listen</span><span class="token punctuation">(</span>listenfd<span class="token punctuation">,</span> <span class="token number">10</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">io_uring_params</span> params<span class="token punctuation">;</span>
<span class="token function">memset</span><span class="token punctuation">(</span><span class="token operator">&</span>params<span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">,</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span>params<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// 初始化队列，内部调用io_uring_setup</span>
<span class="token keyword">struct</span> <span class="token class-name">io_uring</span> ring<span class="token punctuation">;</span>
<span class="token function">io_uring_queue_init_params</span><span class="token punctuation">(</span>ENTRIES_LENGTH<span class="token punctuation">,</span> <span class="token operator">&</span>ring<span class="token punctuation">,</span> <span class="token operator">&</span>params<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token class-name">socklen_t</span> clilen <span class="token operator">=</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span>clientaddr<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token function">set_accept_event</span><span class="token punctuation">(</span><span class="token operator">&</span>ring<span class="token punctuation">,</span> listenfd<span class="token punctuation">,</span> <span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">sockaddr</span><span class="token operator">*</span><span class="token punctuation">)</span><span class="token operator">&</span>clientaddr<span class="token punctuation">,</span> <span class="token operator">&</span>clilen<span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">while</span> <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token comment">// 封装 io_uring_enter</span>
<span class="token comment">// 1、提交io请求：将sqe的偏移信息加入到sq，提交sq到内核，不阻塞等待其完成</span>
<span class="token comment">// 2、等待io完成：内核在io完成后，自动将sqe的偏移信息加入到cq</span>
<span class="token function">io_uring_submit</span><span class="token punctuation">(</span><span class="token operator">&</span>ring<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// 从获取 cqe 的两种方式</span>
<span class="token comment">// 1、阻塞等待io完成，获取 cqe</span>
<span class="token keyword">struct</span> <span class="token class-name">io_uring_cqe</span> <span class="token operator">*</span>cqe<span class="token punctuation">;</span>
<span class="token keyword">int</span> ret <span class="token operator">=</span> <span class="token function">io_uring_wait_cqe</span><span class="token punctuation">(</span><span class="token operator">&</span>ring<span class="token punctuation">,</span> <span class="token operator">&</span>cqe<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// 2、不阻塞等待io完成，没有cqe返回错误，获取 cqe</span>
<span class="token keyword">struct</span> <span class="token class-name">io_uring_cqe</span> <span class="token operator">*</span>cqes<span class="token punctuation">[</span><span class="token number">10</span><span class="token punctuation">]</span><span class="token punctuation">;</span>
<span class="token keyword">int</span> cqecount <span class="token operator">=</span> <span class="token function">io_uring_peek_batch_cqe</span><span class="token punctuation">(</span><span class="token operator">&</span>ring<span class="token punctuation">,</span> cqes<span class="token punctuation">,</span> <span class="token number">10</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">int</span> i <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
<span class="token keyword">unsigned</span> count <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
<span class="token keyword">for</span> <span class="token punctuation">(</span>i <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span> i <span class="token operator"><</span> cqecount<span class="token punctuation">;</span> <span class="token operator">++</span>i<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
cqe <span class="token operator">=</span> cqes<span class="token punctuation">[</span>i<span class="token punctuation">]</span><span class="token punctuation">;</span>
count <span class="token operator">++</span><span class="token punctuation">;</span>
<span class="token keyword">struct</span> <span class="token class-name">conninfo</span> ci<span class="token punctuation">;</span>
<span class="token function">memcpy</span><span class="token punctuation">(</span><span class="token operator">&</span>ci<span class="token punctuation">,</span> <span class="token operator">&</span>cqe<span class="token operator">-></span>user_data<span class="token punctuation">,</span> <span class="token keyword">sizeof</span><span class="token punctuation">(</span>ci<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>ci<span class="token punctuation">.</span>type <span class="token operator">==</span> ACCEPT<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token keyword">int</span> connfd <span class="token operator">=</span> cqe<span class="token operator">-></span>res<span class="token punctuation">;</span>
<span class="token keyword">char</span> <span class="token operator">*</span>buffer <span class="token operator">=</span> buf_table<span class="token punctuation">[</span>connfd<span class="token punctuation">]</span><span class="token punctuation">;</span>
<span class="token function">set_read_event</span><span class="token punctuation">(</span><span class="token operator">&</span>ring<span class="token punctuation">,</span> connfd<span class="token punctuation">,</span> buffer<span class="token punctuation">,</span> <span class="token number">1024</span><span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment">// io_uring 设置一次，触发一次</span>
<span class="token function">set_accept_event</span><span class="token punctuation">(</span><span class="token operator">&</span>ring<span class="token punctuation">,</span> listenfd<span class="token punctuation">,</span> <span class="token punctuation">(</span><span class="token keyword">struct</span> <span class="token class-name">sockaddr</span><span class="token operator">*</span><span class="token punctuation">)</span><span class="token operator">&</span>clientaddr<span class="token punctuation">,</span> <span class="token operator">&</span>clilen<span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token keyword">if</span> <span class="token punctuation">(</span>ci<span class="token punctuation">.</span>type <span class="token operator">==</span> READ<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token keyword">int</span> bytes_read <span class="token operator">=</span> cqe<span class="token operator">-></span>res<span class="token punctuation">;</span>
<span class="token keyword">if</span> <span class="token punctuation">(</span>bytes_read <span class="token operator">==</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token function">close</span><span class="token punctuation">(</span>ci<span class="token punctuation">.</span>connfd<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token keyword">if</span> <span class="token punctuation">(</span>bytes_read <span class="token operator"><</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token punctuation">{<!-- --></span>
<span class="token keyword">char</span> <span class="token operator">*</span>buffer <span class="token operator">=</span> buf_table<span class="token punctuation">[</span>ci<span class="token punctuation">.</span>connfd<span class="token punctuation">]</span><span class="token punctuation">;</span>
<span class="token function">set_write_event</span><span class="token punctuation">(</span><span class="token operator">&</span>ring<span class="token punctuation">,</span> ci<span class="token punctuation">.</span>connfd<span class="token punctuation">,</span> buffer<span class="token punctuation">,</span> bytes_read<span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token keyword">if</span> <span class="token punctuation">(</span>ci<span class="token punctuation">.</span>type <span class="token operator">==</span> WRITE<span class="token punctuation">)</span> <span class="token punctuation">{<!-- --></span>
<span class="token keyword">char</span> <span class="token operator">*</span>buffer <span class="token operator">=</span> buf_table<span class="token punctuation">[</span>ci<span class="token punctuation">.</span>connfd<span class="token punctuation">]</span><span class="token punctuation">;</span>
<span class="token function">set_read_event</span><span class="token punctuation">(</span><span class="token operator">&</span>ring<span class="token punctuation">,</span> ci<span class="token punctuation">.</span>connfd<span class="token punctuation">,</span> buffer<span class="token punctuation">,</span> <span class="token number">1024</span><span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
<span class="token punctuation">}</span>
<span class="token comment">// cq队列一次轮询完成后，因为cqe的取出，需要调整队首的位置，以便下次使用</span>
<span class="token function">io_uring_cq_advance</span><span class="token punctuation">(</span><span class="token operator">&</span>ring<span class="token punctuation">,</span> count<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
}