// https://github.com/torvalds/linux/blob/v5.10/drivers/net/ethernet/mellanox/mlx5/core/main.c

static int __init init(void) {
    mlx5_fpga_ipsec_build_fs_cmds();
    mlx5_register_debugfs();                // /sys/kernel/debug

    pci_register_driver(&mlx5_core_driver); // 初始化 PCI 相关的东西
    mlx5e_init();                           // 初始化 ethernet 相关东西
}

module_init(init);

// https://github.com/torvalds/linux/blob/v5.10/include/linux/module.h

#ifdef MODULE
  /* Creates an alias so file2alias.c can find device table. */
  #define MODULE_DEVICE_TABLE(type, name)                    \
    extern typeof(name) __mod_##type##__##name##_device_table __attribute__ ((unused, alias(__stringify(name))))
#else  /* !MODULE */
  #define MODULE_DEVICE_TABLE(type, name)
#endif

// https://github.com/torvalds/linux/blob/v5.10/drivers/net/ethernet/mellanox/mlx5/core/main.c

static const struct pci_device_id mlx5_core_pci_table[] = {
    { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTIB) },
    { PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF},    /* Connect-IB VF */
    { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4) },
    { PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF},    /* ConnectX-4 VF */
    { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX) },
    { PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF},    /* ConnectX-4LX VF */
    { PCI_VDEVICE(MELLANOX, 0x1017) },            /* ConnectX-5, PCIe 3.0 */
    { PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF},    /* ConnectX-5 VF */
    { PCI_VDEVICE(MELLANOX, 0x1019) },            /* ConnectX-5 Ex */
    { PCI_VDEVICE(MELLANOX, 0x101a), MLX5_PCI_DEV_IS_VF},    /* ConnectX-5 Ex VF */
    { PCI_VDEVICE(MELLANOX, 0x101b) },            /* ConnectX-6 */
    { PCI_VDEVICE(MELLANOX, 0x101c), MLX5_PCI_DEV_IS_VF},    /* ConnectX-6 VF */
    { PCI_VDEVICE(MELLANOX, 0x101d) },            /* ConnectX-6 Dx */
    { PCI_VDEVICE(MELLANOX, 0x101e), MLX5_PCI_DEV_IS_VF},    /* ConnectX Family mlx5Gen Virtual Function */
    { PCI_VDEVICE(MELLANOX, 0x101f) },            /* ConnectX-6 LX */
    { PCI_VDEVICE(MELLANOX, 0x1021) },            /* ConnectX-7 */
    { PCI_VDEVICE(MELLANOX, 0xa2d2) },            /* BlueField integrated ConnectX-5 network controller */
    { PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF},    /* BlueField integrated ConnectX-5 network controller VF */
    { PCI_VDEVICE(MELLANOX, 0xa2d6) },            /* BlueField-2 integrated ConnectX-6 Dx network controller */
    { 0, }
};

MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);

// https://github.com/torvalds/linux/blob/v5.10/drivers/net/ethernet/mellanox/mlx5/core/main.c

static struct pci_driver mlx5_core_driver = {
    .name           = DRIVER_NAME,
    .id_table       = mlx5_core_pci_table,
    .probe          = init_one,              // 初始化时执行这个方法
    .remove         = remove_one,
    .suspend        = mlx5_suspend,
    .resume         = mlx5_resume,
    .shutdown    = shutdown,
    .err_handler    = &mlx5_err_handler,
    .sriov_configure   = mlx5_core_sriov_configure,
};

// https://github.com/torvalds/linux/blob/v5.10/drivers/net/ethernet/mellanox/mlx5/core/main.c

static int init_one(struct pci_dev *pdev, const struct pci_device_id *id) {
    struct devlink       *devlink = mlx5_devlink_alloc();  // 内核通用结构体，包含 netns 等信息
    struct mlx5_core_dev *dev     = devlink_priv(devlink); // 私有数据区域存储的是 mlx5 自己的 device 表示
    dev->device = &pdev->dev;
    dev->pdev = pdev;

    mlx5_mdev_init(dev, prof_sel); // 初始化 mlx5 debugfs 目录
    mlx5_pci_init(dev, pdev, id);  // 初始化 IO/DMA/Capabilities 等功能
    mlx5_load_one(dev, true);      // 初始化 IRQ 等

    request_module_nowait(MLX5_IB_MOD);

    pci_save_state(pdev);
    if (!mlx5_core_is_mp_slave(dev))
        devlink_reload_enable(devlink);
}

// include/net/devlink.h
struct devlink {
    struct list_head list;
    struct list_head port_list;
    struct list_head sb_list;
    struct list_head dpipe_table_list;
    struct list_head resource_list;
    struct list_head param_list;
    struct list_head region_list;
    struct list_head reporter_list;
    struct mutex reporters_lock; /* protects reporter_list */
    struct devlink_dpipe_headers *dpipe_headers;
    struct list_head trap_list;
    struct list_head trap_group_list;
    struct list_head trap_policer_list;
    const struct devlink_ops *ops;
    struct xarray snapshot_ids;
    struct devlink_dev_stats stats;
    struct device *dev;
    possible_net_t _net;      // netns 相关
    u8 reload_failed:1,
       reload_enabled:1,
       registered:1;
    char priv[0] __aligned(NETDEV_ALIGN);
};

// drivers/net/ethernet/mellanox/mlx5/core/devlink.c

struct devlink *mlx5_devlink_alloc(void) {
    return devlink_alloc(&mlx5_devlink_ops, sizeof(struct mlx5_core_dev));
}

static const struct devlink_ops mlx5_devlink_ops = {
    .flash_update   = mlx5_devlink_flash_update,
    .info_get       = mlx5_devlink_info_get,
    .reload_actions = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) | BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE),
    .reload_limits  = BIT(DEVLINK_RELOAD_LIMIT_NO_RESET),
    .reload_down    = mlx5_devlink_reload_down,
    .reload_up      = mlx5_devlink_reload_up,
};

static int
mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev, const struct pci_device_id *id) {
    mlx5_pci_enable_device(dev);
    request_bar(pdev);            // Reserve PCI I/O and memory resources
    pci_set_master(pdev);         // Enables bus-mastering on the device
    set_dma_caps(pdev);           // setting DMA capabilities mask, set max_seg <= 1GB
    dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); // mapping initialization segment

    mlx5_pci_vsc_init(dev);
    dev->caps.embedded_cpu = mlx5_read_embedded_cpu(dev);
    return 0;
}

int mlx5_load_one(struct mlx5_core_dev *dev, bool boot) {
    dev->state = MLX5_DEVICE_STATE_UP;
    mlx5_function_setup(dev, boot);

    if (boot) {
        mlx5_init_once(dev);
    }

    mlx5_load(dev);
    set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);

    if (boot) {
        mlx5_devlink_register(priv_to_devlink(dev), dev->device);
        mlx5_register_device(dev);
    } else {
        mlx5_attach_device(dev);
    }
}

// en_main.c

void mlx5e_init(void) { 
    mlx5e_ipsec_build_inverse_table();
    mlx5e_build_ptys2ethtool_map();
    mlx5_register_interface(&mlx5e_interface);
}

static struct mlx5_interface mlx5e_interface = {
    .add       = mlx5e_add,
    .remove    = mlx5e_remove,
    .attach    = mlx5e_attach,
    .detach    = mlx5e_detach,
    .protocol  = MLX5_INTERFACE_PROTOCOL_ETH, // 接口运行以太网协议
};

mlx5e_nic
  |-mlx5e_ipsec_build_inverse_table();
  |-mlx5e_build_ptys2ethtool_map();
  |-mlx5_register_interface(&mlx5e_interface)
     |-list_add_tail(&intf->list, &intf_list);
     |
     |-for priv in mlx5_dev_list
         mlx5_add_device(intf, priv)
          /
         /
mlx5_add_device(intf, priv)
 |-if !mlx5_lag_intf_add
 |   return // if running in InfiniBand mode, directly return
 |
 |-dev_ctx = kzalloc()
 |-dev_ctx->context = intf->add(dev)
     |-mlx5e_add
        |-netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, nch);
        |  |-mlx5e_nic_init
        |     |-mlx5e_build_nic_netdev(netdev);
        |        |-netdev->netdev_ops = &mlx5e_netdev_ops; // 注册 ethtool_ops, poll
        |-mlx5e_attach
        |  |-mlx5e_attach_netdev
        |      |-profile->init_tx()
        |      |-profile->init_rx()
        |      |-profile->enable()
        |          |-mlx5e_nic_enable
        |              |-mlx5e_init_l2_addr
        |              |-queue_work(priv->wq, &priv->set_rx_mode_work)
        |              |-mlx5e_open(netdev);
        |              |  |-mlx5e_open_locked
        |              |     |-mlx5e_open_channels
        |              |     |  |-mlx5e_open_channel
        |              |     |     |-netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
        |              |     |     |-mlx5e_open_queues
        |              |     |         |-mlx5e_open_cq
        |              |     |         |   |-mlx5e_alloc_cq
        |              |     |         |       |-mlx5e_alloc_cq_common
        |              |     |         |           |-mcq->comp = mlx5e_completion_event;
        |              |     |         |-napi_enable(&c->napi)
        |              |     |-mlx5e_activate_priv_channels
        |              |        |-mlx5e_activate_channels
        |              |            |-for ch in channels:
        |              |                mlx5e_activate_channel
        |              |                  |-mlx5e_activate_rq
        |              |                     |-mlx5e_trigger_irq
        |              |-netif_device_attach(netdev);

// drivers/net/ethernet/mellanox/mlx5/core/dev.c

static LIST_HEAD(intf_list);      // 全局 interface 双向链表
static LIST_HEAD(mlx5_dev_list);  // 全局 mlx5 device 双向链表

int mlx5_register_interface(struct mlx5_interface *intf) {
    list_add_tail(&intf->list, &intf_list);

    list_for_each_entry(struct mlx5_priv *priv, &mlx5_dev_list, dev_list)
        mlx5_add_device(intf, priv); // 如果网卡运行的是 InfiniBand 协议，里面会直接返回
}

static const struct mlx5e_profile mlx5e_nic_profile = {
    .init           = mlx5e_nic_init,                // 网卡以太网相关内容初始化
    .cleanup        = mlx5e_nic_cleanup,
    .init_rx        = mlx5e_init_nic_rx,             // 初始化接收队列（RX）
    .cleanup_rx     = mlx5e_cleanup_nic_rx,
    .init_tx        = mlx5e_init_nic_tx,             // 初始化发送队列（TX）
    .cleanup_tx     = mlx5e_cleanup_nic_tx,
    .enable         = mlx5e_nic_enable,              // 启用网卡时的回调
    .disable        = mlx5e_nic_disable,
    .update_rx      = mlx5e_update_nic_rx,
    .update_stats   = mlx5e_stats_update_ndo_stats,
    .update_carrier = mlx5e_update_carrier,
    .rx_handlers    = &mlx5e_rx_handlers_nic,        // 收包函数
    .max_tc         = MLX5E_MAX_NUM_TC,
    .rq_groups      = MLX5E_NUM_RQ_GROUPS(XSK),
    .stats_grps     = mlx5e_nic_stats_grps,
    .stats_grps_num = mlx5e_nic_stats_grps_num,
};

// drivers/net/ethernet/mellanox/mlx5/core/en_main.c

const struct net_device_ops mlx5e_netdev_ops = {
    .ndo_open                = mlx5e_open,
    .ndo_stop                = mlx5e_close,
    .ndo_start_xmit          = mlx5e_xmit,
    .ndo_setup_tc            = mlx5e_setup_tc,
    .ndo_select_queue        = mlx5e_select_queue,
    .ndo_get_stats64         = mlx5e_get_stats,
    .ndo_set_rx_mode         = mlx5e_set_rx_mode,
    .ndo_set_mac_address     = mlx5e_set_mac,
    .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
    .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
    .ndo_set_features        = mlx5e_set_features,
    .ndo_fix_features        = mlx5e_fix_features,
    .ndo_change_mtu          = mlx5e_change_nic_mtu,
    .ndo_do_ioctl            = mlx5e_ioctl,
    .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
    .ndo_udp_tunnel_add      = udp_tunnel_nic_add_port,
    .ndo_udp_tunnel_del      = udp_tunnel_nic_del_port,
    .ndo_features_check      = mlx5e_features_check,
    .ndo_tx_timeout          = mlx5e_tx_timeout,
    .ndo_bpf                 = mlx5e_xdp,             // BPF
    .ndo_xdp_xmit            = mlx5e_xdp_xmit,        // XDP
    .ndo_xsk_wakeup          = mlx5e_xsk_wakeup,
    .ndo_get_devlink_port    = mlx5e_get_devlink_port,
};

static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) {
    struct mlx5_core_dev *mdev = priv->mdev;

    mlx5e_create_q_counters(priv);
    mlx5e_open_drop_rq(priv, &priv->drop_rq);

    mlx5e_create_indirect_rqt(priv);
    mlx5e_create_direct_rqts(priv, priv->direct_tir);

    mlx5e_create_indirect_tirs(priv, true);
    mlx5e_create_direct_tirs(priv, priv->direct_tir);

    mlx5e_create_direct_rqts(priv, priv->xsk_tir);
    mlx5e_create_direct_tirs(priv, priv->xsk_tir);

    mlx5e_create_flow_steering(priv);
    mlx5e_tc_nic_init(priv);          // 初始化 TC offload 功能
    mlx5e_accel_init_rx(priv);        // 初始化 kTLS offload

#ifdef CONFIG_MLX5_EN_ARFS
    priv->netdev->rx_cpu_rmap =  mlx5_eq_table_get_rmap(priv->mdev);
#endif

const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic = {
    .handle_rx_cqe       = mlx5e_handle_rx_cqe,
    .handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
};

// drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c

static irqreturn_t mlx5_irq_int_handler(int irq, void *nh) {
    atomic_notifier_call_chain(nh, 0, NULL); // 内核函数，见 kernel/notifier.c
    return IRQ_HANDLED;
}

// https://github.com/torvalds/linux/blob/v5.10/drivers/net/ethernet/mellanox/mlx5/core/eq.c

static int mlx5_eq_comp_int(struct notifier_block *nb) {
    struct mlx5_eq_comp *eq_comp = container_of(nb, struct mlx5_eq_comp, irq_nb);
    struct mlx5_eq      *eq      = &eq_comp->core;
    u32 cqn = -1;

    struct mlx5_eqe *eqe = next_eqe_sw(eq); // 从 event queue 拿出一个 entry
    do {
        dma_rmb(); // Make sure we read EQ entry contents after we've checked the ownership bit.

        struct mlx5_core_cq *cq = mlx5_eq_cq_get(eq, cqn); // 获取 EQ completion queue
        if (likely(cq)) {
            ++cq->arm_sn;
            cq->comp(cq, eqe);              // 执行 completion 方法，里面最主要的步骤是执行 napi_schedule()
            mlx5_cq_put(cq);                // refcnt--，释放空间
        }

        ++eq->cons_index;
    } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq)));

    eq_update_ci(eq, 1); // 更新一个硬件寄存器，记录 IRQ 频率
    if (cqn != -1)
        tasklet_schedule(&eq_comp->tasklet_ctx.task);
}

void mlx5e_completion_event(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) {
    struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq);

    napi_schedule(cq->napi);
    cq->event_ctr++;
    cq->channel->stats->events++;
}

// https://github.com/torvalds/linux/blob/v5.10/net/core/dev.c

/**
 * __napi_schedule - schedule for receive
 * @n: entry to schedule
 *
 * The entry's receive function will be scheduled to run.
 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
 */
void __napi_schedule(struct napi_struct *n) {
    unsigned long flags;

    local_irq_save(flags);
    ____napi_schedule(this_cpu_ptr(&softnet_data), n);
    local_irq_restore(flags);
}

// https://github.com/torvalds/linux/blob/v5.10/net/core/dev.c

/* Called with irq disabled */
static inline void ____napi_schedule(struct softnet_data *sd, struct napi_struct *napi) {
  list_add_tail(&napi->poll_list, &sd->poll_list);
  __raise_softirq_irqoff(NET_RX_SOFTIRQ);  // kernel/softirq.c
}

// net/core/dev.c

// Initialize the DEV module. At boot time this walks the device list and
// unhooks any devices that fail to initialise (normally hardware not
// present) and leaves us with a valid list of present and active devices.
static int __init net_dev_init(void) {
    dev_proc_init();        // 注册 /proc/net/{dev,softnet_stat,ptytpe}
    netdev_kobject_init();

    INIT_LIST_HEAD(&ptype_all);
    for (i = 0; i < PTYPE_HASH_SIZE; i++)
        INIT_LIST_HEAD(&ptype_base[i]);

    INIT_LIST_HEAD(&offload_base);

    register_pernet_subsys(&netdev_net_ops);

    // 针对每个 CPU，初始化各种接收队列（packet receive queues）
    for_each_possible_cpu(i) {
        struct work_struct *flush = per_cpu_ptr(&flush_works, i);
        struct softnet_data *sd   = &per_cpu(softnet_data, i);
        INIT_WORK(flush, flush_backlog);

        skb_queue_head_init(&sd->input_pkt_queue);
        skb_queue_head_init(&sd->process_queue);
        skb_queue_head_init(&sd->xfrm_backlog);
        INIT_LIST_HEAD(&sd->poll_list);
        sd->output_queue_tailp = &sd->output_queue;
#ifdef CONFIG_RPS
        sd->csd.func = rps_trigger_softirq;
        sd->csd.info = sd;
        sd->cpu = i;
#endif

        init_gro_hash(&sd->backlog);          // GRO
        sd->backlog.poll = process_backlog;
        sd->backlog.weight = weight_p;
    }

    dev_boot_phase = 0;

    register_pernet_device(&loopback_net_ops)
    register_pernet_device(&default_device_ops)

    // 注册 RX/TX 软中断处理函数
    open_softirq(NET_TX_SOFTIRQ, net_tx_action);
    open_softirq(NET_RX_SOFTIRQ, net_rx_action);

    cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead", NULL, dev_cpu_dead);
}

// https://github.com/torvalds/linux/blob/v5.10/kernel/softirq.c#L730

static struct smp_hotplug_thread softirq_threads = {
    .store            = &ksoftirqd,
    .thread_should_run= ksoftirqd_should_run,  // 调度到该线程是，判断能否执行
    .thread_fn        = run_ksoftirqd,         // 调度到该线程时，执行的回调函数
    .thread_comm      = "ksoftirqd/%u",        // 线程名字，ps -ef 最后一列可以看到
};

static __init int spawn_ksoftirqd(void) {
    cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL, takeover_tasklets);
    BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
}
early_initcall(spawn_ksoftirqd);

// https://github.com/torvalds/linux/blob/v5.10/kernel/smpboot.c#L107

// smpboot_thread_fn - percpu hotplug thread loop function
// @data:    thread data pointer
//
// Returns 1 when the thread should exit, 0 otherwise.
static int smpboot_thread_fn(void *data) {
    struct smpboot_thread_data *td = data;
    struct smp_hotplug_thread *ht = td->ht;

    while (1) {
        set_current_state(TASK_INTERRUPTIBLE); // 设置当前 CPU 为可中断状态
        preempt_disable();

        if (kthread_should_park()) {
            ...
            continue; /* We might have been woken for stop */
        }

        switch (td->status) { /* Check for state change setup */
        case HP_THREAD_NONE:   ...  continue;
        case HP_THREAD_PARKED: ...  continue;
        }

        if (!ht->thread_should_run(td->cpu)) { // 无 pending 的软中断
            preempt_enable_no_resched();
            schedule();
        } else {                               // 有 pending 的软中断
            __set_current_state(TASK_RUNNING);
            preempt_enable();
            ht->thread_fn(td->cpu);            // 执行 `run_ksoftirqd()`
        }
    }
}

// https://github.com/torvalds/linux/blob/v5.10/kernel/softirq.c#L730

static void run_ksoftirqd(unsigned int cpu) {
    local_irq_disable();           // 关闭所在 CPU 的所有硬中断

    if (local_softirq_pending()) {
         // We can safely run softirq on inline stack, as we are not deep in the task stack here.
        __do_softirq();
        local_irq_enable();       // 重新打开所在 CPU 的所有硬中断
        cond_resched();           // 将 CPU 交还给调度器
        return;
    }

    local_irq_enable();            // 重新打开所在 CPU 的所有硬中断
}

// https://github.com/torvalds/linux/blob/v5.10/kernel/softirq.c#L730

asmlinkage __visible void __softirq_entry
__do_softirq(void) {
    unsigned long       end = jiffies + MAX_SOFTIRQ_TIME; // 时间片最晚结束时刻
    unsigned long old_flags = current->flags;
    int max_restart         = MAX_SOFTIRQ_RESTART;        // 最大轮询 pending softirq 次数

    // Mask out PF_MEMALLOC as the current task context is borrowed for the softirq.
    // A softirq handler, such as network RX, might set PF_MEMALLOC again if the socket is related to swapping.
    current->flags &= ~PF_MEMALLOC;

    pending = local_softirq_pending(); // 获取 pending softirq 数量
    account_irq_enter_time(current);

    __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
    in_hardirq = lockdep_softirq_start();

restart:
    set_softirq_pending(0); // Reset the pending bitmask before enabling irqs
    local_irq_enable();     // 打开 IRQ 中断

    struct softirq_action *h = softirq_vec;
    while ((softirq_bit = ffs(pending))) {
        h += softirq_bit - 1;
        unsigned int vec_nr     = h - softirq_vec;
        int          prev_count = preempt_count();
        kstat_incr_softirqs_this_cpu(vec_nr);

        h->action(h);                 // 指向 net_rx_action()
        h++;
        pending >>= softirq_bit;
    }

    if (__this_cpu_read(ksoftirqd) == current)
        rcu_softirq_qs();

    local_irq_disable();               // 关闭 IRQ
    pending = local_softirq_pending(); // 再次获取 pending softirq 的数量
    if (pending) {
        if (time_before(jiffies, end) && !need_resched() && --max_restart)
            goto restart;

        wakeup_softirqd();
    }

    lockdep_softirq_end(in_hardirq);
    account_irq_exit_time(current);
    __local_bh_enable(SOFTIRQ_OFFSET);

    current_restore_flags(old_flags, PF_MEMALLOC);
}

// net/core/dev.c

static __latent_entropy void
net_rx_action(struct softirq_action *h) {
    struct softnet_data *sd  = this_cpu_ptr(&softnet_data);       // 改 CPU 的 softnet_data 统计
    time_limit = jiffies + usecs_to_jiffies(netdev_budget_usecs); // 该 CPU 的所有 NAPI 变量的总 time limit
    budget     = netdev_budget;                                   // 该 CPU 的所有 NAPI 变量的总预算

    LIST_HEAD(list);   // 声明 struct list_head list 变量并初始化;
    LIST_HEAD(repoll); // 声明 struct list_head repoll 变量并初始化;

    local_irq_disable();
    list_splice_init(&sd->poll_list, &list);
    local_irq_enable();

    for (;;) {
        if list_empty(&list) {
            if !sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)  // RPS 相关逻辑，IPI 进程间中断
                goto out;
            break;
        }

        struct napi_struct *n = list_first_entry(&list, struct napi_struct, poll_list);
        budget -= napi_poll(n, &repoll); // 执行网卡驱动注册的 poll() 方法，返回的是处理的数据帧数量，
                                         // 函数返回时，那些数据帧都已经发送到上层栈进行处理了。

        if (budget <= 0 || time_after_eq(jiffies, time_limit)) { // budget 或 time limit 用完了
            sd->time_squeeze++;                                  // 更新 softnet_data.time_squeeze 计数
            break;
        }
    }

    local_irq_disable();
    list_splice_tail_init(&sd->poll_list, &list);
    list_splice_tail(&repoll, &list);
    list_splice(&list, &sd->poll_list);

    if !list_empty(&sd->poll_list)              // 在给定的 time/budget 内，没有能够处理完全部 napi
        __raise_softirq_irqoff(NET_RX_SOFTIRQ); // 关闭 NET_RX_SOFTIRQ 类型软中断，将 CPU 让给其他任务用，
                                                // 主动让出 CPU，不要让这种 softirq 独占 CPU 太久。

    // Receive Packet Steering：唤醒其他 CPU 从 ring buffer 收包。
    net_rps_action_and_irq_enable(sd);
out:
    __kfree_skb_flush();
}

// net/core/dev.c

int netdev_budget __read_mostly = 300;
// Must be at least 2 jiffes to guarantee 1 jiffy timeout
unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ; // 2000us

        if (budget <= 0 || time_after_eq(jiffies, time_limit)) { // budget 或 time limit 用完了
            sd->time_squeeze++;                                  // 更新 softnet_data.time_squeeze 计数
            break;
        }

    netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);

// drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c

int mlx5e_napi_poll(struct napi_struct *napi, int budget) { // 这里的 budget 就是上面的 weight
    struct mlx5e_channel  *c = container_of(napi, struct mlx5e_channel, napi);
    struct mlx5e_ch_stats *ch_stats = c->stats;
    struct mlx5e_xdpsq    *xsksq    = &c->xsksq;
    struct mlx5e_rq       *xskrq    = &c->xskrq;
    struct mlx5e_rq       *rq       = &c->rq;

    bool aff_change = false;
    bool busy_xsk = false;
    bool busy = false;
    int work_done = 0;
    bool xsk_open;

    xsk_open = test_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
    ch_stats->poll++;

    for (i = 0; i < c->num_tc; i++)                      // 遍历 TX channel，发包，释放 ring buffer
        busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget);  // 每次发送最多 MLX5E_TX_CQ_POLL_BUDGET 个包
                                                         // budget 结束时还有包没发完，则返回 busy=true。
    busy |= mlx5e_poll_xdpsq_cq(&c->xdpsq.cq);           // XDP TX 队列发包，有自己独立的 budget 预算

    if (c->xdp)
        busy |= mlx5e_poll_xdpsq_cq(&c->rq_xdpsq.cq);    // XDP RX 队列收包，有自己独立的 budget 预算

    if (likely(budget)) { // 如果 budget!=0，则 poll RX ring 来收包；否则就跳过 RX ring
        if (xsk_open)
            work_done = mlx5e_poll_rx_cq(&xskrq->cq, budget);

        if (likely(budget - work_done))
            work_done += mlx5e_poll_rx_cq(&rq->cq, budget - work_done);

        busy |= work_done == budget;
    }

    mlx5e_poll_ico_cq(&c->icosq.cq);
    if (mlx5e_poll_ico_cq(&c->async_icosq.cq))
        clear_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state);

    busy |= INDIRECT_CALL_2(rq->post_wqes, mlx5e_post_rx_mpwqes, mlx5e_post_rx_wqes, rq);
    if (xsk_open) {
        busy |= mlx5e_poll_xdpsq_cq(&xsksq->cq);
        busy_xsk |= mlx5e_napi_xsk_post(xsksq, xskrq);
    }

    busy |= busy_xsk;

    if (busy) {
        if (likely(mlx5e_channel_no_affinity_change(c))) {
            work_done = budget;
            goto out;
        }

        ch_stats->aff_change++;
        aff_change = true;
        if (budget && work_done == budget)
            work_done--;
    }

    if (unlikely(!napi_complete_done(napi, work_done)))
        goto out;

    ch_stats->arm++;

    for (i = 0; i < c->num_tc; i++) {
        mlx5e_handle_tx_dim(&c->sq[i]);
        mlx5e_cq_arm(&c->sq[i].cq);
    }

    mlx5e_handle_rx_dim(rq);

    mlx5e_cq_arm(&rq->cq);
    mlx5e_cq_arm(&c->icosq.cq);
    mlx5e_cq_arm(&c->async_icosq.cq);
    mlx5e_cq_arm(&c->xdpsq.cq);

    if (xsk_open) {
        mlx5e_handle_rx_dim(xskrq);
        mlx5e_cq_arm(&xsksq->cq);
        mlx5e_cq_arm(&xskrq->cq);
    }

    if (unlikely(aff_change && busy_xsk)) {
        mlx5e_trigger_irq(&c->icosq);
        ch_stats->force_irq++;
    }

    return work_done;
}

// drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) { // budget <= weight (hardcode 64)
    struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
    struct mlx5_cqwq *cqwq = &cq->wq;                            // CQ 的 WQ
    int work_done = 0;

    cqe = mlx5_cqwq_get_cqe(cqwq);

    do {
        mlx5_cqwq_pop(cqwq); // 出队，解除 DMA 映射，释放 ring buffer

        INDIRECT_CALL_2(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq, mlx5e_handle_rx_cqe, rq, cqe);
    } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(cqwq)));

out:
    if (rcu_access_pointer(rq->xdp_prog))  // 如果 attach 了 XDP 程序，
        mlx5e_xdp_rx_poll_complete(rq);    // 处理 XDP RX 队列

    mlx5_cqwq_update_db_record(cqwq);      // 更新 ring buffer 元信息
    wmb();  // 确保 CQ 空间（ring buffer）已经释放了，这样才能放新的 CQE 进来

    return work_done;                      // 返回处理的包数。注意：XDP 的包并没有统计在这里面
}

static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) {
    struct mlx5_wq_cyc         *wq = &rq->wqe.wq;
    u16                         ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
    struct mlx5e_wqe_frag_info *wi = get_frag(rq, ci);
    u32 cqe_bcnt                   = be32_to_cpu(cqe->byte_cnt);

    // 创建 skb，从 ringbuffer DMA 区域复制数据包；从这一步开始，才有了内核 skb 结构体
    struct sk_buff *skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
                                          mlx5e_skb_from_cqe_linear,
                                          mlx5e_skb_from_cqe_nonlinear,
                                          rq, cqe, wi, cqe_bcnt);
    if (!skb) { // 可能是 XDP 包
        if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
            goto wq_cyc_pop; // do not return page to cache, it will be returned on XDP_TX completion.
        }
        goto free_wqe;
    }

    mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);

    if (mlx5e_cqe_regb_chain(cqe))
        if (!mlx5e_tc_update_skb(cqe, skb)) { // TC 信息
            dev_kfree_skb_any(skb);
            goto free_wqe;
        }

    napi_gro_receive(rq->cq.napi, skb);

free_wqe:
    mlx5e_free_rx_wqe(rq, wi, true);
wq_cyc_pop:
    mlx5_wq_cyc_pop(wq);
}

// drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

static struct sk_buff *
mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) {
    struct mlx5e_dma_info *di = wi->di;
    u16 rx_headroom = rq->buff.headroom;
    struct xdp_buff xdp;
    struct sk_buff *skb;
    void *va, *data;
    u32 frag_size;

    va             = page_address(di->page) + wi->offset;
    data           = va + rx_headroom;
    frag_size      = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);

    dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
                      frag_size, DMA_FROM_DEVICE);
    net_prefetchw(va); /* xdp_frame data area */
    net_prefetch(data);

    mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
    if (mlx5e_xdp_handle(rq, di, &cqe_bcnt, &xdp))
        return NULL; /* page/packet was consumed by XDP */

    rx_headroom = xdp.data - xdp.data_hard_start;
    frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
    skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt);
    if (unlikely(!skb))
        return NULL;

    /* queue up for recycling/reuse */
    page_ref_inc(di->page);

    return skb;
}

static inline
struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va, u32 frag_size, u16 headroom, u32 cqe_bcnt) {
    struct sk_buff *skb = build_skb(va, frag_size);  // 通用内核函数
    skb_reserve(skb, headroom);
    skb_put(skb, cqe_bcnt);
    return skb;
}

static inline void
mlx5e_complete_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, u32 cqe_bcnt, struct sk_buff *skb) {
    struct mlx5e_rq_stats *stats = rq->stats;

    stats->packets++;
    stats->bytes += cqe_bcnt;
    mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb); // 初始化 L2 header
}

static inline void
mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, u32 cqe_bcnt, struct mlx5e_rq *rq, struct sk_buff *skb)
{
    u8 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
    struct net_device *netdev = rq->netdev;

    skb->mac_len = ETH_HLEN; // L2 header length

    mlx5e_tls_handle_rx_skb(rq, skb, cqe, &cqe_bcnt);
    if (unlikely(mlx5_ipsec_is_rx_flow(cqe)))
        mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, cqe);

    if (lro_num_seg > 1) {
        mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt);
        skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg);
        stats->packets += lro_num_seg - 1;
        stats->lro_packets++;
        stats->lro_bytes += cqe_bcnt;
    }

    if (unlikely(mlx5e_rx_hw_stamp(rq->tstamp))) // 硬件打时间戳
        skb_hwtstamps(skb)->hwtstamp = mlx5_timecounter_cyc2time(rq->clock, get_cqe_ts(cqe));

    skb_record_rx_queue(skb, rq->ix);

    if (likely(netdev->features & NETIF_F_RXHASH))
        mlx5e_skb_set_hash(cqe, skb);

    if (cqe_has_vlan(cqe)) {  // 添加 VLAN header
        __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), be16_to_cpu(cqe->vlan_info));
        stats->removed_vlan_packets++;
    }

    skb->mark = be32_to_cpu(cqe->sop_drop_qpn) & MLX5E_TC_FLOW_ID_MASK;
    mlx5e_handle_csum(netdev, cqe, rq, skb, !!lro_num_seg); // L2 校验和
    if (unlikely(cqe->ml_path & MLX5E_CE_BIT_MASK))
        mlx5e_enable_ecn(rq, skb);                          // L2 纠错

    skb->protocol = eth_type_trans(skb, netdev);            // 函数里面还会设置 skb->pkt_type
}

INDIRECT_CALLABLE_SCOPE bool
mlx5e_post_rx_wqes(struct mlx5e_rq *rq) {
    struct mlx5_wq_cyc *wq = &rq->wqe.wq;
    u8 wqe_bulk = rq->wqe.info.wqe_bulk;

    if (mlx5_wq_cyc_missing(wq) < wqe_bulk)
        return false;

    do {
        u16 head = mlx5_wq_cyc_get_head(wq);
        mlx5e_alloc_rx_wqes(rq, head, wqe_bulk);

        mlx5_wq_cyc_push_n(wq, wqe_bulk);
    } while (mlx5_wq_cyc_missing(wq) >= wqe_bulk);

    dma_wmb(); /* ensure wqes are visible to device before updating doorbell record */

    mlx5_wq_cyc_update_db_record(wq);

    return !!err;
}

// net/core/dev.c

gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) {
    skb_mark_napi_id(skb, napi);
    skb_gro_reset_offset(skb, 0);

    return napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));
}

// https://github.com/torvalds/linux/blob/v5.10/net/core/dev.c

static struct list_head offload_base __read_mostly;  // offload list

static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) {
    struct list_head *gro_head = gro_list_prepare(napi, skb);

    struct sk_buff *pp;
    struct packet_offload *ptype;                 // packet offload 相关结构体
    struct list_head      *head = &offload_base;
    list_for_each_entry_rcu(ptype, head, list) {
        if (ptype->type != skb->protocol || !ptype->callbacks.gro_receive)
            continue; // L2/L3 协议类型不同，或者没有回调方法，只能跳过

        skb_set_network_header(skb, skb_gro_offset(skb));
        skb_reset_mac_len(skb);
        NAPI_GRO_CB(skb)->same_flow = 0;
        NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
        NAPI_GRO_CB(skb)->free = 0;
        NAPI_GRO_CB(skb)->encap_mark = 0;
        NAPI_GRO_CB(skb)->recursion_counter = 0;
        NAPI_GRO_CB(skb)->is_fou = 0;
        NAPI_GRO_CB(skb)->gro_remcsum_start = 0;

        /* Setup for GRO checksum validation */
        ...

        pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive, ipv6_gro_receive, inet_gro_receive, gro_head, skb);
        break;
    }

    if (&ptype->list == head)
        goto normal;

    u32 hash = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
    if (pp) { // 如果 pp 非空，说明这已经是一个完成的包，可以送到更上层去处理了
        skb_list_del_init(pp);
        napi_gro_complete(napi, pp);    // 将包送到协议栈
        napi->gro_hash[hash].count--;
    }

    // 执行到这里说明这个包不完整，需要执行 GRO 合并
    int same_flow = NAPI_GRO_CB(skb)->same_flow;
    if (same_flow) // 这个包属于已经存在的一个 flow，需要和其他包合并
        goto ok;

    if (NAPI_GRO_CB(skb)->flush) // 合并完成了，已经拿到完整包
        goto normal;

    if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) { // 新建一个 entry 加到本 CPU 的 NAPI 变量的 gro_list
        gro_flush_oldest(napi, gro_head);
    } else {
        napi->gro_hash[hash].count++;
    }
    NAPI_GRO_CB(skb)->count = 1;
    NAPI_GRO_CB(skb)->age = jiffies;
    NAPI_GRO_CB(skb)->last = skb;
    skb_shinfo(skb)->gso_size = skb_gro_len(skb);
    list_add(&skb->list, gro_head);
    ret = GRO_HELD;

pull:
    int grow = skb_gro_offset(skb) - skb_headlen(skb);
    if (grow > 0)
        gro_pull_from_frag0(skb, grow);
ok:
    if (napi->gro_hash[hash].count) {
        if (!test_bit(hash, &napi->gro_bitmask))
            __set_bit(hash, &napi->gro_bitmask);
    } else if (test_bit(hash, &napi->gro_bitmask)) {
        __clear_bit(hash, &napi->gro_bitmask);
    }

    return ret;

normal:
    ret = GRO_NORMAL;
    goto pull;
}

static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb) {
    if (NAPI_GRO_CB(skb)->count == 1) {
        skb_shinfo(skb)->gso_size = 0;
        goto out;
    }

    struct packet_offload *ptype;
    struct list_head      *head = &offload_base;
    list_for_each_entry_rcu(ptype, head, list) {
        if (ptype->type != skb->protocol || !ptype->callbacks.gro_complete)
            continue;

        INDIRECT_CALL_INET(ptype->callbacks.gro_complete, ipv6_gro_complete, inet_gro_complete, skb, 0);
        break;
    }

out:
    gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count);
    return NET_RX_SUCCESS;
}

// net/ipv4/af_inet.c
int inet_gro_complete(struct sk_buff *skb, int nhoff) {
    struct iphdr *iph = (struct iphdr *)(skb->data + nhoff); // IP header 起始位置

    if (skb->encapsulation) {
        skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IP));
        skb_set_inner_network_header(skb, nhoff);
    }

    __be16 newlen = htons(skb->len - nhoff);
    csum_replace2(&iph->check, iph->tot_len, newlen);        // IP header checksum
    iph->tot_len = newlen;

    const struct net_offload *ops = rcu_dereference(inet_offloads[iph->protocol]);

    /* Only need to add sizeof(*iph) to get to the next hdr below
     * because any hdr with option will have been flushed in
     * inet_gro_receive().
     */
    INDIRECT_CALL_2(ops->callbacks.gro_complete, tcp4_gro_complete, udp4_gro_complete, skb, nhoff + sizeof(*iph));
}

static gro_result_t napi_skb_finish(struct napi_struct *napi, struct sk_buff *skb, gro_result_t ret) {
    switch (ret) {
    case GRO_NORMAL:
        gro_normal_one(napi, skb, 1); break;
    case GRO_DROP:
        kfree_skb(skb); break;
    case GRO_MERGED_FREE:  // 已经被合并，释放 skb
        if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
            napi_skb_free_stolen_head(skb);
        else
            __kfree_skb(skb);
        break;

    case GRO_HELD:
    case GRO_MERGED:
    case GRO_CONSUMED:
        break;
    }

    return ret;
}

// Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, pass the whole batch up to the stack.
static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs) {
    napi->rx_count += segs;
    if (napi->rx_count >= gro_normal_batch) // int gro_normal_batch __read_mostly = 8;
        gro_normal_list(napi); // 
}

// Pass the currently batched GRO_NORMAL SKBs up to the stack.
static void gro_normal_list(struct napi_struct *napi) {
    if (!napi->rx_count)            // 没有包的话直接返回，什么都不做
        return;

    netif_receive_skb_list_internal(&napi->rx_list); // 注意这个函数没有返回值，所有错误都在内部处理

    INIT_LIST_HEAD(&napi->rx_list); // 把这个 NAPI 重新入队，并放到队首（下次最后一个被处理，因为刚处理过一次了）
    napi->rx_count = 0;             // 重置计数
}

// net/core/net-procfs.c

static int softnet_seq_show(struct seq_file *seq, void *v) {
    struct softnet_data *sd = v;

    seq_printf(seq,
           "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
           sd->processed, sd->dropped, sd->time_squeeze, 0,
           0, 0, 0, 0, /* was fastroute */
           0,    /* was cpu_collision */
           sd->received_rps, flow_limit_count,
           softnet_backlog_len(sd), (int)seq->index);
    return 0;
}

// net/core/dev.c

static void netif_receive_skb_list_internal(struct list_head *head) {
    struct list_head sublist;
    list_for_each_entry_safe(skb, next, head, list) { // struct sk_buff *skb, *next;
        net_timestamp_check(netdev_tstamp_prequeue, skb);
        skb_list_del_init(skb);
        if !skb_defer_rx_timestamp(skb)
            list_add_tail(&skb->list, &sublist);
    }
    list_splice_init(&sublist, head);

    if static_branch_unlikely(&rps_needed)             // RPS 处理逻辑
        list_for_each_entry_safe(skb, next, head, list) {
            int cpu = get_rps_cpu(skb->dev, skb, &rflow);
            if (cpu >= 0) {
                skb_list_del_init(skb); /* Will be handled, remove from list */
                enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
            }
        }

    __netif_receive_skb_list(head);
}

static void __netif_receive_skb_list(struct list_head *head) {
    unsigned long noreclaim_flag = 0;
    struct sk_buff *skb, *next;
    bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */

    list_for_each_entry_safe(skb, next, head, list) {
        if (sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc {
            list_cut_before(&sublist, head, &skb->list); /* Handle the previous sublist */
            if (!list_empty(&sublist))
                __netif_receive_skb_list_core(&sublist, pfmemalloc);

            pfmemalloc = !pfmemalloc;
            if (pfmemalloc) /* See comments in __netif_receive_skb */
                noreclaim_flag = memalloc_noreclaim_save();
            else
                memalloc_noreclaim_restore(noreclaim_flag);
        }
    }

    if !list_empty(head) /* Handle the remaining sublist */
        __netif_receive_skb_list_core(head, pfmemalloc);

    if (pfmemalloc) /* Restore pflags */
        memalloc_noreclaim_restore(noreclaim_flag);
}

static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc) {
    // Fast-path assumptions:
    // - There is no RX handler.
    // - Only one packet_type matches.
    //
    // If either of these fails, we will end up doing some per-packet
    // processing in-line, then handling the 'last ptype' for the whole
    // sublist.  This can't cause out-of-order delivery to any single ptype,
    // because the 'last ptype' must be constant across the sublist, and all
    // other ptypes are handled per-packet.

    struct packet_type *pt_curr = NULL; // Current (common) ptype of sublist */
    struct net_device  *od_curr = NULL; // Current (common) orig_dev of sublist */
    struct sk_buff *skb, *next;

    INIT_LIST_HEAD(&sublist);
    list_for_each_entry_safe(skb, next, head, list) {
        struct net_device *orig_dev = skb->dev;
        struct packet_type *pt_prev = NULL;

        skb_list_del_init(skb);
        __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
        if (!pt_prev)
            continue;

        if (pt_curr != pt_prev || od_curr != orig_dev) {
            __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr); /* dispatch old sublist */

            INIT_LIST_HEAD(&sublist); /* start new sublist */
            pt_curr = pt_prev;
            od_curr = orig_dev;
        }
        list_add_tail(&skb->list, &sublist);
    }

    /* dispatch final sublist */
    __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
}

// https://github.com/torvalds/linux/blob/v5.10/net/core/dev.c

static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc, struct packet_type **ppt_prev) {
    struct sk_buff *skb = *pskb;
    net_timestamp_check(!netdev_tstamp_prequeue, skb); // 检查时间戳

    skb_reset_network_header(skb);
    if (!skb_transport_header_was_set(skb))
        skb_reset_transport_header(skb);
    skb_reset_mac_len(skb);

    struct packet_type *ptype    = NULL;
    struct net_device  *orig_dev = skb->dev;  // 记录 skb 原来所在的网络设备

another_round:
    skb->skb_iif = skb->dev->ifindex;         // 设置 skb 是从那个网络设备接收的
    __this_cpu_inc(softnet_data.processed);   // 更新 softnet_data.processed 计数

    if (static_branch_unlikely(&generic_xdp_needed_key)) { // Generic XDP（软件实现 XDP 功能）
        preempt_disable();
        ret2 = do_xdp_generic(skb->dev->xdp_prog, skb);
        preempt_enable();

        if (ret2 != XDP_PASS) {
            ret = NET_RX_DROP;
            goto out;
        }
        skb_reset_mac_len(skb);
    }

    if (skb->protocol == ETH_P_8021Q || skb->protocol == ETH_P_8021AD)
        skb = skb_vlan_untag(skb);

    if (skb_skip_tc_classify(skb))
        goto skip_classify;

    if (pfmemalloc)        // 跳过抓包逻辑
        goto skip_taps;

    list_for_each_entry_rcu(ptype, &ptype_all, list) {          // 抓包
        if (pt_prev)
            ret = deliver_skb(skb, pt_prev, orig_dev);
        pt_prev = ptype;
    }

    list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { // 抓包
        if (pt_prev)
            ret = deliver_skb(skb, pt_prev, orig_dev);
        pt_prev = ptype;
    }

skip_taps: // 如果是使用 goto 跳转过来的，那跳过了抓包逻辑（libpcap、tcpdump 等）
    if (static_branch_unlikely(&ingress_needed_key)) {           // TC ingress 处理
        bool another = false;
        skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev, &another);
        if (another)             // TCP BPF 优化，通过 another round 将包从宿主机网卡直接送到容器 netns 内网卡
            goto another_round;  // 需要内核 5.10+
        if (!skb)
            goto out;

        if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)       // Netfilter  ingress 处理
            goto out;
    }
    skb_reset_redirect(skb);                                     // 重置 redirect

skip_classify: // 如果是使用 goto 跳转过来的，那跳过了抓包、TC、Netfilter 逻辑
    if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
        goto drop;

    if (skb_vlan_tag_present(skb)) { // 处理 VLAN 头
        if (pt_prev) {
            ret = deliver_skb(skb, pt_prev, orig_dev);
            pt_prev = NULL;
        }
        if (vlan_do_receive(&skb))
            goto another_round;
    }

    bool deliver_exact = false;
    rx_handler = rcu_dereference(skb->dev->rx_handler);
    if (rx_handler) {
        if (pt_prev) {
            ret = deliver_skb(skb, pt_prev, orig_dev);
            pt_prev = NULL;
        }
        switch (rx_handler(&skb)) {
        case RX_HANDLER_CONSUMED:
            ret = NET_RX_SUCCESS;
            goto out;
        case RX_HANDLER_ANOTHER:
            goto another_round;
        case RX_HANDLER_EXACT:
            deliver_exact = true;
        case RX_HANDLER_PASS:
            break;
        }
    }

    if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
check_vlan_id:
        // 一些 VLAN 特殊处理
    }

    type = skb->protocol;

    /* deliver only exact match when indicated */
    if (likely(!deliver_exact)) {
        deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, &ptype_base[ntohs(type) & PTYPE_HASH_MASK]);
    }

    deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, &orig_dev->ptype_specific);

    if (unlikely(skb->dev != orig_dev)) {
        deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, &skb->dev->ptype_specific);
    }

    if (pt_prev) {
        if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
            goto drop;
        *ppt_prev = pt_prev;
    } else {
drop:
        if (!deliver_exact)
            atomic_long_inc(&skb->dev->rx_dropped);   // 更新网卡的 rx_dropped 统计
        else
            atomic_long_inc(&skb->dev->rx_nohandler); // 更新网卡的 rx_nohandler 统计
        kfree_skb(skb);
        ret = NET_RX_DROP;
    }

out:
    *pskb = skb;
    return ret;
}

// https://github.com/torvalds/linux/blob/v5.10/net/core/dev.c

extern struct list_head ptype_all                   __read_mostly;
extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;

static inline void __netif_receive_skb_list_ptype(struct list_head *head,
                          struct packet_type *pt_prev, struct net_device *orig_dev) {
    if (!pt_prev)
        return;

    if (pt_prev->list_func != NULL)
        INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv, ip_list_rcv, head, pt_prev, orig_dev);
    else
        list_for_each_entry_safe(skb, next, head, list) { // 遍历 list 中的每个 skb
            skb_list_del_init(skb);
            pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
        }
}

// https://github.com/torvalds/linux/blob/v5.10/net/ipv4/af_inet.c

static struct packet_type ip_packet_type __read_mostly = {
    .type = cpu_to_be16(ETH_P_IP),
    .func = ip_rcv,
    .list_func = ip_list_rcv,
};

static int __init inet_init(void) {
    // 分配各种 L4 协议的内存空间和协议号（proto index），并插入 L4 协议链表
    proto_register(&tcp_prot, 1);
    proto_register(&udp_prot, 1);
    proto_register(&raw_prot, 1);
    proto_register(&ping_prot, 1);

    (void)sock_register(&inet_family_ops); //  Tell SOCKET that we are alive...

    ip_static_sysctl_init(); // 注册路由相关的 sysctl 参数：net.ipv4.route.*

    // 注册几种基础 L4 协议的 rx handler
    inet_add_protocol(&icmp_protocol, IPPROTO_ICMP);
    inet_add_protocol(&udp_protocol,  IPPROTO_UDP);
    inet_add_protocol(&tcp_protocol,  IPPROTO_TCP);
    inet_add_protocol(&igmp_protocol, IPPROTO_IGMP);

    /* Register the socket-side information for inet_create. */
    for (struct list_head *r = &inetsw[0]; r < &inetsw[SOCK_MAX]; ++r)
        INIT_LIST_HEAD(r);
    for (struct inet_protosw *q = inetsw_array; q < &inetsw_array[INETSW_ARRAY_LEN]; ++q)
        inet_register_protosw(q);

    // 各种协议模块的初始化
    arp_init();          // Set the ARP module up
    ip_init();           // Set the IP module up
    init_ipv4_mibs();    // Initialise per-cpu ipv4 mibs
    tcp_init();          // Setup TCP slab cache for open requests
    udp_init();          // Setup UDP memory threshold
    udplite4_register(); // Add UDP-Lite (RFC 3828)
    raw_init();
    ping_init();
    icmp_init();         // Set the ICMP layer up
    ip_mr_init();        //  Initialise the multicast router

    init_inet_pernet_ops();

    ipv4_proc_init();    // 注册 /proc/net/*， 例如 cat /proc/net/tcp 能看到所有 TCP socket 的状态统计
    ipfrag_init();

    dev_add_pack(&ip_packet_type); // 注册 L3 rx handler：ip_rcv()/ip_list_rcv()

    ip_tunnel_core_init();
}

// https://github.com/torvalds/linux/blob/v5.10/net/ipv4/ip_input.c

// IP receive entry point
int ip_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) {
    struct net *net = dev_net(dev);

    skb = ip_rcv_core(skb, net);
    if (skb == NULL)
        return NET_RX_DROP;

    return NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING, net, NULL, skb, dev, NULL, ip_rcv_finish);
}

// https://github.com/torvalds/linux/blob/v5.10/net/ipv4/ip_input.c

static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb) {
    struct net_device *dev = skb->dev;

    skb = l3mdev_ip_rcv(skb); // if ingress device is enslaved to an L3 master device
    if (!skb)                 // pass the skb to its handler for processing
        return NET_RX_SUCCESS;

    ret = ip_rcv_finish_core(net, sk, skb, dev, NULL);
    if (ret != NET_RX_DROP)
        ret = dst_input(skb);
    return ret;
}

static int ip_rcv_finish_core(struct net *net, struct sock *sk, struct sk_buff *skb, struct net_device *dev, const struct sk_buff *hint) {
    const struct iphdr *iph = ip_hdr(skb);
    int (*edemux)(struct sk_buff *skb);
    struct rtable *rt;
    int err;

    if (ip_can_use_hint(skb, iph, hint)) {
        err = ip_route_use_hint(skb, iph->daddr, iph->saddr, iph->tos, dev, hint);
        if (unlikely(err))
            goto drop_error;
    }

    if (net->ipv4.sysctl_ip_early_demux && !skb_dst(skb) && !skb->sk && !ip_is_fragment(iph)) {
        const struct net_protocol *ipprot;
        int protocol = iph->protocol;

        ipprot = rcu_dereference(inet_protos[protocol]);
        if (ipprot && (edemux = READ_ONCE(ipprot->early_demux))) {
            err = INDIRECT_CALL_2(edemux, tcp_v4_early_demux, udp_v4_early_demux, skb);
            if (unlikely(err))
                goto drop_error;
            /* must reload iph, skb->head might have changed */
            iph = ip_hdr(skb);
        }
    }

    /*
     *    Initialise the virtual path cache for the packet. It describes
     *    how the packet travels inside Linux networking.
     */
    if (!skb_valid_dst(skb)) {
        err = ip_route_input_noref(skb, iph->daddr, iph->saddr, iph->tos, dev);
        if (unlikely(err))
            goto drop_error;
    }

#ifdef CONFIG_IP_ROUTE_CLASSID
    if (unlikely(skb_dst(skb)->tclassid)) {
        struct ip_rt_acct *st = this_cpu_ptr(ip_rt_acct);
        u32 idx = skb_dst(skb)->tclassid;
        st[idx&0xFF].o_packets++;
        st[idx&0xFF].o_bytes += skb->len;
        st[(idx>>16)&0xFF].i_packets++;
        st[(idx>>16)&0xFF].i_bytes += skb->len;
    }
#endif

    if (iph->ihl > 5 && ip_rcv_options(skb, dev))
        goto drop;

    rt = skb_rtable(skb);
    if (rt->rt_type == RTN_MULTICAST) {
        __IP_UPD_PO_STATS(net, IPSTATS_MIB_INMCAST, skb->len);
    } else if (rt->rt_type == RTN_BROADCAST) {
        __IP_UPD_PO_STATS(net, IPSTATS_MIB_INBCAST, skb->len);
    } else if (skb->pkt_type == PACKET_BROADCAST ||
           skb->pkt_type == PACKET_MULTICAST) {
        struct in_device *in_dev = __in_dev_get_rcu(dev);

        if (in_dev && IN_DEV_ORCONF(in_dev, DROP_UNICAST_IN_L2_MULTICAST))
            goto drop;
    }

    return NET_RX_SUCCESS;

drop:
    kfree_skb(skb);
    return NET_RX_DROP;

drop_error:
    if (err == -EXDEV)
        __NET_INC_STATS(net, LINUX_MIB_IPRPFILTER);
    goto drop;
}