From f6baca2d32ead51b10e15f1eef812d7c6a7b9a40 Mon Sep 17 00:00:00 2001 From: Andrew Melnychenko Date: Mon, 15 Jan 2024 21:48:40 +0200 Subject: [PATCH 01/35] vhost: Added pad cleanup if vnet_hdr is not present. When the Qemu launched with vhost but without tap vnet_hdr, vhost tries to copy vnet_hdr from socket iter with size 0 to the page that may contain some trash. That trash can be interpreted as unpredictable values for vnet_hdr. That leads to dropping some packets and in some cases to stalling vhost routine when the vhost_net tries to process packets and fails in a loop. Qemu options: -netdev tap,vhost=on,vnet_hdr=off,... Signed-off-by: Andrew Melnychenko Message-Id: <20240115194840.1183077-1-andrew@daynix.com> Signed-off-by: Michael S. Tsirkin --- drivers/vhost/net.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index f2ed7167c848..57411ac2d08b 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -735,6 +735,9 @@ static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq, hdr = buf; gso = &hdr->gso; + if (!sock_hlen) + memset(buf, 0, pad); + if ((gso->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && vhost16_to_cpu(vq, gso->csum_start) + vhost16_to_cpu(vq, gso->csum_offset) + 2 > From ec6ecb844d14d38b7dae8beb74e3d65db9c7b3e6 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Thu, 25 Jan 2024 23:20:39 +0000 Subject: [PATCH 02/35] virtio: uapi: Drop __packed attribute in linux/virtio_pci.h Commit 92792ac752aa ("virtio-pci: Introduce admin command sending function") added "__packed" structures to UAPI header linux/virtio_pci.h. This triggers build failures in the consumer userspace applications without proper "definition" of __packed (e.g., kvmtool build fails). Moreover, the structures are already packed well, and doesn't need explicit packing, similar to the rest of the structures in all virtio_* headers. Remove the __packed attribute. Fixes: 92792ac752aa ("virtio-pci: Introduce admin command sending function") Cc: Feng Liu Cc: Michael S. Tsirkin Cc: Yishai Hadas Cc: Alex Williamson Cc: Jean-Philippe Brucker Reviewed-by: Jean-Philippe Brucker Acked-by: Michael S. Tsirkin Signed-off-by: Suzuki K Poulose Message-Id: <20240125232039.913606-1-suzuki.poulose@arm.com> Signed-off-by: Michael S. Tsirkin --- include/uapi/linux/virtio_pci.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/uapi/linux/virtio_pci.h b/include/uapi/linux/virtio_pci.h index ef3810dee7ef..a8208492e822 100644 --- a/include/uapi/linux/virtio_pci.h +++ b/include/uapi/linux/virtio_pci.h @@ -240,7 +240,7 @@ struct virtio_pci_cfg_cap { #define VIRTIO_ADMIN_CMD_LEGACY_DEV_CFG_READ 0x5 #define VIRTIO_ADMIN_CMD_LEGACY_NOTIFY_INFO 0x6 -struct __packed virtio_admin_cmd_hdr { +struct virtio_admin_cmd_hdr { __le16 opcode; /* * 1 - SR-IOV @@ -252,20 +252,20 @@ struct __packed virtio_admin_cmd_hdr { __le64 group_member_id; }; -struct __packed virtio_admin_cmd_status { +struct virtio_admin_cmd_status { __le16 status; __le16 status_qualifier; /* Unused, reserved for future extensions. */ __u8 reserved2[4]; }; -struct __packed virtio_admin_cmd_legacy_wr_data { +struct virtio_admin_cmd_legacy_wr_data { __u8 offset; /* Starting offset of the register(s) to write. */ __u8 reserved[7]; __u8 registers[]; }; -struct __packed virtio_admin_cmd_legacy_rd_data { +struct virtio_admin_cmd_legacy_rd_data { __u8 offset; /* Starting offset of the register(s) to read. */ }; @@ -275,7 +275,7 @@ struct __packed virtio_admin_cmd_legacy_rd_data { #define VIRTIO_ADMIN_CMD_MAX_NOTIFY_INFO 4 -struct __packed virtio_admin_cmd_notify_info_data { +struct virtio_admin_cmd_notify_info_data { __u8 flags; /* 0 = end of list, 1 = owner device, 2 = member device */ __u8 bar; /* BAR of the member or the owner device */ __u8 padding[6]; From 9588e7fc511f9c55b9835f14916e90ab940061b7 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 9 Feb 2024 14:30:07 -0800 Subject: [PATCH 03/35] vdpa_sim: reset must not run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vdpasim_do_reset sets running to true, which is wrong, as it allows vdpasim_kick_vq to post work requests before the device has been configured. To fix, do not set running until VIRTIO_CONFIG_S_DRIVER_OK is set. Fixes: 0c89e2a3a9d0 ("vdpa_sim: Implement suspend vdpa op") Signed-off-by: Steve Sistare Reviewed-by: Eugenio Pérez Acked-by: Jason Wang Message-Id: <1707517807-137331-1-git-send-email-steven.sistare@oracle.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_sim/vdpa_sim.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index be2925d0d283..18584ce70bf0 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -160,7 +160,7 @@ static void vdpasim_do_reset(struct vdpasim *vdpasim, u32 flags) } } - vdpasim->running = true; + vdpasim->running = false; spin_unlock(&vdpasim->iommu_lock); vdpasim->features = 0; @@ -483,6 +483,7 @@ static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status) mutex_lock(&vdpasim->mutex); vdpasim->status = status; + vdpasim->running = (status & VIRTIO_CONFIG_S_DRIVER_OK) != 0; mutex_unlock(&vdpasim->mutex); } From 310227f42882c52356b523e2f4e11690eebcd2ab Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 13 Feb 2024 14:54:25 +0100 Subject: [PATCH 04/35] virtio: reenable config if freezing device failed Currently, we don't reenable the config if freezing the device failed. For example, virtio-mem currently doesn't support suspend+resume, and trying to freeze the device will always fail. Afterwards, the device will no longer respond to resize requests, because it won't get notified about config changes. Let's fix this by re-enabling the config if freezing fails. Fixes: 22b7050a024d ("virtio: defer config changed notifications") Cc: Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Xuan Zhuo Signed-off-by: David Hildenbrand Message-Id: <20240213135425.795001-1-david@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index f4080692b351..f513ee21b1c1 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -510,8 +510,10 @@ int virtio_device_freeze(struct virtio_device *dev) if (drv && drv->freeze) { ret = drv->freeze(dev); - if (ret) + if (ret) { + virtio_config_enable(dev); return ret; + } } if (dev->config->destroy_avq) From c4e8b5aed7cd18bb8377b6dd461e336688c5d503 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Tue, 13 Feb 2024 06:25:58 -0800 Subject: [PATCH 05/35] vdpa: skip suspend/resume ops if not DRIVER_OK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a vdpa device is not in state DRIVER_OK, then there is no driver state to preserve, so no need to call the suspend and resume driver ops. Suggested-by: Eugenio Perez Martin " Signed-off-by: Steve Sistare Message-Id: <1707834358-165470-1-git-send-email-steven.sistare@oracle.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Eugenio Pérez --- drivers/vhost/vdpa.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index bc4a51e4638b..aef92a7c57f3 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -595,6 +595,9 @@ static long vhost_vdpa_suspend(struct vhost_vdpa *v) const struct vdpa_config_ops *ops = vdpa->config; int ret; + if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK)) + return 0; + if (!ops->suspend) return -EOPNOTSUPP; @@ -615,6 +618,9 @@ static long vhost_vdpa_resume(struct vhost_vdpa *v) const struct vdpa_config_ops *ops = vdpa->config; int ret; + if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK)) + return 0; + if (!ops->resume) return -EOPNOTSUPP; From 749a4016839270163efc36ecddddd01de491a16b Mon Sep 17 00:00:00 2001 From: Jonah Palmer Date: Fri, 16 Feb 2024 09:25:02 -0500 Subject: [PATCH 06/35] vdpa/mlx5: Allow CVQ size changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MLX driver was not updating its control virtqueue size at set_vq_num and instead always initialized to MLX5_CVQ_MAX_ENT (16) at setup_cvq_vring. Qemu would try to set the size to 64 by default, however, because the CVQ size always was initialized to 16, an error would be thrown when sending >16 control messages (as used-ring entry 17 is initialized to 0). For example, starting a guest with x-svq=on and then executing the following command would produce the error below: # for i in {1..20}; do ifconfig eth0 hw ether XX:xx:XX:xx:XX:XX; done qemu-system-x86_64: Insufficient written data (0) [ 435.331223] virtio_net virtio0: Failed to set mac address by vq command. SIOCSIFHWADDR: Invalid argument Acked-by: Dragos Tatulea Acked-by: Eugenio Pérez Signed-off-by: Jonah Palmer Message-Id: <20240216142502.78095-1-jonah.palmer@oracle.com> Signed-off-by: Michael S. Tsirkin Tested-by: Lei Yang Fixes: 5262912ef3cf ("vdpa/mlx5: Add support for control VQ and MAC setting") --- drivers/vdpa/mlx5/net/mlx5_vnet.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index 778821bab7d9..ecfc16151d61 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -151,8 +151,6 @@ static void teardown_driver(struct mlx5_vdpa_net *ndev); static bool mlx5_vdpa_debug; -#define MLX5_CVQ_MAX_ENT 16 - #define MLX5_LOG_VIO_FLAG(_feature) \ do { \ if (features & BIT_ULL(_feature)) \ @@ -2276,9 +2274,16 @@ static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); struct mlx5_vdpa_virtqueue *mvq; - if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) + if (!is_index_valid(mvdev, idx)) return; + if (is_ctrl_vq_idx(mvdev, idx)) { + struct mlx5_control_vq *cvq = &mvdev->cvq; + + cvq->vring.vring.num = num; + return; + } + mvq = &ndev->vqs[idx]; mvq->num_ent = num; } @@ -2963,7 +2968,7 @@ static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev) u16 idx = cvq->vring.last_avail_idx; err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features, - MLX5_CVQ_MAX_ENT, false, + cvq->vring.vring.num, false, (struct vring_desc *)(uintptr_t)cvq->desc_addr, (struct vring_avail *)(uintptr_t)cvq->driver_addr, (struct vring_used *)(uintptr_t)cvq->device_addr); From d7b4e3287ca3a7baf66efd9158498e551a9550da Mon Sep 17 00:00:00 2001 From: Maxime Coquelin Date: Mon, 19 Feb 2024 18:06:06 +0100 Subject: [PATCH 07/35] vduse: implement DMA sync callbacks Since commit 295525e29a5b ("virtio_net: merge dma operations when filling mergeable buffers"), VDUSE device require support for DMA's .sync_single_for_cpu() operation as the memory is non-coherent between the device and CPU because of the use of a bounce buffer. This patch implements both .sync_single_for_cpu() and .sync_single_for_device() callbacks, and also skip bounce buffer copies during DMA map and unmap operations if the DMA_ATTR_SKIP_CPU_SYNC attribute is set to avoid extra copies of the same buffer. Signed-off-by: Maxime Coquelin Message-Id: <20240219170606.587290-1-maxime.coquelin@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_user/iova_domain.c | 27 ++++++++++++++++++++++++--- drivers/vdpa/vdpa_user/iova_domain.h | 8 ++++++++ drivers/vdpa/vdpa_user/vduse_dev.c | 22 ++++++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c index 5e4a77b9bae6..791d38d6284c 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.c +++ b/drivers/vdpa/vdpa_user/iova_domain.c @@ -373,6 +373,26 @@ static void vduse_domain_free_iova(struct iova_domain *iovad, free_iova_fast(iovad, iova >> shift, iova_len); } +void vduse_domain_sync_single_for_device(struct vduse_iova_domain *domain, + dma_addr_t dma_addr, size_t size, + enum dma_data_direction dir) +{ + read_lock(&domain->bounce_lock); + if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) + vduse_domain_bounce(domain, dma_addr, size, DMA_TO_DEVICE); + read_unlock(&domain->bounce_lock); +} + +void vduse_domain_sync_single_for_cpu(struct vduse_iova_domain *domain, + dma_addr_t dma_addr, size_t size, + enum dma_data_direction dir) +{ + read_lock(&domain->bounce_lock); + if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) + vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE); + read_unlock(&domain->bounce_lock); +} + dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, @@ -393,7 +413,8 @@ dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain, if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa)) goto err_unlock; - if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE); read_unlock(&domain->bounce_lock); @@ -411,9 +432,9 @@ void vduse_domain_unmap_page(struct vduse_iova_domain *domain, enum dma_data_direction dir, unsigned long attrs) { struct iova_domain *iovad = &domain->stream_iovad; - read_lock(&domain->bounce_lock); - if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE); vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size); diff --git a/drivers/vdpa/vdpa_user/iova_domain.h b/drivers/vdpa/vdpa_user/iova_domain.h index 173e979b84a9..f92f22a7267d 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.h +++ b/drivers/vdpa/vdpa_user/iova_domain.h @@ -44,6 +44,14 @@ int vduse_domain_set_map(struct vduse_iova_domain *domain, void vduse_domain_clear_map(struct vduse_iova_domain *domain, struct vhost_iotlb *iotlb); +void vduse_domain_sync_single_for_device(struct vduse_iova_domain *domain, + dma_addr_t dma_addr, size_t size, + enum dma_data_direction dir); + +void vduse_domain_sync_single_for_cpu(struct vduse_iova_domain *domain, + dma_addr_t dma_addr, size_t size, + enum dma_data_direction dir); + dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index 1d24da79c399..75354ce186a1 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -798,6 +798,26 @@ static const struct vdpa_config_ops vduse_vdpa_config_ops = { .free = vduse_vdpa_free, }; +static void vduse_dev_sync_single_for_device(struct device *dev, + dma_addr_t dma_addr, size_t size, + enum dma_data_direction dir) +{ + struct vduse_dev *vdev = dev_to_vduse(dev); + struct vduse_iova_domain *domain = vdev->domain; + + vduse_domain_sync_single_for_device(domain, dma_addr, size, dir); +} + +static void vduse_dev_sync_single_for_cpu(struct device *dev, + dma_addr_t dma_addr, size_t size, + enum dma_data_direction dir) +{ + struct vduse_dev *vdev = dev_to_vduse(dev); + struct vduse_iova_domain *domain = vdev->domain; + + vduse_domain_sync_single_for_cpu(domain, dma_addr, size, dir); +} + static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, @@ -858,6 +878,8 @@ static size_t vduse_dev_max_mapping_size(struct device *dev) } static const struct dma_map_ops vduse_dev_dma_ops = { + .sync_single_for_device = vduse_dev_sync_single_for_device, + .sync_single_for_cpu = vduse_dev_sync_single_for_cpu, .map_page = vduse_dev_map_page, .unmap_page = vduse_dev_unmap_page, .alloc = vduse_dev_alloc_coherent, From ba6faaa68ddab7961fc1be10193ee731c6e2a1f5 Mon Sep 17 00:00:00 2001 From: Shannon Nelson Date: Mon, 19 Feb 2024 17:10:50 -0800 Subject: [PATCH 08/35] vdpa/pds: fixes for VF vdpa flr-aer handling This addresses a couple of things found while testing the FLR and AER handling with the VFs. - release irqs before calling vp_modern_remove() - make sure we have a valid struct pointer before using it to release irqs - make sure the FW is alive before trying to add a new device Signed-off-by: Shannon Nelson Message-Id: <20240220011050.30913-1-shannon.nelson@amd.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/pds/aux_drv.c | 2 +- drivers/vdpa/pds/vdpa_dev.c | 20 +++++++++++++++++--- drivers/vdpa/pds/vdpa_dev.h | 1 + 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c index 186e9ee22eb1..f57330cf9024 100644 --- a/drivers/vdpa/pds/aux_drv.c +++ b/drivers/vdpa/pds/aux_drv.c @@ -93,8 +93,8 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev) struct device *dev = &aux_dev->dev; vdpa_mgmtdev_unregister(&vdpa_aux->vdpa_mdev); + pds_vdpa_release_irqs(vdpa_aux->pdsv); vp_modern_remove(&vdpa_aux->vd_mdev); - pci_free_irq_vectors(vdpa_aux->padev->vf_pdev); pds_vdpa_debugfs_del_vdpadev(vdpa_aux); kfree(vdpa_aux); diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c index 25c0fe5ec3d5..301d95e08596 100644 --- a/drivers/vdpa/pds/vdpa_dev.c +++ b/drivers/vdpa/pds/vdpa_dev.c @@ -426,12 +426,18 @@ err_release: return err; } -static void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv) +void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv) { - struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev; - struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux; + struct pds_vdpa_aux *vdpa_aux; + struct pci_dev *pdev; int qid; + if (!pdsv) + return; + + pdev = pdsv->vdpa_aux->padev->vf_pdev; + vdpa_aux = pdsv->vdpa_aux; + if (!vdpa_aux->nintrs) return; @@ -612,6 +618,7 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, struct device *dma_dev; struct pci_dev *pdev; struct device *dev; + u8 status; int err; int i; @@ -638,6 +645,13 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, dma_dev = &pdev->dev; pdsv->vdpa_dev.dma_dev = dma_dev; + status = pds_vdpa_get_status(&pdsv->vdpa_dev); + if (status == 0xff) { + dev_err(dev, "Broken PCI - status %#x\n", status); + err = -ENXIO; + goto err_unmap; + } + pdsv->supported_features = mgmt->supported_features; if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h index d984ba24a7da..84bdb45871ff 100644 --- a/drivers/vdpa/pds/vdpa_dev.h +++ b/drivers/vdpa/pds/vdpa_dev.h @@ -46,5 +46,6 @@ struct pds_vdpa_device { #define PDS_VDPA_PACKED_INVERT_IDX 0x8000 +void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv); int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux); #endif /* _VDPA_DEV_H_ */ From 1496c47065f9f8413296780c63374daee9bdae20 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:38:56 +0800 Subject: [PATCH 09/35] vhost-vdpa: uapi to support reporting per vq size The size of a virtqueue is a per vq configuration. This commit introduce a new ioctl uAPI to support this flexibility. Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-2-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- include/uapi/linux/vhost.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h index 649560c685f1..bea697390613 100644 --- a/include/uapi/linux/vhost.h +++ b/include/uapi/linux/vhost.h @@ -227,4 +227,11 @@ */ #define VHOST_VDPA_GET_VRING_DESC_GROUP _IOWR(VHOST_VIRTIO, 0x7F, \ struct vhost_vring_state) + +/* Get the queue size of a specific virtqueue. + * userspace set the vring index in vhost_vring_state.index + * kernel set the queue size in vhost_vring_state.num + */ +#define VHOST_VDPA_GET_VRING_SIZE _IOWR(VHOST_VIRTIO, 0x80, \ + struct vhost_vring_state) #endif From 0a926fc972532788719fd03c4a44724ec23c1875 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:38:57 +0800 Subject: [PATCH 10/35] vDPA: introduce get_vq_size to vdpa_config_ops This commit introduces a new interface get_vq_size to vDPA config ops, this new interface intends to report the size of a specific virtqueue Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-3-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vdpa.c | 8 ++++++++ include/linux/vdpa.h | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index aef92a7c57f3..ba52d128aeb7 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -687,6 +687,14 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, if (!ops->set_group_asid) return -EOPNOTSUPP; return ops->set_group_asid(vdpa, idx, s.num); + case VHOST_VDPA_GET_VRING_SIZE: + if (!ops->get_vq_size) + return -EOPNOTSUPP; + s.index = idx; + s.num = ops->get_vq_size(vdpa, idx); + if (copy_to_user(argp, &s, sizeof(s))) + return -EFAULT; + return 0; case VHOST_GET_VRING_BASE: r = ops->get_vq_state(v->vdpa, idx, &vq_state); if (r) diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index db15ac07f8a6..4097e8e92860 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -195,6 +195,10 @@ struct vdpa_map_file { * @idx: virtqueue index * Returns int: irq number of a virtqueue, * negative number if no irq assigned. + * @get_vq_size: Get the size of a specific virtqueue (optional) + * @vdev: vdpa device + * @idx: virtqueue index + * Return u16: the size of the virtqueue * @get_vq_align: Get the virtqueue align requirement * for the device * @vdev: vdpa device @@ -386,6 +390,7 @@ struct vdpa_config_ops { (*get_vq_notification)(struct vdpa_device *vdev, u16 idx); /* vq irq is not expected to be changed once DRIVER_OK is set */ int (*get_vq_irq)(struct vdpa_device *vdev, u16 idx); + u16 (*get_vq_size)(struct vdpa_device *vdev, u16 idx); /* Device ops */ u32 (*get_vq_align)(struct vdpa_device *vdev); From 36503e5e06a986ae9dbf042589d0ef6e4cbfe15a Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:38:58 +0800 Subject: [PATCH 11/35] vDPA/ifcvf: implement vdpa_config_ops.get_vq_size This commit implements vdpa_ops.get_vq_size to report the size of a specific virtqueue. Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-4-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/ifcvf/ifcvf_base.c | 5 ++++- drivers/vdpa/ifcvf/ifcvf_base.h | 1 + drivers/vdpa/ifcvf/ifcvf_main.c | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c index 060f837a4f9f..d78621fdd621 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.c +++ b/drivers/vdpa/ifcvf/ifcvf_base.c @@ -69,10 +69,13 @@ static int ifcvf_read_config_range(struct pci_dev *dev, return 0; } -static u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid) +u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid) { u16 queue_size; + if (qid >= hw->nr_vring) + return 0; + vp_iowrite16(qid, &hw->common_cfg->queue_select); queue_size = vp_ioread16(&hw->common_cfg->queue_size); diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h index b57849c643f6..5a4be3a4f939 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.h +++ b/drivers/vdpa/ifcvf/ifcvf_base.h @@ -131,4 +131,5 @@ void ifcvf_set_vq_ready(struct ifcvf_hw *hw, u16 qid, bool ready); void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features); u64 ifcvf_get_driver_features(struct ifcvf_hw *hw); u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw); +u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid); #endif /* _IFCVF_H_ */ diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index e98fa8100f3c..cd4e9cb020c4 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -597,6 +597,14 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, return -EINVAL; } +static u16 ifcvf_vdpa_get_vq_size(struct vdpa_device *vdpa_dev, + u16 qid) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + return ifcvf_get_vq_size(vf, qid); +} + static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev, u16 idx) { @@ -632,6 +640,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = { .set_vq_num = ifcvf_vdpa_set_vq_num, .set_vq_address = ifcvf_vdpa_set_vq_address, .get_vq_irq = ifcvf_vdpa_get_vq_irq, + .get_vq_size = ifcvf_vdpa_get_vq_size, .kick_vq = ifcvf_vdpa_kick_vq, .get_generation = ifcvf_vdpa_get_generation, .get_device_id = ifcvf_vdpa_get_device_id, From a97f9c8fcc6ed16f956346368fde3b1198163f6c Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:38:59 +0800 Subject: [PATCH 12/35] vp_vdpa: implement vdpa_config_ops.get_vq_size This commit implements vdpa_config_ops.get_vq_size in vp_vdpa, which reports per virtqueue size. Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-5-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/virtio_pci/vp_vdpa.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c index 281287fae89f..df5f4a3bccb5 100644 --- a/drivers/vdpa/virtio_pci/vp_vdpa.c +++ b/drivers/vdpa/virtio_pci/vp_vdpa.c @@ -328,6 +328,13 @@ static void vp_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 qid, vp_modern_set_queue_size(mdev, qid, num); } +static u16 vp_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 qid) +{ + struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); + + return vp_modern_get_queue_size(mdev, qid); +} + static int vp_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 qid, u64 desc_area, u64 driver_area, u64 device_area) @@ -449,6 +456,7 @@ static const struct vdpa_config_ops vp_vdpa_ops = { .set_vq_ready = vp_vdpa_set_vq_ready, .get_vq_ready = vp_vdpa_get_vq_ready, .set_vq_num = vp_vdpa_set_vq_num, + .get_vq_size = vp_vdpa_get_vq_size, .set_vq_address = vp_vdpa_set_vq_address, .kick_vq = vp_vdpa_kick_vq, .get_generation = vp_vdpa_get_generation, From 1da13e647012a8034420915bf6fcb9db37d41fc3 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:39:00 +0800 Subject: [PATCH 13/35] eni_vdpa: implement vdpa_config_ops.get_vq_size This commit implements get_vq_size which report per vq size in vdpa_config_ops Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-6-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/alibaba/eni_vdpa.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/vdpa/alibaba/eni_vdpa.c b/drivers/vdpa/alibaba/eni_vdpa.c index cce3d1837104..ad7f3447fe90 100644 --- a/drivers/vdpa/alibaba/eni_vdpa.c +++ b/drivers/vdpa/alibaba/eni_vdpa.c @@ -254,6 +254,13 @@ static u16 eni_vdpa_get_vq_num_min(struct vdpa_device *vdpa) return vp_legacy_get_queue_size(ldev, 0); } +static u16 eni_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 qid) +{ + struct virtio_pci_legacy_device *ldev = vdpa_to_ldev(vdpa); + + return vp_legacy_get_queue_size(ldev, qid); +} + static int eni_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 qid, struct vdpa_vq_state *state) { @@ -416,6 +423,7 @@ static const struct vdpa_config_ops eni_vdpa_ops = { .reset = eni_vdpa_reset, .get_vq_num_max = eni_vdpa_get_vq_num_max, .get_vq_num_min = eni_vdpa_get_vq_num_min, + .get_vq_size = eni_vdpa_get_vq_size, .get_vq_state = eni_vdpa_get_vq_state, .set_vq_state = eni_vdpa_set_vq_state, .set_vq_cb = eni_vdpa_set_vq_cb, From f6fa2f7ea07db70d883a9f8944e6119c10934c5f Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:39:01 +0800 Subject: [PATCH 14/35] vdpa_sim: implement vdpa_config_ops.get_vq_size for vDPA simulator This commit implements vdpa_config_ops.get_vq_size for vDPA simulator, this new interface can help report per vq size. Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-7-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_sim/vdpa_sim.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index 18584ce70bf0..8ffea8430f95 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -311,6 +311,17 @@ static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) vq->num = num; } +static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + + if (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK) + return vq->num; + else + return VDPASIM_QUEUE_MAX; +} + static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx) { struct vdpasim *vdpasim = vdpa_to_sim(vdpa); @@ -775,6 +786,7 @@ static const struct vdpa_config_ops vdpasim_config_ops = { .get_driver_features = vdpasim_get_driver_features, .set_config_cb = vdpasim_set_config_cb, .get_vq_num_max = vdpasim_get_vq_num_max, + .get_vq_size = vdpasim_get_vq_size, .get_device_id = vdpasim_get_device_id, .get_vendor_id = vdpasim_get_vendor_id, .get_status = vdpasim_get_status, From 47e62e6d62fcae0a00b9f00228c458661595b380 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:39:02 +0800 Subject: [PATCH 15/35] vduse: implement vdpa_config_ops.get_vq_size for vduse This commit implements get_vq_size for vdpa_config_ops. This new interface is used to report per vq size. Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-8-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_user/vduse_dev.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index 75354ce186a1..73c89701fc9d 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -541,6 +541,17 @@ static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) vq->num = num; } +static u16 vduse_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 idx) +{ + struct vduse_dev *dev = vdpa_to_vduse(vdpa); + struct vduse_virtqueue *vq = dev->vqs[idx]; + + if (vq->num) + return vq->num; + else + return vq->num_max; +} + static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready) { @@ -773,6 +784,7 @@ static const struct vdpa_config_ops vduse_vdpa_config_ops = { .kick_vq = vduse_vdpa_kick_vq, .set_vq_cb = vduse_vdpa_set_vq_cb, .set_vq_num = vduse_vdpa_set_vq_num, + .get_vq_size = vduse_vdpa_get_vq_size, .set_vq_ready = vduse_vdpa_set_vq_ready, .get_vq_ready = vduse_vdpa_get_vq_ready, .set_vq_state = vduse_vdpa_set_vq_state, From 273ae08fa03f907d09d6fe815a3982a7298ba804 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:39:03 +0800 Subject: [PATCH 16/35] virtio_vdpa: create vqs with the actual size The size of a virtqueue is a per vq configuration, this commit allows virtio_vdpa to create virtqueues with the actual size of a specific vq size that supported by the backend device. Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-9-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_vdpa.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 8d63e5923d24..e803db0da307 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -183,8 +183,11 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, info = kmalloc(sizeof(*info), GFP_KERNEL); if (!info) return ERR_PTR(-ENOMEM); + if (ops->get_vq_size) + max_num = ops->get_vq_size(vdpa, index); + else + max_num = ops->get_vq_num_max(vdpa); - max_num = ops->get_vq_num_max(vdpa); if (max_num == 0) { err = -ENOENT; goto error_new_virtqueue; From cd2147067409f0945ad7668cf74c347d3b426057 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:39:04 +0800 Subject: [PATCH 17/35] vDPA/ifcvf: get_max_vq_size to return max size Since we already implemented vdpa_config_ops.get_vq_size, so get_max_vq_size can return the acutal max size of the virtqueues other than the max allowed safe size. Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-10-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/ifcvf/ifcvf_base.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c index d78621fdd621..472daa588a9d 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.c +++ b/drivers/vdpa/ifcvf/ifcvf_base.c @@ -82,10 +82,6 @@ u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid) return queue_size; } -/* This function returns the max allowed safe size for - * all virtqueues. It is the minimal size that can be - * suppprted by all virtqueues. - */ u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw) { u16 queue_size, max_size, qid; @@ -97,7 +93,7 @@ u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw) if (!queue_size) continue; - max_size = min(queue_size, max_size); + max_size = max(queue_size, max_size); } return max_size; From 56d61ae558bf7b052c6149c06a902dba0ba33fb1 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:39:05 +0800 Subject: [PATCH 18/35] vDPA/ifcvf: implement vdpa_config_ops.get_vq_num_min IFCVF HW supports operation with vq size less than the max size, as the spec required. This commit implements vdpa_config_ops.get_vq_num_min to report the minimal size of the virtqueues, which gives vDPA framework a chance to reduce the vring size. We need at least one descriptor to be functional, but it is better no less than 64 to meet ceratin performance requirements. Actually the framework would allocate at least a PAGE for the vq. Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-11-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/ifcvf/ifcvf_base.h | 1 + drivers/vdpa/ifcvf/ifcvf_main.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h index 5a4be3a4f939..0f347717021a 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.h +++ b/drivers/vdpa/ifcvf/ifcvf_base.h @@ -28,6 +28,7 @@ #define IFCVF_PCI_MAX_RESOURCE 6 #define IFCVF_LM_BAR 4 +#define IFCVF_MIN_VQ_SIZE 64 #define IFCVF_ERR(pdev, fmt, ...) dev_err(&pdev->dev, fmt, ##__VA_ARGS__) #define IFCVF_DBG(pdev, fmt, ...) dev_dbg(&pdev->dev, fmt, ##__VA_ARGS__) diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index cd4e9cb020c4..80d0a0460885 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -456,6 +456,11 @@ static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev) return ifcvf_get_max_vq_size(vf); } +static u16 ifcvf_vdpa_get_vq_num_min(struct vdpa_device *vdpa_dev) +{ + return IFCVF_MIN_VQ_SIZE; +} + static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid, struct vdpa_vq_state *state) { @@ -632,6 +637,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = { .set_status = ifcvf_vdpa_set_status, .reset = ifcvf_vdpa_reset, .get_vq_num_max = ifcvf_vdpa_get_vq_num_max, + .get_vq_num_min = ifcvf_vdpa_get_vq_num_min, .get_vq_state = ifcvf_vdpa_get_vq_state, .set_vq_state = ifcvf_vdpa_set_vq_state, .set_vq_cb = ifcvf_vdpa_set_vq_cb, From 8169ed622060035ba6b5b8c10388dfc9a31f1c98 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sun, 4 Feb 2024 17:50:45 -0300 Subject: [PATCH 19/35] vdpa: make vdpa_bus const Now that the driver core can properly handle constant struct bus_type, move the vdpa_bus variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Message-Id: <20240204-bus_cleanup-vdpa-v1-1-1745eccb0a5c@marliere.net> Signed-off-by: Michael S. Tsirkin Reviewed-by: Greg Kroah-Hartman --- drivers/vdpa/vdpa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index d0695680b282..c73c06102e7e 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -115,7 +115,7 @@ static const struct attribute_group vdpa_dev_group = { }; __ATTRIBUTE_GROUPS(vdpa_dev); -static struct bus_type vdpa_bus = { +static const struct bus_type vdpa_bus = { .name = "vdpa", .dev_groups = vdpa_dev_groups, .match = vdpa_dev_match, From 2b666ee29618f452b84aeacf08e1267894de7234 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sun, 4 Feb 2024 17:52:51 -0300 Subject: [PATCH 20/35] virtio: make virtio_bus const Now that the driver core can properly handle constant struct bus_type, move the virtio_bus variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Message-Id: <20240204-bus_cleanup-virtio-v1-1-3bcb2212aaa0@marliere.net> Signed-off-by: Michael S. Tsirkin Reviewed-by: Greg Kroah-Hartman Acked-by: Jason Wang --- drivers/virtio/virtio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index f513ee21b1c1..f173587893cb 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -353,7 +353,7 @@ static void virtio_dev_remove(struct device *_d) of_node_put(dev->dev.of_node); } -static struct bus_type virtio_bus = { +static const struct bus_type virtio_bus = { .name = "virtio", .match = virtio_dev_match, .dev_groups = virtio_dev_groups, From c2475a9a789721bfdcc1b16aaf61ccfecb891914 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:55:57 +0800 Subject: [PATCH 21/35] vDPA: report virtio-block capacity to user space This commit allows userspace to query capacity of a virtio-block device. Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-2-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 35 +++++++++++++++++++++++++++++++++++ include/linux/vdpa.h | 1 + include/uapi/linux/vdpa.h | 2 ++ 3 files changed, 38 insertions(+) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index c73c06102e7e..613bec31f59e 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -944,6 +944,38 @@ static int vdpa_dev_net_config_fill(struct vdpa_device *vdev, struct sk_buff *ms return vdpa_dev_net_mq_config_fill(msg, features_device, &config); } +static int +vdpa_dev_blk_capacity_config_fill(struct sk_buff *msg, + const struct virtio_blk_config *config) +{ + u64 val_u64; + + val_u64 = __virtio64_to_cpu(true, config->capacity); + + return nla_put_u64_64bit(msg, VDPA_ATTR_DEV_BLK_CFG_CAPACITY, + val_u64, VDPA_ATTR_PAD); +} + +static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, + struct sk_buff *msg) +{ + struct virtio_blk_config config = {}; + u64 features_device; + + vdev->config->get_config(vdev, 0, &config, sizeof(config)); + + features_device = vdev->config->get_device_features(vdev); + + if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_FEATURES, features_device, + VDPA_ATTR_PAD)) + return -EMSGSIZE; + + if (vdpa_dev_blk_capacity_config_fill(msg, &config)) + return -EMSGSIZE; + + return 0; +} + static int vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq, int flags, struct netlink_ext_ack *extack) @@ -988,6 +1020,9 @@ vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, case VIRTIO_ID_NET: err = vdpa_dev_net_config_fill(vdev, msg); break; + case VIRTIO_ID_BLOCK: + err = vdpa_dev_blk_config_fill(vdev, msg); + break; default: err = -EOPNOTSUPP; break; diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 4097e8e92860..7977ca03ac7a 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -7,6 +7,7 @@ #include #include #include +#include #include /** diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 54b649ab0f22..1bf69226cb96 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -56,6 +56,8 @@ enum vdpa_attr { /* virtio features that are provisioned to the vDPA device */ VDPA_ATTR_DEV_FEATURES, /* u64 */ + VDPA_ATTR_DEV_BLK_CFG_CAPACITY, /* u64 */ + /* new attributes must be added above here */ VDPA_ATTR_MAX, }; From 330b8aea692410fbe8f0d058e01aea2986613ff7 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:55:58 +0800 Subject: [PATCH 22/35] vDPA: report virtio-block max segment size to user space This commit allows reporting the max size of any single segment of virtio-block devices to user space. Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-3-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 17 +++++++++++++++++ include/uapi/linux/vdpa.h | 1 + 2 files changed, 18 insertions(+) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 613bec31f59e..db355d4683ba 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -956,6 +956,20 @@ vdpa_dev_blk_capacity_config_fill(struct sk_buff *msg, val_u64, VDPA_ATTR_PAD); } +static int +vdpa_dev_blk_seg_size_config_fill(struct sk_buff *msg, u64 features, + const struct virtio_blk_config *config) +{ + u32 val_u32; + + if ((features & BIT_ULL(VIRTIO_BLK_F_SIZE_MAX)) == 0) + return 0; + + val_u32 = __virtio32_to_cpu(true, config->size_max); + + return nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_SEG_SIZE, val_u32); +} + static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) { @@ -973,6 +987,9 @@ static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, if (vdpa_dev_blk_capacity_config_fill(msg, &config)) return -EMSGSIZE; + if (vdpa_dev_blk_seg_size_config_fill(msg, features_device, &config)) + return -EMSGSIZE; + return 0; } diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 1bf69226cb96..586bce3c906a 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -57,6 +57,7 @@ enum vdpa_attr { VDPA_ATTR_DEV_FEATURES, /* u64 */ VDPA_ATTR_DEV_BLK_CFG_CAPACITY, /* u64 */ + VDPA_ATTR_DEV_BLK_CFG_SEG_SIZE, /* u32 */ /* new attributes must be added above here */ VDPA_ATTR_MAX, From 3a1d33fb7e5f96471aa7bfe97dd09afc2aa03b39 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:55:59 +0800 Subject: [PATCH 23/35] vDPA: report virtio-block block-size to user space This commit allows reporting the block size of a virtio-block device to user space. Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-4-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 18 ++++++++++++++++++ include/uapi/linux/vdpa.h | 1 + 2 files changed, 19 insertions(+) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index db355d4683ba..d96dc2ad48aa 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -970,6 +970,21 @@ vdpa_dev_blk_seg_size_config_fill(struct sk_buff *msg, u64 features, return nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_SEG_SIZE, val_u32); } +/* fill the block size*/ +static int +vdpa_dev_blk_block_size_config_fill(struct sk_buff *msg, u64 features, + const struct virtio_blk_config *config) +{ + u32 val_u32; + + if ((features & BIT_ULL(VIRTIO_BLK_F_BLK_SIZE)) == 0) + return 0; + + val_u32 = __virtio32_to_cpu(true, config->blk_size); + + return nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_BLK_SIZE, val_u32); +} + static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) { @@ -990,6 +1005,9 @@ static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, if (vdpa_dev_blk_seg_size_config_fill(msg, features_device, &config)) return -EMSGSIZE; + if (vdpa_dev_blk_block_size_config_fill(msg, features_device, &config)) + return -EMSGSIZE; + return 0; } diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 586bce3c906a..f3baf7ea7d6d 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -58,6 +58,7 @@ enum vdpa_attr { VDPA_ATTR_DEV_BLK_CFG_CAPACITY, /* u64 */ VDPA_ATTR_DEV_BLK_CFG_SEG_SIZE, /* u32 */ + VDPA_ATTR_DEV_BLK_CFG_BLK_SIZE, /* u32 */ /* new attributes must be added above here */ VDPA_ATTR_MAX, From 81f64e1d911550832b8065631c28c87c96ffca36 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:56:00 +0800 Subject: [PATCH 24/35] vDPA: report virtio-block max segments in a request to user space This commit allows vDPA reporting the maximum number of segments in a request of virtio-block devices to user space. Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-5-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 17 +++++++++++++++++ include/uapi/linux/vdpa.h | 1 + 2 files changed, 18 insertions(+) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index d96dc2ad48aa..3e3c84bd0337 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -985,6 +985,20 @@ vdpa_dev_blk_block_size_config_fill(struct sk_buff *msg, u64 features, return nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_BLK_SIZE, val_u32); } +static int +vdpa_dev_blk_seg_max_config_fill(struct sk_buff *msg, u64 features, + const struct virtio_blk_config *config) +{ + u32 val_u32; + + if ((features & BIT_ULL(VIRTIO_BLK_F_SEG_MAX)) == 0) + return 0; + + val_u32 = __virtio32_to_cpu(true, config->seg_max); + + return nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_SEG_MAX, val_u32); +} + static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) { @@ -1008,6 +1022,9 @@ static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, if (vdpa_dev_blk_block_size_config_fill(msg, features_device, &config)) return -EMSGSIZE; + if (vdpa_dev_blk_seg_max_config_fill(msg, features_device, &config)) + return -EMSGSIZE; + return 0; } diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index f3baf7ea7d6d..2dc92fca2052 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -59,6 +59,7 @@ enum vdpa_attr { VDPA_ATTR_DEV_BLK_CFG_CAPACITY, /* u64 */ VDPA_ATTR_DEV_BLK_CFG_SEG_SIZE, /* u32 */ VDPA_ATTR_DEV_BLK_CFG_BLK_SIZE, /* u32 */ + VDPA_ATTR_DEV_BLK_CFG_SEG_MAX, /* u32 */ /* new attributes must be added above here */ VDPA_ATTR_MAX, From 54fb04b02efd8f56e388443d5f9e03e68eef48e6 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:56:01 +0800 Subject: [PATCH 25/35] vDPA: report virtio-block MQ info to user space This commits allows vDPA reporting virtio-block multi-queue configuration to user sapce. Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-6-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 16 ++++++++++++++++ include/uapi/linux/vdpa.h | 1 + 2 files changed, 17 insertions(+) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 3e3c84bd0337..4c348da1f2f7 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -999,6 +999,19 @@ vdpa_dev_blk_seg_max_config_fill(struct sk_buff *msg, u64 features, return nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_SEG_MAX, val_u32); } +static int vdpa_dev_blk_mq_config_fill(struct sk_buff *msg, u64 features, + const struct virtio_blk_config *config) +{ + u16 val_u16; + + if ((features & BIT_ULL(VIRTIO_BLK_F_MQ)) == 0) + return 0; + + val_u16 = __virtio16_to_cpu(true, config->num_queues); + + return nla_put_u16(msg, VDPA_ATTR_DEV_BLK_CFG_NUM_QUEUES, val_u16); +} + static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) { @@ -1025,6 +1038,9 @@ static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, if (vdpa_dev_blk_seg_max_config_fill(msg, features_device, &config)) return -EMSGSIZE; + if (vdpa_dev_blk_mq_config_fill(msg, features_device, &config)) + return -EMSGSIZE; + return 0; } diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 2dc92fca2052..3c8e3c87a864 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -60,6 +60,7 @@ enum vdpa_attr { VDPA_ATTR_DEV_BLK_CFG_SEG_SIZE, /* u32 */ VDPA_ATTR_DEV_BLK_CFG_BLK_SIZE, /* u32 */ VDPA_ATTR_DEV_BLK_CFG_SEG_MAX, /* u32 */ + VDPA_ATTR_DEV_BLK_CFG_NUM_QUEUES, /* u16 */ /* new attributes must be added above here */ VDPA_ATTR_MAX, From c9d989b4ababe3ec7291b204f365eaadec0afa26 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:56:02 +0800 Subject: [PATCH 26/35] vDPA: report virtio-block topology info to user space This commit allows vDPA reporting topology information of virtio-blk devices to user space, includes: 1) the number of logical blocks per physical block 2) offset of first aligned logical block 3) suggested minimum I/O size in blocks 4) optimal (suggested maximum) I/O size in blocks Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-7-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 32 ++++++++++++++++++++++++++++++++ include/uapi/linux/vdpa.h | 4 ++++ 2 files changed, 36 insertions(+) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 4c348da1f2f7..76b2e31d6229 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -1012,6 +1012,35 @@ static int vdpa_dev_blk_mq_config_fill(struct sk_buff *msg, u64 features, return nla_put_u16(msg, VDPA_ATTR_DEV_BLK_CFG_NUM_QUEUES, val_u16); } +static int vdpa_dev_blk_topology_config_fill(struct sk_buff *msg, u64 features, + const struct virtio_blk_config *config) +{ + u16 min_io_size; + u32 opt_io_size; + + if ((features & BIT_ULL(VIRTIO_BLK_F_TOPOLOGY)) == 0) + return 0; + + min_io_size = __virtio16_to_cpu(true, config->min_io_size); + opt_io_size = __virtio32_to_cpu(true, config->opt_io_size); + + if (nla_put_u8(msg, VDPA_ATTR_DEV_BLK_CFG_PHY_BLK_EXP, + config->physical_block_exp)) + return -EMSGSIZE; + + if (nla_put_u8(msg, VDPA_ATTR_DEV_BLK_CFG_ALIGN_OFFSET, + config->alignment_offset)) + return -EMSGSIZE; + + if (nla_put_u16(msg, VDPA_ATTR_DEV_BLK_CFG_MIN_IO_SIZE, min_io_size)) + return -EMSGSIZE; + + if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_OPT_IO_SIZE, opt_io_size)) + return -EMSGSIZE; + + return 0; +} + static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) { @@ -1041,6 +1070,9 @@ static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, if (vdpa_dev_blk_mq_config_fill(msg, features_device, &config)) return -EMSGSIZE; + if (vdpa_dev_blk_topology_config_fill(msg, features_device, &config)) + return -EMSGSIZE; + return 0; } diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 3c8e3c87a864..5cc70614c97a 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -61,6 +61,10 @@ enum vdpa_attr { VDPA_ATTR_DEV_BLK_CFG_BLK_SIZE, /* u32 */ VDPA_ATTR_DEV_BLK_CFG_SEG_MAX, /* u32 */ VDPA_ATTR_DEV_BLK_CFG_NUM_QUEUES, /* u16 */ + VDPA_ATTR_DEV_BLK_CFG_PHY_BLK_EXP, /* u8 */ + VDPA_ATTR_DEV_BLK_CFG_ALIGN_OFFSET, /* u8 */ + VDPA_ATTR_DEV_BLK_CFG_MIN_IO_SIZE, /* u16 */ + VDPA_ATTR_DEV_BLK_CFG_OPT_IO_SIZE, /* u32 */ /* new attributes must be added above here */ VDPA_ATTR_MAX, From 65848f46e1d6c01fbd0d79a1c051d978a6e6002e Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:56:03 +0800 Subject: [PATCH 27/35] vDPA: report virtio-block discarding configuration to user space This commit reports virtio-blk discarding configuration to user space,includes: 1) the maximum discard sectors 2) maximum number of discard segments for the block driver to use 3) the alignment for splitting a discarding request Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-8-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 26 ++++++++++++++++++++++++++ include/uapi/linux/vdpa.h | 3 +++ 2 files changed, 29 insertions(+) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 76b2e31d6229..9348818b8bf9 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -1041,6 +1041,29 @@ static int vdpa_dev_blk_topology_config_fill(struct sk_buff *msg, u64 features, return 0; } +static int vdpa_dev_blk_discard_config_fill(struct sk_buff *msg, u64 features, + const struct virtio_blk_config *config) +{ + u32 val_u32; + + if ((features & BIT_ULL(VIRTIO_BLK_F_DISCARD)) == 0) + return 0; + + val_u32 = __virtio32_to_cpu(true, config->max_discard_sectors); + if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEC, val_u32)) + return -EMSGSIZE; + + val_u32 = __virtio32_to_cpu(true, config->max_discard_seg); + if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEG, val_u32)) + return -EMSGSIZE; + + val_u32 = __virtio32_to_cpu(true, config->discard_sector_alignment); + if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_DISCARD_SEC_ALIGN, val_u32)) + return -EMSGSIZE; + + return 0; +} + static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) { @@ -1073,6 +1096,9 @@ static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, if (vdpa_dev_blk_topology_config_fill(msg, features_device, &config)) return -EMSGSIZE; + if (vdpa_dev_blk_discard_config_fill(msg, features_device, &config)) + return -EMSGSIZE; + return 0; } diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 5cc70614c97a..30887931a220 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -65,6 +65,9 @@ enum vdpa_attr { VDPA_ATTR_DEV_BLK_CFG_ALIGN_OFFSET, /* u8 */ VDPA_ATTR_DEV_BLK_CFG_MIN_IO_SIZE, /* u16 */ VDPA_ATTR_DEV_BLK_CFG_OPT_IO_SIZE, /* u32 */ + VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEC, /* u32 */ + VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEG, /* u32 */ + VDPA_ATTR_DEV_BLK_CFG_DISCARD_SEC_ALIGN,/* u32 */ /* new attributes must be added above here */ VDPA_ATTR_MAX, From 6bdc7846e627ec21fc79a6d2554f69546a7b4985 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:56:04 +0800 Subject: [PATCH 28/35] vDPA: report virtio-block write zeroes configuration to user space This commits reports write zeroes configuration of virtio-block devices to user space, includes: 1)maximum write zeroes sectors size 2)maximum write zeroes segment number Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-9-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 23 +++++++++++++++++++++++ include/uapi/linux/vdpa.h | 2 ++ 2 files changed, 25 insertions(+) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 9348818b8bf9..bcea9f880042 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -1064,6 +1064,26 @@ static int vdpa_dev_blk_discard_config_fill(struct sk_buff *msg, u64 features, return 0; } +static int +vdpa_dev_blk_write_zeroes_config_fill(struct sk_buff *msg, u64 features, + const struct virtio_blk_config *config) +{ + u32 val_u32; + + if ((features & BIT_ULL(VIRTIO_BLK_F_WRITE_ZEROES)) == 0) + return 0; + + val_u32 = __virtio32_to_cpu(true, config->max_write_zeroes_sectors); + if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEC, val_u32)) + return -EMSGSIZE; + + val_u32 = __virtio32_to_cpu(true, config->max_write_zeroes_seg); + if (nla_put_u32(msg, VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEG, val_u32)) + return -EMSGSIZE; + + return 0; +} + static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) { @@ -1099,6 +1119,9 @@ static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, if (vdpa_dev_blk_discard_config_fill(msg, features_device, &config)) return -EMSGSIZE; + if (vdpa_dev_blk_write_zeroes_config_fill(msg, features_device, &config)) + return -EMSGSIZE; + return 0; } diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 30887931a220..797d5708492f 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -68,6 +68,8 @@ enum vdpa_attr { VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEC, /* u32 */ VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEG, /* u32 */ VDPA_ATTR_DEV_BLK_CFG_DISCARD_SEC_ALIGN,/* u32 */ + VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEC, /* u32 */ + VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEG, /* u32 */ /* new attributes must be added above here */ VDPA_ATTR_MAX, From ae1374b7f72c973631ce881a654664fd2b40c4b9 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:56:05 +0800 Subject: [PATCH 29/35] vDPA: report virtio-block read-only info to user space This commit report read-only information of virtio-blk devices to user space. Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-10-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 14 ++++++++++++++ include/uapi/linux/vdpa.h | 1 + 2 files changed, 15 insertions(+) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index bcea9f880042..2d02ea60af0c 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -1084,6 +1084,17 @@ vdpa_dev_blk_write_zeroes_config_fill(struct sk_buff *msg, u64 features, return 0; } +static int vdpa_dev_blk_ro_config_fill(struct sk_buff *msg, u64 features) +{ + u8 ro; + + ro = ((features & BIT_ULL(VIRTIO_BLK_F_RO)) == 0) ? 0 : 1; + if (nla_put_u8(msg, VDPA_ATTR_DEV_BLK_CFG_READ_ONLY, ro)) + return -EMSGSIZE; + + return 0; +} + static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) { @@ -1122,6 +1133,9 @@ static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, if (vdpa_dev_blk_write_zeroes_config_fill(msg, features_device, &config)) return -EMSGSIZE; + if (vdpa_dev_blk_ro_config_fill(msg, features_device)) + return -EMSGSIZE; + return 0; } diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 797d5708492f..4be8e3a15874 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -70,6 +70,7 @@ enum vdpa_attr { VDPA_ATTR_DEV_BLK_CFG_DISCARD_SEC_ALIGN,/* u32 */ VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEC, /* u32 */ VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEG, /* u32 */ + VDPA_ATTR_DEV_BLK_CFG_READ_ONLY, /* u8 */ /* new attributes must be added above here */ VDPA_ATTR_MAX, From 1ac61ddfee93278f87370f6c43ed669b845f9037 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:56:06 +0800 Subject: [PATCH 30/35] vDPA: report virtio-blk flush info to user space This commit reports whether a virtio-blk device support cache flush command to user space Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-11-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 14 ++++++++++++++ include/uapi/linux/vdpa.h | 1 + 2 files changed, 15 insertions(+) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 2d02ea60af0c..b246067e074b 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -1095,6 +1095,17 @@ static int vdpa_dev_blk_ro_config_fill(struct sk_buff *msg, u64 features) return 0; } +static int vdpa_dev_blk_flush_config_fill(struct sk_buff *msg, u64 features) +{ + u8 flush; + + flush = ((features & BIT_ULL(VIRTIO_BLK_F_FLUSH)) == 0) ? 0 : 1; + if (nla_put_u8(msg, VDPA_ATTR_DEV_BLK_CFG_FLUSH, flush)) + return -EMSGSIZE; + + return 0; +} + static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) { @@ -1136,6 +1147,9 @@ static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, if (vdpa_dev_blk_ro_config_fill(msg, features_device)) return -EMSGSIZE; + if (vdpa_dev_blk_flush_config_fill(msg, features_device)) + return -EMSGSIZE; + return 0; } diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 4be8e3a15874..43c51698195c 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -71,6 +71,7 @@ enum vdpa_attr { VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEC, /* u32 */ VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEG, /* u32 */ VDPA_ATTR_DEV_BLK_CFG_READ_ONLY, /* u8 */ + VDPA_ATTR_DEV_BLK_CFG_FLUSH, /* u8 */ /* new attributes must be added above here */ VDPA_ATTR_MAX, From d5c0ed17fea60cca9bc3bf1278b49ba79242bbcd Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Fri, 23 Feb 2024 15:18:33 +0800 Subject: [PATCH 31/35] virtio: packed: fix unmap leak for indirect desc table When use_dma_api and premapped are true, then the do_unmap is false. Because the do_unmap is false, vring_unmap_extra_packed is not called by detach_buf_packed. if (unlikely(vq->do_unmap)) { curr = id; for (i = 0; i < state->num; i++) { vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]); curr = vq->packed.desc_extra[curr].next; } } So the indirect desc table is not unmapped. This causes the unmap leak. So here, we check vq->use_dma_api instead. Synchronously, dma info is updated based on use_dma_api judgment This bug does not occur, because no driver use the premapped with indirect. Fixes: b319940f83c2 ("virtio_ring: skip unmap for premapped") Signed-off-by: Xuan Zhuo Message-Id: <20240223071833.26095-1-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 49299b1f9ec7..6f7e5010a673 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1340,7 +1340,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq, sizeof(struct vring_packed_desc)); vq->packed.vring.desc[head].id = cpu_to_le16(id); - if (vq->do_unmap) { + if (vq->use_dma_api) { vq->packed.desc_extra[id].addr = addr; vq->packed.desc_extra[id].len = total_sg * sizeof(struct vring_packed_desc); @@ -1481,7 +1481,7 @@ static inline int virtqueue_add_packed(struct virtqueue *_vq, desc[i].len = cpu_to_le32(sg->length); desc[i].id = cpu_to_le16(id); - if (unlikely(vq->do_unmap)) { + if (unlikely(vq->use_dma_api)) { vq->packed.desc_extra[curr].addr = addr; vq->packed.desc_extra[curr].len = sg->length; vq->packed.desc_extra[curr].flags = @@ -1615,7 +1615,7 @@ static void detach_buf_packed(struct vring_virtqueue *vq, vq->free_head = id; vq->vq.num_free += state->num; - if (unlikely(vq->do_unmap)) { + if (unlikely(vq->use_dma_api)) { curr = id; for (i = 0; i < state->num; i++) { vring_unmap_extra_packed(vq, From b9f7425239a09903629f71f719654983481b00aa Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 20 Jul 2023 04:38:38 -0400 Subject: [PATCH 32/35] virtio-net: convert rx mode setting to use workqueue This patch convert rx mode setting to be done in a workqueue, this is a must for allow to sleep when waiting for the cvq command to response since current code is executed under addr spin lock. Note that we need to disable and flush the workqueue during freeze, this means the rx mode setting is lost after resuming. This is not the bug of this patch as we never try to restore rx mode setting during resume. Signed-off-by: Jason Wang Message-Id: <20230720083839.481487-2-jasowang@redhat.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Shannon Nelson --- drivers/net/virtio_net.c | 55 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index d7ce4a1011ea..46fc86fe62df 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -304,6 +304,12 @@ struct virtnet_info { /* Work struct for config space updates */ struct work_struct config_work; + /* Work struct for setting rx mode */ + struct work_struct rx_mode_work; + + /* OK to queue work setting RX mode? */ + bool rx_mode_work_enabled; + /* Does the affinity hint is set for virtqueues? */ bool affinity_hint_set; @@ -447,6 +453,20 @@ static void disable_delayed_refill(struct virtnet_info *vi) spin_unlock_bh(&vi->refill_lock); } +static void enable_rx_mode_work(struct virtnet_info *vi) +{ + rtnl_lock(); + vi->rx_mode_work_enabled = true; + rtnl_unlock(); +} + +static void disable_rx_mode_work(struct virtnet_info *vi) +{ + rtnl_lock(); + vi->rx_mode_work_enabled = false; + rtnl_unlock(); +} + static void virtqueue_napi_schedule(struct napi_struct *napi, struct virtqueue *vq) { @@ -2706,9 +2726,11 @@ static int virtnet_close(struct net_device *dev) return 0; } -static void virtnet_set_rx_mode(struct net_device *dev) +static void virtnet_rx_mode_work(struct work_struct *work) { - struct virtnet_info *vi = netdev_priv(dev); + struct virtnet_info *vi = + container_of(work, struct virtnet_info, rx_mode_work); + struct net_device *dev = vi->dev; struct scatterlist sg[2]; struct virtio_net_ctrl_mac *mac_data; struct netdev_hw_addr *ha; @@ -2721,6 +2743,8 @@ static void virtnet_set_rx_mode(struct net_device *dev) if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) return; + rtnl_lock(); + vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0); vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0); @@ -2738,14 +2762,19 @@ static void virtnet_set_rx_mode(struct net_device *dev) dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n", vi->ctrl->allmulti ? "en" : "dis"); + netif_addr_lock_bh(dev); + uc_count = netdev_uc_count(dev); mc_count = netdev_mc_count(dev); /* MAC filter - use one buffer for both lists */ buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) + (2 * sizeof(mac_data->entries)), GFP_ATOMIC); mac_data = buf; - if (!buf) + if (!buf) { + netif_addr_unlock_bh(dev); + rtnl_unlock(); return; + } sg_init_table(sg, 2); @@ -2766,6 +2795,8 @@ static void virtnet_set_rx_mode(struct net_device *dev) netdev_for_each_mc_addr(ha, dev) memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); + netif_addr_unlock_bh(dev); + sg_set_buf(&sg[1], mac_data, sizeof(mac_data->entries) + (mc_count * ETH_ALEN)); @@ -2773,9 +2804,19 @@ static void virtnet_set_rx_mode(struct net_device *dev) VIRTIO_NET_CTRL_MAC_TABLE_SET, sg)) dev_warn(&dev->dev, "Failed to set MAC filter table.\n"); + rtnl_unlock(); + kfree(buf); } +static void virtnet_set_rx_mode(struct net_device *dev) +{ + struct virtnet_info *vi = netdev_priv(dev); + + if (vi->rx_mode_work_enabled) + schedule_work(&vi->rx_mode_work); +} + static int virtnet_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) { @@ -3856,6 +3897,8 @@ static void virtnet_freeze_down(struct virtio_device *vdev) /* Make sure no work handler is accessing the device */ flush_work(&vi->config_work); + disable_rx_mode_work(vi); + flush_work(&vi->rx_mode_work); netif_tx_lock_bh(vi->dev); netif_device_detach(vi->dev); @@ -3878,6 +3921,7 @@ static int virtnet_restore_up(struct virtio_device *vdev) virtio_device_ready(vdev); enable_delayed_refill(vi); + enable_rx_mode_work(vi); if (netif_running(vi->dev)) { err = virtnet_open(vi->dev); @@ -4676,6 +4720,7 @@ static int virtnet_probe(struct virtio_device *vdev) vdev->priv = vi; INIT_WORK(&vi->config_work, virtnet_config_changed_work); + INIT_WORK(&vi->rx_mode_work, virtnet_rx_mode_work); spin_lock_init(&vi->refill_lock); if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) { @@ -4798,6 +4843,8 @@ static int virtnet_probe(struct virtio_device *vdev) if (vi->has_rss || vi->has_rss_hash_report) virtnet_init_default_rss(vi); + enable_rx_mode_work(vi); + /* serialize netdev register + virtio_device_ready() with ndo_open() */ rtnl_lock(); @@ -4895,6 +4942,8 @@ static void virtnet_remove(struct virtio_device *vdev) /* Make sure no work handler is accessing the device. */ flush_work(&vi->config_work); + disable_rx_mode_work(vi); + flush_work(&vi->rx_mode_work); unregister_netdev(vi->dev); From 0d197a14716481626f863d2091f4c328bf5412e3 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 20 Jul 2023 04:38:39 -0400 Subject: [PATCH 33/35] virtio-net: add cond_resched() to the command waiting loop Adding cond_resched() to the command waiting loop for a better co-operation with the scheduler. This allows to give CPU a breath to run other task(workqueue) instead of busy looping when preemption is not allowed on a device whose CVQ might be slow. Signed-off-by: Jason Wang Message-Id: <20230720083839.481487-3-jasowang@redhat.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Shannon Nelson --- drivers/net/virtio_net.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 46fc86fe62df..e709d4411810 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2570,8 +2570,10 @@ static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, * into the hypervisor, so the request should be handled immediately. */ while (!virtqueue_get_buf(vi->cvq, &tmp) && - !virtqueue_is_broken(vi->cvq)) + !virtqueue_is_broken(vi->cvq)) { + cond_resched(); cpu_relax(); + } return vi->ctrl->status == VIRTIO_NET_OK; } From b1dc24aba789190dad0047c1f4c16049dd698769 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Thu, 29 Feb 2024 15:20:42 +0800 Subject: [PATCH 34/35] virtio_net: unify the code for recycling the xmit ptr There are two completely similar and independent implementations. This is inconvenient for the subsequent addition of new types. So extract a function from this piece of code and call this function uniformly to recover old xmit ptr. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20240229072044.77388-18-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/net/virtio_net.c | 82 +++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index e709d4411810..bc9217b3286b 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -80,6 +80,11 @@ struct virtnet_stat_desc { size_t offset; }; +struct virtnet_sq_free_stats { + u64 packets; + u64 bytes; +}; + struct virtnet_sq_stats { struct u64_stats_sync syncp; u64_stats_t packets; @@ -372,6 +377,31 @@ static struct xdp_frame *ptr_to_xdp(void *ptr) return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG); } +static void __free_old_xmit(struct send_queue *sq, bool in_napi, + struct virtnet_sq_free_stats *stats) +{ + unsigned int len; + void *ptr; + + while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { + ++stats->packets; + + if (!is_xdp_frame(ptr)) { + struct sk_buff *skb = ptr; + + pr_debug("Sent skb %p\n", skb); + + stats->bytes += skb->len; + napi_consume_skb(skb, in_napi); + } else { + struct xdp_frame *frame = ptr_to_xdp(ptr); + + stats->bytes += xdp_get_frame_len(frame); + xdp_return_frame(frame); + } + } +} + /* Converting between virtqueue no. and kernel tx/rx queue no. * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq */ @@ -798,37 +828,19 @@ static void virtnet_rq_unmap_free_buf(struct virtqueue *vq, void *buf) static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi) { - unsigned int len; - unsigned int packets = 0; - unsigned int bytes = 0; - void *ptr; + struct virtnet_sq_free_stats stats = {0}; - while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { - if (likely(!is_xdp_frame(ptr))) { - struct sk_buff *skb = ptr; - - pr_debug("Sent skb %p\n", skb); - - bytes += skb->len; - napi_consume_skb(skb, in_napi); - } else { - struct xdp_frame *frame = ptr_to_xdp(ptr); - - bytes += xdp_get_frame_len(frame); - xdp_return_frame(frame); - } - packets++; - } + __free_old_xmit(sq, in_napi, &stats); /* Avoid overhead when no packets have been processed * happens when called speculatively from start_xmit. */ - if (!packets) + if (!stats.packets) return; u64_stats_update_begin(&sq->stats.syncp); - u64_stats_add(&sq->stats.bytes, bytes); - u64_stats_add(&sq->stats.packets, packets); + u64_stats_add(&sq->stats.bytes, stats.bytes); + u64_stats_add(&sq->stats.packets, stats.packets); u64_stats_update_end(&sq->stats.syncp); } @@ -967,15 +979,12 @@ static int virtnet_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, u32 flags) { struct virtnet_info *vi = netdev_priv(dev); + struct virtnet_sq_free_stats stats = {0}; struct receive_queue *rq = vi->rq; struct bpf_prog *xdp_prog; struct send_queue *sq; - unsigned int len; - int packets = 0; - int bytes = 0; int nxmit = 0; int kicks = 0; - void *ptr; int ret; int i; @@ -994,20 +1003,7 @@ static int virtnet_xdp_xmit(struct net_device *dev, } /* Free up any pending old buffers before queueing new ones. */ - while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { - if (likely(is_xdp_frame(ptr))) { - struct xdp_frame *frame = ptr_to_xdp(ptr); - - bytes += xdp_get_frame_len(frame); - xdp_return_frame(frame); - } else { - struct sk_buff *skb = ptr; - - bytes += skb->len; - napi_consume_skb(skb, false); - } - packets++; - } + __free_old_xmit(sq, false, &stats); for (i = 0; i < n; i++) { struct xdp_frame *xdpf = frames[i]; @@ -1027,8 +1023,8 @@ static int virtnet_xdp_xmit(struct net_device *dev, } out: u64_stats_update_begin(&sq->stats.syncp); - u64_stats_add(&sq->stats.bytes, bytes); - u64_stats_add(&sq->stats.packets, packets); + u64_stats_add(&sq->stats.bytes, stats.bytes); + u64_stats_add(&sq->stats.packets, stats.packets); u64_stats_add(&sq->stats.xdp_tx, n); u64_stats_add(&sq->stats.xdp_tx_drops, n - nxmit); u64_stats_add(&sq->stats.kicks, kicks); From 5da7137de79ca6ffae3ace77050588cdf5263d33 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Thu, 29 Feb 2024 15:20:43 +0800 Subject: [PATCH 35/35] virtio_net: rename free_old_xmit_skbs to free_old_xmit Since free_old_xmit_skbs not only deals with skb, but also xdp frame and subsequent added xsk, so change the name of this function to free_old_xmit. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20240229072044.77388-19-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/net/virtio_net.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index bc9217b3286b..c22d1118a133 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -826,7 +826,7 @@ static void virtnet_rq_unmap_free_buf(struct virtqueue *vq, void *buf) virtnet_rq_free_buf(vi, rq, buf); } -static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi) +static void free_old_xmit(struct send_queue *sq, bool in_napi) { struct virtnet_sq_free_stats stats = {0}; @@ -880,7 +880,7 @@ static void check_sq_full_and_disable(struct virtnet_info *vi, virtqueue_napi_schedule(&sq->napi, sq->vq); } else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) { /* More just got used, free them then recheck. */ - free_old_xmit_skbs(sq, false); + free_old_xmit(sq, false); if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) { netif_start_subqueue(dev, qnum); virtqueue_disable_cb(sq->vq); @@ -2176,7 +2176,7 @@ static void virtnet_poll_cleantx(struct receive_queue *rq) do { virtqueue_disable_cb(sq->vq); - free_old_xmit_skbs(sq, true); + free_old_xmit(sq, true); } while (unlikely(!virtqueue_enable_cb_delayed(sq->vq))); if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) @@ -2324,7 +2324,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget) txq = netdev_get_tx_queue(vi->dev, index); __netif_tx_lock(txq, raw_smp_processor_id()); virtqueue_disable_cb(sq->vq); - free_old_xmit_skbs(sq, true); + free_old_xmit(sq, true); if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) netif_tx_wake_queue(txq); @@ -2414,7 +2414,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) if (use_napi) virtqueue_disable_cb(sq->vq); - free_old_xmit_skbs(sq, false); + free_old_xmit(sq, false); } while (use_napi && kick && unlikely(!virtqueue_enable_cb_delayed(sq->vq)));