Kernel/linux-next
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git synced 2025-01-04 04:02:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

virtio_ring: put split ring fields in a sub struct

Browse Source 
Put the split ring specific fields in a sub-struct named
as "split" to avoid misuse after introducing packed ring.
There is no functional change.

Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Tiwei Bie 2018-11-21 18:03:21 +08:00 committed by David S. Miller
parent e6f633e5be
commit e593bf9751
1 changed files with 91 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
drivers/virtio/virtio_ring.c
View File

@ -63,9 +63,6 @@ struct vring_desc_state {
struct vring_virtqueue { struct vring_virtqueue {
struct virtqueue vq; struct virtqueue vq;
/* Actual memory layout for this queue */
struct vring vring;
/* Can we use weak barriers? */ /* Can we use weak barriers? */
bool weak_barriers; bool weak_barriers;
@ -86,11 +83,16 @@ struct vring_virtqueue {
/* Last used index we've seen. */ /* Last used index we've seen. */
u16 last_used_idx; u16 last_used_idx;
/* Last written value to avail->flags */ struct {
u16 avail_flags_shadow; /* Actual memory layout for this queue */
struct vring vring;
/* Last written value to avail->idx in guest byte order */ /* Last written value to avail->flags */
u16 avail_idx_shadow; u16 avail_flags_shadow;
/* Last written value to avail->idx in guest byte order */
u16 avail_idx_shadow;
} split;
/* How to notify other side. FIXME: commonalize hcalls! */ /* How to notify other side. FIXME: commonalize hcalls! */
bool (*notify)(struct virtqueue *vq); bool (*notify)(struct virtqueue *vq);
@ -316,7 +318,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
desc = alloc_indirect_split(_vq, total_sg, gfp); desc = alloc_indirect_split(_vq, total_sg, gfp);
else { else {
desc = NULL; desc = NULL;
WARN_ON_ONCE(total_sg > vq->vring.num && !vq->indirect); WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
} }
if (desc) { if (desc) {
@ -327,7 +329,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
descs_used = 1; descs_used = 1;
} else { } else {
indirect = false; indirect = false;
desc = vq->vring.desc; desc = vq->split.vring.desc;
i = head; i = head;
descs_used = total_sg; descs_used = total_sg;
} }
@ -383,10 +385,13 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
if (vring_mapping_error(vq, addr)) if (vring_mapping_error(vq, addr))
goto unmap_release; goto unmap_release;
vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT); vq->split.vring.desc[head].flags = cpu_to_virtio16(_vq->vdev,
vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, addr); VRING_DESC_F_INDIRECT);
vq->split.vring.desc[head].addr = cpu_to_virtio64(_vq->vdev,
addr);
vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc)); vq->split.vring.desc[head].len = cpu_to_virtio32(_vq->vdev,
total_sg * sizeof(struct vring_desc));
} }
/* We're using some buffers from the free list. */ /* We're using some buffers from the free list. */
@ -394,7 +399,8 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
/* Update free pointer */ /* Update free pointer */
if (indirect) if (indirect)
vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next); vq->free_head = virtio16_to_cpu(_vq->vdev,
vq->split.vring.desc[head].next);
else else
vq->free_head = i; vq->free_head = i;
@ -407,14 +413,15 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
/* Put entry in available array (but don't update avail->idx until they /* Put entry in available array (but don't update avail->idx until they
* do sync). */ * do sync). */
avail = vq->avail_idx_shadow & (vq->vring.num - 1); avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head); vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
/* Descriptors and available array need to be set before we expose the /* Descriptors and available array need to be set before we expose the
* new available array entries. */ * new available array entries. */
virtio_wmb(vq->weak_barriers); virtio_wmb(vq->weak_barriers);
vq->avail_idx_shadow++; vq->split.avail_idx_shadow++;
vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow); vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
vq->split.avail_idx_shadow);
vq->num_added++; vq->num_added++;
pr_debug("Added buffer head %i to %p\n", head, vq); pr_debug("Added buffer head %i to %p\n", head, vq);
@ -435,7 +442,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
if (i == err_idx) if (i == err_idx)
break; break;
vring_unmap_one_split(vq, &desc[i]); vring_unmap_one_split(vq, &desc[i]);
i = virtio16_to_cpu(_vq->vdev, vq->vring.desc[i].next); i = virtio16_to_cpu(_vq->vdev, vq->split.vring.desc[i].next);
} }
if (indirect) if (indirect)
@ -456,8 +463,8 @@ static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
* event. */ * event. */
virtio_mb(vq->weak_barriers); virtio_mb(vq->weak_barriers);
old = vq->avail_idx_shadow - vq->num_added; old = vq->split.avail_idx_shadow - vq->num_added;
new = vq->avail_idx_shadow; new = vq->split.avail_idx_shadow;
vq->num_added = 0; vq->num_added = 0;
#ifdef DEBUG #ifdef DEBUG
@ -469,10 +476,13 @@ static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
#endif #endif
if (vq->event) { if (vq->event) {
needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)), needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
vring_avail_event(&vq->split.vring)),
new, old); new, old);
} else { } else {
needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY)); needs_kick = !(vq->split.vring.used->flags &
cpu_to_virtio16(_vq->vdev,
VRING_USED_F_NO_NOTIFY));
} }
END_USE(vq); END_USE(vq);
return needs_kick; return needs_kick;
@ -490,14 +500,15 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
/* Put back on free list: unmap first-level descriptors and find end */ /* Put back on free list: unmap first-level descriptors and find end */
i = head; i = head;
while (vq->vring.desc[i].flags & nextflag) { while (vq->split.vring.desc[i].flags & nextflag) {
vring_unmap_one_split(vq, &vq->vring.desc[i]); vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next); i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next);
vq->vq.num_free++; vq->vq.num_free++;
} }
vring_unmap_one_split(vq, &vq->vring.desc[i]); vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head); vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev,
vq->free_head);
vq->free_head = head; vq->free_head = head;
/* Plus final descriptor */ /* Plus final descriptor */
@ -511,9 +522,10 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
if (!indir_desc) if (!indir_desc)
return; return;
len = virtio32_to_cpu(vq->vq.vdev, vq->vring.desc[head].len); len = virtio32_to_cpu(vq->vq.vdev,
vq->split.vring.desc[head].len);
BUG_ON(!(vq->vring.desc[head].flags & BUG_ON(!(vq->split.vring.desc[head].flags &
cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT))); cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)));
BUG_ON(len == 0 || len % sizeof(struct vring_desc)); BUG_ON(len == 0 || len % sizeof(struct vring_desc));
@ -529,7 +541,8 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
static inline bool more_used_split(const struct vring_virtqueue *vq) static inline bool more_used_split(const struct vring_virtqueue *vq)
{ {
return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx); return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
vq->split.vring.used->idx);
} }
static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq, static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
@ -557,11 +570,13 @@ static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
/* Only get used array entries after they have been exposed by host. */ /* Only get used array entries after they have been exposed by host. */
virtio_rmb(vq->weak_barriers); virtio_rmb(vq->weak_barriers);
last_used = (vq->last_used_idx & (vq->vring.num - 1)); last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id); i = virtio32_to_cpu(_vq->vdev,
*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len); vq->split.vring.used->ring[last_used].id);
*len = virtio32_to_cpu(_vq->vdev,
vq->split.vring.used->ring[last_used].len);
if (unlikely(i >= vq->vring.num)) { if (unlikely(i >= vq->split.vring.num)) {
BAD_RING(vq, "id %u out of range\n", i); BAD_RING(vq, "id %u out of range\n", i);
return NULL; return NULL;
} }
@ -577,9 +592,9 @@ static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
/* If we expect an interrupt for the next entry, tell host /* If we expect an interrupt for the next entry, tell host
* by writing event index and flush out the write before * by writing event index and flush out the write before
* the read in the next get_buf call. */ * the read in the next get_buf call. */
if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
virtio_store_mb(vq->weak_barriers, virtio_store_mb(vq->weak_barriers,
&vring_used_event(&vq->vring), &vring_used_event(&vq->split.vring),
cpu_to_virtio16(_vq->vdev, vq->last_used_idx)); cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
#ifdef DEBUG #ifdef DEBUG
@ -594,10 +609,12 @@ static void virtqueue_disable_cb_split(struct virtqueue *_vq)
{ {
struct vring_virtqueue *vq = to_vvq(_vq); struct vring_virtqueue *vq = to_vvq(_vq);
if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
if (!vq->event) if (!vq->event)
vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); vq->split.vring.avail->flags =
cpu_to_virtio16(_vq->vdev,
vq->split.avail_flags_shadow);
} }
} }
@ -613,12 +630,15 @@ static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
/* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
* either clear the flags bit or point the event index at the next * either clear the flags bit or point the event index at the next
* entry. Always do both to keep code simple. */ * entry. Always do both to keep code simple. */
if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
if (!vq->event) if (!vq->event)
vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); vq->split.vring.avail->flags =
cpu_to_virtio16(_vq->vdev,
vq->split.avail_flags_shadow);
} }
vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx); vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
last_used_idx = vq->last_used_idx);
END_USE(vq); END_USE(vq);
return last_used_idx; return last_used_idx;
} }
@ -628,7 +648,7 @@ static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned last_used_idx)
struct vring_virtqueue *vq = to_vvq(_vq); struct vring_virtqueue *vq = to_vvq(_vq);
return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
vq->vring.used->idx); vq->split.vring.used->idx);
} }
static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq) static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
@ -643,19 +663,22 @@ static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
/* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
* either clear the flags bit or point the event index at the next * either clear the flags bit or point the event index at the next
* entry. Always update the event index to keep code simple. */ * entry. Always update the event index to keep code simple. */
if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
if (!vq->event) if (!vq->event)
vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); vq->split.vring.avail->flags =
cpu_to_virtio16(_vq->vdev,
vq->split.avail_flags_shadow);
} }
/* TODO: tune this threshold */ /* TODO: tune this threshold */
bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4; bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
virtio_store_mb(vq->weak_barriers, virtio_store_mb(vq->weak_barriers,
&vring_used_event(&vq->vring), &vring_used_event(&vq->split.vring),
cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs)); cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) { if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
- vq->last_used_idx) > bufs)) {
END_USE(vq); END_USE(vq);
return false; return false;
} }
@ -672,19 +695,20 @@ static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
START_USE(vq); START_USE(vq);
for (i = 0; i < vq->vring.num; i++) { for (i = 0; i < vq->split.vring.num; i++) {
if (!vq->desc_state[i].data) if (!vq->desc_state[i].data)
continue; continue;
/* detach_buf_split clears data, so grab it now. */ /* detach_buf_split clears data, so grab it now. */
buf = vq->desc_state[i].data; buf = vq->desc_state[i].data;
detach_buf_split(vq, i, NULL); detach_buf_split(vq, i, NULL);
vq->avail_idx_shadow--; vq->split.avail_idx_shadow--;
vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow); vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
vq->split.avail_idx_shadow);
END_USE(vq); END_USE(vq);
return buf; return buf;
} }
/* That should have freed everything. */ /* That should have freed everything. */
BUG_ON(vq->vq.num_free != vq->vring.num); BUG_ON(vq->vq.num_free != vq->split.vring.num);
END_USE(vq); END_USE(vq);
return NULL; return NULL;
@ -1046,7 +1070,6 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index,
if (!vq) if (!vq)
return NULL; return NULL;
vq->vring = vring;
vq->vq.callback = callback; vq->vq.callback = callback;
vq->vq.vdev = vdev; vq->vq.vdev = vdev;
vq->vq.name = name; vq->vq.name = name;
@ -1059,8 +1082,6 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index,
vq->weak_barriers = weak_barriers; vq->weak_barriers = weak_barriers;
vq->broken = false; vq->broken = false;
vq->last_used_idx = 0; vq->last_used_idx = 0;
vq->avail_flags_shadow = 0;
vq->avail_idx_shadow = 0;
vq->num_added = 0; vq->num_added = 0;
list_add_tail(&vq->vq.list, &vdev->vqs); list_add_tail(&vq->vq.list, &vdev->vqs);
#ifdef DEBUG #ifdef DEBUG
@ -1072,17 +1093,22 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index,
!context; !context;
vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
vq->split.vring = vring;
vq->split.avail_flags_shadow = 0;
vq->split.avail_idx_shadow = 0;
/* No callback? Tell other side not to bother us. */ /* No callback? Tell other side not to bother us. */
if (!callback) { if (!callback) {
vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
if (!vq->event) if (!vq->event)
vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow); vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
vq->split.avail_flags_shadow);
} }
/* Put everything in free lists. */ /* Put everything in free lists. */
vq->free_head = 0; vq->free_head = 0;
for (i = 0; i < vring.num-1; i++) for (i = 0; i < vring.num-1; i++)
vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1); vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
memset(vq->desc_state, 0, vring.num * sizeof(struct vring_desc_state)); memset(vq->desc_state, 0, vring.num * sizeof(struct vring_desc_state));
return &vq->vq; return &vq->vq;
@ -1218,7 +1244,7 @@ void vring_del_virtqueue(struct virtqueue *_vq)
if (vq->we_own_ring) { if (vq->we_own_ring) {
vring_free_queue(vq->vq.vdev, vq->queue_size_in_bytes, vring_free_queue(vq->vq.vdev, vq->queue_size_in_bytes,
vq->vring.desc, vq->queue_dma_addr); vq->split.vring.desc, vq->queue_dma_addr);
} }
list_del(&_vq->list); list_del(&_vq->list);
kfree(vq); kfree(vq);
@ -1260,7 +1286,7 @@ unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
struct vring_virtqueue *vq = to_vvq(_vq); struct vring_virtqueue *vq = to_vvq(_vq);
return vq->vring.num; return vq->split.vring.num;
} }
EXPORT_SYMBOL_GPL(virtqueue_get_vring_size); EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
@ -1304,7 +1330,7 @@ dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
BUG_ON(!vq->we_own_ring); BUG_ON(!vq->we_own_ring);
return vq->queue_dma_addr + return vq->queue_dma_addr +
((char *)vq->vring.avail - (char *)vq->vring.desc); ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
} }
EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr); EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
@ -1315,13 +1341,13 @@ dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
BUG_ON(!vq->we_own_ring); BUG_ON(!vq->we_own_ring);
return vq->queue_dma_addr + return vq->queue_dma_addr +
((char *)vq->vring.used - (char *)vq->vring.desc); ((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
} }
EXPORT_SYMBOL_GPL(virtqueue_get_used_addr); EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
const struct vring *virtqueue_get_vring(struct virtqueue *vq) const struct vring *virtqueue_get_vring(struct virtqueue *vq)
{ {
return &to_vvq(vq)->vring; return &to_vvq(vq)->split.vring;
} }
EXPORT_SYMBOL_GPL(virtqueue_get_vring); EXPORT_SYMBOL_GPL(virtqueue_get_vring);