From f2908e792d084828b66528bea6afa674354f5f66 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 23 Jun 2025 14:06:41 +0200 Subject: [PATCH 01/20] ixgbevf: remove legacy Rx Similarly as in commit 53844673d555 ("iavf: kill "legacy-rx" for good"), drop skb construction logic in favor of only using napi_build_skb() as a superior option that reduces the need to allocate and copy memory. As IXGBEVF_PRIV_FLAGS_LEGACY_RX is the only private flag in ixgbevf, entirely remove private flags support from the driver. When compared to iavf changes, ixgbevf has a single complication: MAC type 82599 cannot finely limit the DMA write size with RXDCTL.RLPML, only 1024 increments through SRRCTL are available, see commit fe68195daf34 ("ixgbevf: Require large buffers for build_skb on 82599VF") and commit 2bafa8fac19a ("ixgbe: don't set RXDCTL.RLPML for 82599"). Therefore, this is a special case requiring legacy RX unless large buffers are used. For now, solve this by always using large buffers for this MAC type. Suggested-by: Alexander Lobakin Reviewed-by: Aleksandr Loktionov Reviewed-by: Alexander Lobakin Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ethtool.c | 48 -------- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 13 +-- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 106 ++---------------- 3 files changed, 13 insertions(+), 154 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c index 537a60d5276f0f..274eef39c58618 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c +++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c @@ -72,13 +72,6 @@ static const char ixgbe_gstrings_test[][ETH_GSTRING_LEN] = { #define IXGBEVF_TEST_LEN (sizeof(ixgbe_gstrings_test) / ETH_GSTRING_LEN) -static const char ixgbevf_priv_flags_strings[][ETH_GSTRING_LEN] = { -#define IXGBEVF_PRIV_FLAGS_LEGACY_RX BIT(0) - "legacy-rx", -}; - -#define IXGBEVF_PRIV_FLAGS_STR_LEN ARRAY_SIZE(ixgbevf_priv_flags_strings) - static int ixgbevf_get_link_ksettings(struct net_device *netdev, struct ethtool_link_ksettings *cmd) { @@ -216,8 +209,6 @@ static void ixgbevf_get_drvinfo(struct net_device *netdev, strscpy(drvinfo->driver, ixgbevf_driver_name, sizeof(drvinfo->driver)); strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); - - drvinfo->n_priv_flags = IXGBEVF_PRIV_FLAGS_STR_LEN; } static void ixgbevf_get_ringparam(struct net_device *netdev, @@ -409,8 +400,6 @@ static int ixgbevf_get_sset_count(struct net_device *netdev, int stringset) return IXGBEVF_TEST_LEN; case ETH_SS_STATS: return IXGBEVF_STATS_LEN; - case ETH_SS_PRIV_FLAGS: - return IXGBEVF_PRIV_FLAGS_STR_LEN; default: return -EINVAL; } @@ -538,10 +527,6 @@ static void ixgbevf_get_strings(struct net_device *netdev, u32 stringset, p += ETH_GSTRING_LEN; } break; - case ETH_SS_PRIV_FLAGS: - memcpy(data, ixgbevf_priv_flags_strings, - IXGBEVF_PRIV_FLAGS_STR_LEN * ETH_GSTRING_LEN); - break; } } @@ -931,37 +916,6 @@ static int ixgbevf_get_rxfh(struct net_device *netdev, return err; } -static u32 ixgbevf_get_priv_flags(struct net_device *netdev) -{ - struct ixgbevf_adapter *adapter = netdev_priv(netdev); - u32 priv_flags = 0; - - if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX) - priv_flags |= IXGBEVF_PRIV_FLAGS_LEGACY_RX; - - return priv_flags; -} - -static int ixgbevf_set_priv_flags(struct net_device *netdev, u32 priv_flags) -{ - struct ixgbevf_adapter *adapter = netdev_priv(netdev); - unsigned int flags = adapter->flags; - - flags &= ~IXGBEVF_FLAGS_LEGACY_RX; - if (priv_flags & IXGBEVF_PRIV_FLAGS_LEGACY_RX) - flags |= IXGBEVF_FLAGS_LEGACY_RX; - - if (flags != adapter->flags) { - adapter->flags = flags; - - /* reset interface to repopulate queues */ - if (netif_running(netdev)) - ixgbevf_reinit_locked(adapter); - } - - return 0; -} - static const struct ethtool_ops ixgbevf_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS, .get_drvinfo = ixgbevf_get_drvinfo, @@ -984,8 +938,6 @@ static const struct ethtool_ops ixgbevf_ethtool_ops = { .get_rxfh_key_size = ixgbevf_get_rxfh_key_size, .get_rxfh = ixgbevf_get_rxfh, .get_link_ksettings = ixgbevf_get_link_ksettings, - .get_priv_flags = ixgbevf_get_priv_flags, - .set_priv_flags = ixgbevf_set_priv_flags, }; void ixgbevf_set_ethtool_ops(struct net_device *netdev) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index 516a6fdd23d076..ae2763fea2be2e 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -73,7 +73,6 @@ struct ixgbevf_rx_queue_stats { enum ixgbevf_ring_state_t { __IXGBEVF_RX_3K_BUFFER, - __IXGBEVF_RX_BUILD_SKB_ENABLED, __IXGBEVF_TX_DETECT_HANG, __IXGBEVF_HANG_CHECK_ARMED, __IXGBEVF_TX_XDP_RING, @@ -176,21 +175,13 @@ struct ixgbevf_ring { #define clear_ring_uses_large_buffer(ring) \ clear_bit(__IXGBEVF_RX_3K_BUFFER, &(ring)->state) -#define ring_uses_build_skb(ring) \ - test_bit(__IXGBEVF_RX_BUILD_SKB_ENABLED, &(ring)->state) -#define set_ring_build_skb_enabled(ring) \ - set_bit(__IXGBEVF_RX_BUILD_SKB_ENABLED, &(ring)->state) -#define clear_ring_build_skb_enabled(ring) \ - clear_bit(__IXGBEVF_RX_BUILD_SKB_ENABLED, &(ring)->state) - static inline unsigned int ixgbevf_rx_bufsz(struct ixgbevf_ring *ring) { #if (PAGE_SIZE < 8192) if (ring_uses_large_buffer(ring)) return IXGBEVF_RXBUFFER_3072; - if (ring_uses_build_skb(ring)) - return IXGBEVF_MAX_FRAME_BUILD_SKB; + return IXGBEVF_MAX_FRAME_BUILD_SKB; #endif return IXGBEVF_RXBUFFER_2048; } @@ -377,8 +368,6 @@ struct ixgbevf_adapter { u32 flags; bool link_state; -#define IXGBEVF_FLAGS_LEGACY_RX BIT(1) - #ifdef CONFIG_XFRM struct ixgbevf_ipsec *ipsec; #endif /* CONFIG_XFRM */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index d5ce20f47def1f..fc48c89c7bb857 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -602,7 +602,7 @@ static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, static inline unsigned int ixgbevf_rx_offset(struct ixgbevf_ring *rx_ring) { - return ring_uses_build_skb(rx_ring) ? IXGBEVF_SKB_PAD : 0; + return IXGBEVF_SKB_PAD; } static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, @@ -832,9 +832,7 @@ static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, #if (PAGE_SIZE < 8192) unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; #else - unsigned int truesize = ring_uses_build_skb(rx_ring) ? - SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) : - SKB_DATA_ALIGN(size); + unsigned int truesize = SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size); #endif skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, rx_buffer->page_offset, size, truesize); @@ -845,74 +843,6 @@ static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, #endif } -static -struct sk_buff *ixgbevf_construct_skb(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int size = xdp->data_end - xdp->data; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); -#endif - unsigned int headlen; - struct sk_buff *skb; - - /* prefetch first cache line of first page */ - net_prefetch(xdp->data); - - /* Note, we get here by enabling legacy-rx via: - * - * ethtool --set-priv-flags legacy-rx on - * - * In this mode, we currently get 0 extra XDP headroom as - * opposed to having legacy-rx off, where we process XDP - * packets going to stack via ixgbevf_build_skb(). - * - * For ixgbevf_construct_skb() mode it means that the - * xdp->data_meta will always point to xdp->data, since - * the helper cannot expand the head. Should this ever - * changed in future for legacy-rx mode on, then lets also - * add xdp->data_meta handling here. - */ - - /* allocate a skb to store the frags */ - skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBEVF_RX_HDR_SIZE); - if (unlikely(!skb)) - return NULL; - - /* Determine available headroom for copy */ - headlen = size; - if (headlen > IXGBEVF_RX_HDR_SIZE) - headlen = eth_get_headlen(skb->dev, xdp->data, - IXGBEVF_RX_HDR_SIZE); - - /* align pull length to size of long to optimize memcpy performance */ - memcpy(__skb_put(skb, headlen), xdp->data, - ALIGN(headlen, sizeof(long))); - - /* update all of the pointers */ - size -= headlen; - if (size) { - skb_add_rx_frag(skb, 0, rx_buffer->page, - (xdp->data + headlen) - - page_address(rx_buffer->page), - size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - } else { - rx_buffer->pagecnt_bias++; - } - - return skb; -} - static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, u32 qmask) { @@ -1092,10 +1022,8 @@ static unsigned int ixgbevf_rx_frame_truesize(struct ixgbevf_ring *rx_ring, #if (PAGE_SIZE < 8192) truesize = ixgbevf_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ #else - truesize = ring_uses_build_skb(rx_ring) ? - SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : - SKB_DATA_ALIGN(size); + truesize = SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); #endif return truesize; } @@ -1182,12 +1110,9 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, total_rx_bytes += size; } else if (skb) { ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size); - } else if (ring_uses_build_skb(rx_ring)) { + } else { skb = ixgbevf_build_skb(rx_ring, rx_buffer, &xdp, rx_desc); - } else { - skb = ixgbevf_construct_skb(rx_ring, rx_buffer, - &xdp, rx_desc); } /* exit if we failed to retrieve a buffer */ @@ -1958,8 +1883,7 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, #if (PAGE_SIZE < 8192) /* Limit the maximum frame size so we don't overrun the skb */ - if (ring_uses_build_skb(ring) && - !ring_uses_large_buffer(ring)) + if (!ring_uses_large_buffer(ring)) rxdctl |= IXGBEVF_MAX_FRAME_BUILD_SKB | IXGBE_RXDCTL_RLPML_EN; #endif @@ -1978,22 +1902,16 @@ static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter, struct net_device *netdev = adapter->netdev; unsigned int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - /* set build_skb and buffer size flags */ - clear_ring_build_skb_enabled(rx_ring); + /* set buffer size flags */ clear_ring_uses_large_buffer(rx_ring); - if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX) - return; - if (PAGE_SIZE < 8192) - if (max_frame > IXGBEVF_MAX_FRAME_BUILD_SKB) + /* 82599 can't rely on RXDCTL.RLPML to restrict + * the size of the frame + */ + if (max_frame > IXGBEVF_MAX_FRAME_BUILD_SKB || + adapter->hw.mac.type == ixgbe_mac_82599_vf) set_ring_uses_large_buffer(rx_ring); - - /* 82599 can't rely on RXDCTL.RLPML to restrict the size of the frame */ - if (adapter->hw.mac.type == ixgbe_mac_82599_vf && !ring_uses_large_buffer(rx_ring)) - return; - - set_ring_build_skb_enabled(rx_ring); } /** From d7bf74b613322bb4682073988e4f98baece9370d Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 24 Jun 2025 12:49:22 +0200 Subject: [PATCH 02/20] ixgbevf: do not share pages between packets Again, same as in the related iavf commit 920d86f3c552 ("iavf: drop page splitting and recycling"), as an intermediate step, drop the page sharing and recycling logic in a preparation to offload it to page_pool. Instead of the previous sharing and recycling, just allocate a new page every time. Suggested-by: Alexander Lobakin Reviewed-by: Aleksandr Loktionov Reviewed-by: Alexander Lobakin Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 44 +--- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 239 ++---------------- 2 files changed, 28 insertions(+), 255 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index ae2763fea2be2e..2d7ca3f86868bc 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -45,12 +45,7 @@ struct ixgbevf_tx_buffer { struct ixgbevf_rx_buffer { dma_addr_t dma; struct page *page; -#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) __u32 page_offset; -#else - __u16 page_offset; -#endif - __u16 pagecnt_bias; }; struct ixgbevf_stats { @@ -72,7 +67,6 @@ struct ixgbevf_rx_queue_stats { }; enum ixgbevf_ring_state_t { - __IXGBEVF_RX_3K_BUFFER, __IXGBEVF_TX_DETECT_HANG, __IXGBEVF_HANG_CHECK_ARMED, __IXGBEVF_TX_XDP_RING, @@ -143,8 +137,7 @@ struct ixgbevf_ring { #define IXGBEVF_MIN_RXD 64 /* Supported Rx Buffer Sizes */ -#define IXGBEVF_RXBUFFER_256 256 /* Used for packet split */ -#define IXGBEVF_RXBUFFER_2048 2048 +#define IXGBEVF_RXBUFFER_256 256 #define IXGBEVF_RXBUFFER_3072 3072 #define IXGBEVF_RX_HDR_SIZE IXGBEVF_RXBUFFER_256 @@ -152,12 +145,6 @@ struct ixgbevf_ring { #define MAXIMUM_ETHERNET_VLAN_SIZE (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN) #define IXGBEVF_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) -#if (PAGE_SIZE < 8192) -#define IXGBEVF_MAX_FRAME_BUILD_SKB \ - (SKB_WITH_OVERHEAD(IXGBEVF_RXBUFFER_2048) - IXGBEVF_SKB_PAD) -#else -#define IXGBEVF_MAX_FRAME_BUILD_SKB IXGBEVF_RXBUFFER_2048 -#endif #define IXGBE_TX_FLAGS_CSUM BIT(0) #define IXGBE_TX_FLAGS_VLAN BIT(1) @@ -168,35 +155,6 @@ struct ixgbevf_ring { #define IXGBE_TX_FLAGS_VLAN_PRIO_MASK 0x0000e000 #define IXGBE_TX_FLAGS_VLAN_SHIFT 16 -#define ring_uses_large_buffer(ring) \ - test_bit(__IXGBEVF_RX_3K_BUFFER, &(ring)->state) -#define set_ring_uses_large_buffer(ring) \ - set_bit(__IXGBEVF_RX_3K_BUFFER, &(ring)->state) -#define clear_ring_uses_large_buffer(ring) \ - clear_bit(__IXGBEVF_RX_3K_BUFFER, &(ring)->state) - -static inline unsigned int ixgbevf_rx_bufsz(struct ixgbevf_ring *ring) -{ -#if (PAGE_SIZE < 8192) - if (ring_uses_large_buffer(ring)) - return IXGBEVF_RXBUFFER_3072; - - return IXGBEVF_MAX_FRAME_BUILD_SKB; -#endif - return IXGBEVF_RXBUFFER_2048; -} - -static inline unsigned int ixgbevf_rx_pg_order(struct ixgbevf_ring *ring) -{ -#if (PAGE_SIZE < 8192) - if (ring_uses_large_buffer(ring)) - return 1; -#endif - return 0; -} - -#define ixgbevf_rx_pg_size(_ring) (PAGE_SIZE << ixgbevf_rx_pg_order(_ring)) - #define check_for_tx_hang(ring) \ test_bit(__IXGBEVF_TX_DETECT_HANG, &(ring)->state) #define set_check_for_tx_hang(ring) \ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index fc48c89c7bb857..05baf28823c869 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -112,9 +112,6 @@ static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); -static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer); -static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *old_buff); static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) { @@ -537,40 +534,20 @@ struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring, rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; prefetchw(rx_buffer->page); - /* we are reusing so sync this buffer for CPU use */ dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, size, DMA_FROM_DEVICE); - rx_buffer->pagecnt_bias--; - return rx_buffer; } static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - struct sk_buff *skb) + struct ixgbevf_rx_buffer *rx_buffer) { - if (ixgbevf_can_reuse_rx_page(rx_buffer)) { - /* hand second half of page back to the ring */ - ixgbevf_reuse_rx_page(rx_ring, rx_buffer); - } else { - if (IS_ERR(skb)) - /* We are not reusing the buffer so unmap it and free - * any references we are holding to it - */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbevf_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBEVF_RX_DMA_ATTR); - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); - } - - /* clear contents of rx_buffer */ - rx_buffer->page = NULL; + dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE, + DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); } /** @@ -600,38 +577,28 @@ static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, return true; } -static inline unsigned int ixgbevf_rx_offset(struct ixgbevf_ring *rx_ring) -{ - return IXGBEVF_SKB_PAD; -} - static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, struct ixgbevf_rx_buffer *bi) { - struct page *page = bi->page; + struct page *page; dma_addr_t dma; - /* since we are recycling buffers we should seldom need to alloc */ - if (likely(page)) - return true; - /* alloc new page for storage */ - page = dev_alloc_pages(ixgbevf_rx_pg_order(rx_ring)); + page = dev_alloc_page(); if (unlikely(!page)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; } /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, - ixgbevf_rx_pg_size(rx_ring), + dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); /* if mapping failed free memory back to system since * there isn't much point in holding memory we can't use */ if (dma_mapping_error(rx_ring->dev, dma)) { - __free_pages(page, ixgbevf_rx_pg_order(rx_ring)); + __free_page(page); rx_ring->rx_stats.alloc_rx_page_failed++; return false; @@ -639,8 +606,7 @@ static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, bi->dma = dma; bi->page = page; - bi->page_offset = ixgbevf_rx_offset(rx_ring); - bi->pagecnt_bias = 1; + bi->page_offset = IXGBEVF_SKB_PAD; rx_ring->rx_stats.alloc_rx_page++; return true; @@ -673,7 +639,7 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, /* sync the buffer for use by the device */ dma_sync_single_range_for_device(rx_ring->dev, bi->dma, bi->page_offset, - ixgbevf_rx_bufsz(rx_ring), + IXGBEVF_RXBUFFER_3072, DMA_FROM_DEVICE); /* Refresh the desc even if pkt_addr didn't change @@ -755,66 +721,6 @@ static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, return false; } -/** - * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring - * @rx_ring: rx descriptor ring to store buffers on - * @old_buff: donor buffer to have page reused - * - * Synchronizes page for reuse by the adapter - **/ -static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *old_buff) -{ - struct ixgbevf_rx_buffer *new_buff; - u16 nta = rx_ring->next_to_alloc; - - new_buff = &rx_ring->rx_buffer_info[nta]; - - /* update, and store next to alloc */ - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - /* transfer page from old buffer to new buffer */ - new_buff->page = old_buff->page; - new_buff->dma = old_buff->dma; - new_buff->page_offset = old_buff->page_offset; - new_buff->pagecnt_bias = old_buff->pagecnt_bias; -} - -static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer) -{ - unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; - struct page *page = rx_buffer->page; - - /* avoid re-using remote and pfmemalloc pages */ - if (!dev_page_is_reusable(page)) - return false; - -#if (PAGE_SIZE < 8192) - /* if we are only owner of page we can reuse it */ - if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) - return false; -#else -#define IXGBEVF_LAST_OFFSET \ - (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBEVF_RXBUFFER_2048) - - if (rx_buffer->page_offset > IXGBEVF_LAST_OFFSET) - return false; - -#endif - - /* If we have drained the page fragment pool we need to update - * the pagecnt_bias and page count so that we fully restock the - * number of references the driver holds. - */ - if (unlikely(!pagecnt_bias)) { - page_ref_add(page, USHRT_MAX); - rx_buffer->pagecnt_bias = USHRT_MAX; - } - - return true; -} - /** * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff * @rx_ring: rx descriptor ring to transact packets on @@ -829,18 +735,10 @@ static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, struct sk_buff *skb, unsigned int size) { -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; -#else unsigned int truesize = SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size); -#endif + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, rx_buffer->page_offset, size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif } static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, @@ -857,13 +755,9 @@ static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc) { unsigned int metasize = xdp->data - xdp->data_meta; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; -#else unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + SKB_DATA_ALIGN(xdp->data_end - xdp->data_hard_start); -#endif struct sk_buff *skb; /* Prefetch first cache line of first page. If xdp->data_meta @@ -884,13 +778,6 @@ static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, if (metasize) skb_metadata_set(skb, metasize); - /* update buffer offset */ -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - return skb; } @@ -1014,38 +901,11 @@ static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, return result; } -static unsigned int ixgbevf_rx_frame_truesize(struct ixgbevf_ring *rx_ring, - unsigned int size) -{ - unsigned int truesize; - -#if (PAGE_SIZE < 8192) - truesize = ixgbevf_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ -#else - truesize = SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); -#endif - return truesize; -} - -static void ixgbevf_rx_buffer_flip(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - unsigned int size) -{ - unsigned int truesize = ixgbevf_rx_frame_truesize(rx_ring, size); - -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif -} - static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *rx_ring, int budget) { - unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbevf_adapter *adapter = q_vector->adapter; u16 cleaned_count = ixgbevf_desc_unused(rx_ring); struct sk_buff *skb = rx_ring->skb; @@ -1054,10 +914,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, int xdp_res = 0; /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ -#if (PAGE_SIZE < 8192) - frame_sz = ixgbevf_rx_frame_truesize(rx_ring, 0); -#endif - xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); + xdp_init_buff(&xdp, IXGBEVF_RXBUFFER_3072, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { struct ixgbevf_rx_buffer *rx_buffer; @@ -1081,31 +938,24 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, */ rmb(); - rx_buffer = ixgbevf_get_rx_buffer(rx_ring, size); + rx_buffer = + ixgbevf_get_rx_buffer(rx_ring, IXGBEVF_RXBUFFER_3072); /* retrieve a buffer from the ring */ if (!skb) { - unsigned int offset = ixgbevf_rx_offset(rx_ring); + unsigned int offset = rx_buffer->page_offset; unsigned char *hard_start; hard_start = page_address(rx_buffer->page) + rx_buffer->page_offset - offset; xdp_prepare_buff(&xdp, hard_start, offset, size, true); -#if (PAGE_SIZE > 4096) - /* At larger PAGE_SIZE, frame_sz depend on len size */ - xdp.frame_sz = ixgbevf_rx_frame_truesize(rx_ring, size); -#endif xdp_res = ixgbevf_run_xdp(adapter, rx_ring, &xdp); } if (xdp_res) { - if (xdp_res == IXGBEVF_XDP_TX) { + if (xdp_res == IXGBEVF_XDP_TX) xdp_xmit = true; - ixgbevf_rx_buffer_flip(rx_ring, rx_buffer, - size); - } else { - rx_buffer->pagecnt_bias++; - } + total_rx_packets++; total_rx_bytes += size; } else if (skb) { @@ -1118,11 +968,13 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* exit if we failed to retrieve a buffer */ if (!xdp_res && !skb) { rx_ring->rx_stats.alloc_rx_buff_failed++; - rx_buffer->pagecnt_bias++; break; } - ixgbevf_put_rx_buffer(rx_ring, rx_buffer, skb); + ixgbevf_put_rx_buffer(rx_ring, rx_buffer); + if (xdp_res == IXGBEVF_XDP_CONSUMED) + __free_page(rx_buffer->page); + rx_buffer->page = NULL; cleaned_count++; /* fetch next buffer in frame if non-eop */ @@ -1699,10 +1551,7 @@ static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, srrctl = IXGBE_SRRCTL_DROP_EN; srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - if (ring_uses_large_buffer(ring)) - srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - else - srrctl |= IXGBEVF_RXBUFFER_2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; + srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); @@ -1880,13 +1729,6 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, if (adapter->hw.mac.type != ixgbe_mac_82599_vf) { rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); - -#if (PAGE_SIZE < 8192) - /* Limit the maximum frame size so we don't overrun the skb */ - if (!ring_uses_large_buffer(ring)) - rxdctl |= IXGBEVF_MAX_FRAME_BUILD_SKB | - IXGBE_RXDCTL_RLPML_EN; -#endif } rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; @@ -1896,24 +1738,6 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); } -static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *rx_ring) -{ - struct net_device *netdev = adapter->netdev; - unsigned int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - - /* set buffer size flags */ - clear_ring_uses_large_buffer(rx_ring); - - if (PAGE_SIZE < 8192) - /* 82599 can't rely on RXDCTL.RLPML to restrict - * the size of the frame - */ - if (max_frame > IXGBEVF_MAX_FRAME_BUILD_SKB || - adapter->hw.mac.type == ixgbe_mac_82599_vf) - set_ring_uses_large_buffer(rx_ring); -} - /** * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset * @adapter: board private structure @@ -1944,7 +1768,6 @@ static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) for (i = 0; i < adapter->num_rx_queues; i++) { struct ixgbevf_ring *rx_ring = adapter->rx_ring[i]; - ixgbevf_set_rx_buffer_len(adapter, rx_ring); ixgbevf_configure_rx_ring(adapter, rx_ring); } } @@ -2323,19 +2146,13 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, - ixgbevf_rx_bufsz(rx_ring), + IXGBEVF_RXBUFFER_3072, DMA_FROM_DEVICE); /* free resources associated with mapping */ - dma_unmap_page_attrs(rx_ring->dev, - rx_buffer->dma, - ixgbevf_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBEVF_RX_DMA_ATTR); - - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); - + ixgbevf_put_rx_buffer(rx_ring, rx_buffer); + __free_page(rx_buffer->page); + rx_buffer->page = NULL; i++; if (i == rx_ring->count) i = 0; @@ -4394,9 +4211,7 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) /* verify ixgbevf ring attributes are sufficient for XDP */ for (i = 0; i < adapter->num_rx_queues; i++) { - struct ixgbevf_ring *ring = adapter->rx_ring[i]; - - if (frame_size > ixgbevf_rx_bufsz(ring)) + if (frame_size > IXGBEVF_RXBUFFER_3072) return -EINVAL; } From 0cb83cd3caefa7e390b259f56d714a9e645b7e6c Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 9 Jul 2025 09:28:02 +0200 Subject: [PATCH 03/20] ixgbevf: use libeth in Rx processing Use page_pool buffers by the means of libeth in the Rx queues, this significantly reduces code complexity of the driver itself. Suggested-by: Alexander Lobakin Reviewed-by: Alexander Lobakin Reviewed-by: Aleksandr Loktionov Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/Kconfig | 1 + drivers/net/ethernet/intel/ixgbevf/defines.h | 2 +- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 21 +- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 352 +++++++----------- 4 files changed, 139 insertions(+), 237 deletions(-) diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig index 288fa8ce53af0a..b513baf3cbb296 100644 --- a/drivers/net/ethernet/intel/Kconfig +++ b/drivers/net/ethernet/intel/Kconfig @@ -203,6 +203,7 @@ config IXGBE_IPSEC config IXGBEVF tristate "Intel(R) 10GbE PCI Express Virtual Function Ethernet support" depends on PCI_MSI + select LIBETH_XDP help This driver supports Intel(R) PCI Express virtual functions for the Intel(R) ixgbe driver. For more information on how to identify your diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index e177d1d58696aa..afc927dd14381b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -71,7 +71,7 @@ typedef u32 ixgbe_link_speed; #define IXGBE_PSRTYPE_L2HDR 0x00001000 /* SRRCTL bit definitions */ -#define IXGBE_SRRCTL_BSIZEPKT_SHIFT 10 /* so many KBs */ +#define IXGBE_SRRCTL_BSIZEPKT_STEP 1024 #define IXGBE_SRRCTL_RDMTS_SHIFT 22 #define IXGBE_SRRCTL_RDMTS_MASK 0x01C00000 #define IXGBE_SRRCTL_DROP_EN 0x10000000 diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index 2d7ca3f86868bc..ebf771f0caa4b3 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -42,12 +42,6 @@ struct ixgbevf_tx_buffer { u32 tx_flags; }; -struct ixgbevf_rx_buffer { - dma_addr_t dma; - struct page *page; - __u32 page_offset; -}; - struct ixgbevf_stats { u64 packets; u64 bytes; @@ -84,19 +78,22 @@ struct ixgbevf_ring { struct ixgbevf_ring *next; struct ixgbevf_q_vector *q_vector; /* backpointer to q_vector */ struct net_device *netdev; - struct bpf_prog *xdp_prog; - struct device *dev; + struct bpf_prog __rcu *xdp_prog; + union { + struct page_pool *pp; /* Rx ring */ + struct device *dev; /* Tx ring */ + }; void *desc; /* descriptor ring memory */ dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ + u32 truesize; /* Rx buffer full size */ u16 count; /* amount of descriptors */ u16 next_to_use; u16 next_to_clean; - u16 next_to_alloc; union { + struct libeth_fqe *rx_fqes; struct ixgbevf_tx_buffer *tx_buffer_info; - struct ixgbevf_rx_buffer *rx_buffer_info; }; unsigned long state; struct ixgbevf_stats stats; @@ -115,6 +112,7 @@ struct ixgbevf_ring { */ u16 reg_idx; int queue_index; /* needed for multiqueue queue management */ + u32 rx_buf_len; } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -144,7 +142,8 @@ struct ixgbevf_ring { #define MAXIMUM_ETHERNET_VLAN_SIZE (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN) -#define IXGBEVF_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) +#define IXGBEVF_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) #define IXGBE_TX_FLAGS_CSUM BIT(0) #define IXGBE_TX_FLAGS_VLAN BIT(1) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 05baf28823c869..cedbf0a4d0a546 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "ixgbevf.h" @@ -82,6 +83,7 @@ static const struct pci_device_id ixgbevf_pci_tbl[] = { MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); +MODULE_IMPORT_NS("LIBETH"); MODULE_LICENSE("GPL v2"); #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) @@ -304,7 +306,7 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, /* free the skb */ if (ring_is_xdp(tx_ring)) - page_frag_free(tx_buffer->data); + libeth_xdp_return_va(tx_buffer->data, true); else napi_consume_skb(tx_buffer->skb, napi_budget); @@ -521,33 +523,6 @@ static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP)) ixgbevf_ipsec_rx(rx_ring, rx_desc, skb); - - skb->protocol = eth_type_trans(skb, rx_ring->netdev); -} - -static -struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring, - const unsigned int size) -{ - struct ixgbevf_rx_buffer *rx_buffer; - - rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - prefetchw(rx_buffer->page); - - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - size, - DMA_FROM_DEVICE); - - return rx_buffer; -} - -static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer) -{ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE, - DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); } /** @@ -577,41 +552,6 @@ static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, return true; } -static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *bi) -{ - struct page *page; - dma_addr_t dma; - - /* alloc new page for storage */ - page = dev_alloc_page(); - if (unlikely(!page)) { - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, - DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); - - /* if mapping failed free memory back to system since - * there isn't much point in holding memory we can't use - */ - if (dma_mapping_error(rx_ring->dev, dma)) { - __free_page(page); - - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - bi->dma = dma; - bi->page = page; - bi->page_offset = IXGBEVF_SKB_PAD; - rx_ring->rx_stats.alloc_rx_page++; - - return true; -} - /** * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on @@ -621,39 +561,34 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, u16 cleaned_count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbevf_rx_buffer *bi; - unsigned int i = rx_ring->next_to_use; + const struct libeth_fq_fp fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + .truesize = rx_ring->truesize, + .count = rx_ring->count, + }; + u16 ntu = rx_ring->next_to_use; /* nothing to do or no valid netdev defined */ - if (!cleaned_count || !rx_ring->netdev) + if (unlikely(!cleaned_count || !rx_ring->netdev)) return; - rx_desc = IXGBEVF_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; - i -= rx_ring->count; + rx_desc = IXGBEVF_RX_DESC(rx_ring, ntu); do { - if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) + dma_addr_t addr; + + addr = libeth_rx_alloc(&fq, ntu); + if (addr == DMA_MAPPING_ERROR) break; - /* sync the buffer for use by the device */ - dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, - IXGBEVF_RXBUFFER_3072, - DMA_FROM_DEVICE); - - /* Refresh the desc even if pkt_addr didn't change - * because each write-back erases this info. - */ - rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); + rx_desc->read.pkt_addr = cpu_to_le64(addr); rx_desc++; - bi++; - i++; - if (unlikely(!i)) { + ntu++; + if (unlikely(ntu == fq.count)) { rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; - i -= rx_ring->count; + ntu = 0; } /* clear the length for the next_to_use descriptor */ @@ -662,14 +597,9 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, cleaned_count--; } while (cleaned_count); - i += rx_ring->count; - - if (rx_ring->next_to_use != i) { + if (likely(rx_ring->next_to_use != ntu)) { /* record the next descriptor to use */ - rx_ring->next_to_use = i; - - /* update next to alloc since we have filled the ring */ - rx_ring->next_to_alloc = i; + rx_ring->next_to_use = ntu; /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only @@ -677,7 +607,7 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, * such as IA-64). */ wmb(); - ixgbevf_write_tail(rx_ring, i); + ixgbevf_write_tail(rx_ring, ntu); } } @@ -714,10 +644,6 @@ static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, } } - /* if eth_skb_pad returns an error the skb was freed */ - if (eth_skb_pad(skb)) - return true; - return false; } @@ -730,15 +656,15 @@ static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, * * This function will add the data contained in rx_buffer->page to the skb. **/ -static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, +static void ixgbevf_add_rx_frag(const struct libeth_fqe *rx_buffer, struct sk_buff *skb, unsigned int size) { - unsigned int truesize = SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size); + u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, - rx_buffer->page_offset, size, truesize); + skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, + rx_buffer->netmem, rx_buffer->offset + hr, + size, rx_buffer->truesize); } static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, @@ -749,38 +675,6 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); } -static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int metasize = xdp->data - xdp->data_meta; - unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + - SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); - struct sk_buff *skb; - - /* Prefetch first cache line of first page. If xdp->data_meta - * is unused, this points to xdp->data, otherwise, we likely - * have a consumer accessing first few bytes of meta data, - * and then actual data. - */ - net_prefetch(xdp->data_meta); - - /* build an skb around the page buffer */ - skb = napi_build_skb(xdp->data_hard_start, truesize); - if (unlikely(!skb)) - return NULL; - - /* update pointers within the skb to store the data */ - skb_reserve(skb, xdp->data - xdp->data_hard_start); - __skb_put(skb, xdp->data_end - xdp->data); - if (metasize) - skb_metadata_set(skb, metasize); - - return skb; -} - #define IXGBEVF_XDP_PASS 0 #define IXGBEVF_XDP_CONSUMED 1 #define IXGBEVF_XDP_TX 2 @@ -864,25 +758,25 @@ static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring, static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring, - struct xdp_buff *xdp) + struct libeth_xdp_buff *xdp) { int result = IXGBEVF_XDP_PASS; struct ixgbevf_ring *xdp_ring; struct bpf_prog *xdp_prog; u32 act; - xdp_prog = READ_ONCE(rx_ring->xdp_prog); + xdp_prog = rcu_dereference(rx_ring->xdp_prog); if (!xdp_prog) goto xdp_out; - act = bpf_prog_run_xdp(xdp_prog, xdp); + act = bpf_prog_run_xdp(xdp_prog, &xdp->base); switch (act) { case XDP_PASS: break; case XDP_TX: xdp_ring = adapter->xdp_ring[rx_ring->queue_index]; - result = ixgbevf_xmit_xdp_ring(xdp_ring, xdp); + result = ixgbevf_xmit_xdp_ring(xdp_ring, &xdp->base); if (result == IXGBEVF_XDP_CONSUMED) goto out_failure; break; @@ -895,6 +789,7 @@ static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: result = IXGBEVF_XDP_CONSUMED; + libeth_xdp_return_buff(xdp); break; } xdp_out: @@ -909,16 +804,15 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_adapter *adapter = q_vector->adapter; u16 cleaned_count = ixgbevf_desc_unused(rx_ring); struct sk_buff *skb = rx_ring->skb; + LIBETH_XDP_ONSTACK_BUFF(xdp); bool xdp_xmit = false; - struct xdp_buff xdp; int xdp_res = 0; - /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ - xdp_init_buff(&xdp, IXGBEVF_RXBUFFER_3072, &rx_ring->xdp_rxq); + xdp->base.rxq = &rx_ring->xdp_rxq; while (likely(total_rx_packets < budget)) { - struct ixgbevf_rx_buffer *rx_buffer; union ixgbe_adv_rx_desc *rx_desc; + struct libeth_fqe *rx_buffer; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -929,7 +823,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); - if (!size) + if (unlikely(!size)) break; /* This memory barrier is needed to keep us from reading @@ -938,18 +832,14 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, */ rmb(); - rx_buffer = - ixgbevf_get_rx_buffer(rx_ring, IXGBEVF_RXBUFFER_3072); + rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; + libeth_rx_sync_for_cpu(rx_buffer, size); /* retrieve a buffer from the ring */ if (!skb) { - unsigned int offset = rx_buffer->page_offset; - unsigned char *hard_start; - - hard_start = page_address(rx_buffer->page) + - rx_buffer->page_offset - offset; - xdp_prepare_buff(&xdp, hard_start, offset, size, true); - xdp_res = ixgbevf_run_xdp(adapter, rx_ring, &xdp); + libeth_xdp_prepare_buff(xdp, rx_buffer, size); + prefetch(xdp->data); + xdp_res = ixgbevf_run_xdp(adapter, rx_ring, xdp); } if (xdp_res) { @@ -959,22 +849,18 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, total_rx_packets++; total_rx_bytes += size; } else if (skb) { - ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size); + ixgbevf_add_rx_frag(rx_buffer, skb, size); } else { - skb = ixgbevf_build_skb(rx_ring, rx_buffer, - &xdp, rx_desc); + skb = xdp_build_skb_from_buff(&xdp->base); } /* exit if we failed to retrieve a buffer */ - if (!xdp_res && !skb) { + if (unlikely(!xdp_res && !skb)) { + libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; break; } - ixgbevf_put_rx_buffer(rx_ring, rx_buffer); - if (xdp_res == IXGBEVF_XDP_CONSUMED) - __free_page(rx_buffer->page); - rx_buffer->page = NULL; cleaned_count++; /* fetch next buffer in frame if non-eop */ @@ -982,21 +868,19 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, continue; /* verify the packet layout is correct */ - if (xdp_res || ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { + if (xdp_res || + unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; - /* Workaround hardware that can't do proper VEPA multicast * source pruning. */ - if ((skb->pkt_type == PACKET_BROADCAST || - skb->pkt_type == PACKET_MULTICAST) && - ether_addr_equal(rx_ring->netdev->dev_addr, - eth_hdr(skb)->h_source)) { + if (unlikely((skb->pkt_type == PACKET_BROADCAST || + skb->pkt_type == PACKET_MULTICAST) && + ether_addr_equal(rx_ring->netdev->dev_addr, + eth_hdr(skb)->h_source))) { dev_kfree_skb_irq(skb); continue; } @@ -1004,13 +888,14 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* populate checksum, VLAN, and protocol */ ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); + /* probably a little skewed due to removing CRC */ + total_rx_bytes += skb->len; + total_rx_packets++; + ixgbevf_rx_skb(q_vector, skb); /* reset skb pointer */ skb = NULL; - - /* update budget accounting */ - total_rx_packets++; } /* place incomplete frames back on ring for completion */ @@ -1543,7 +1428,8 @@ static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring, int index) + struct ixgbevf_ring *ring, int index, + bool rlpml_valid) { struct ixgbe_hw *hw = &adapter->hw; u32 srrctl; @@ -1551,7 +1437,11 @@ static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, srrctl = IXGBE_SRRCTL_DROP_EN; srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; + if (rlpml_valid) + srrctl |= DIV_ROUND_UP(ring->rx_buf_len, + IXGBE_SRRCTL_BSIZEPKT_STEP); + else + srrctl |= ring->rx_buf_len / IXGBE_SRRCTL_BSIZEPKT_STEP; srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); @@ -1682,9 +1572,10 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, { struct ixgbe_hw *hw = &adapter->hw; union ixgbe_adv_rx_desc *rx_desc; + u8 reg_idx = ring->reg_idx; + bool rlpml_valid = false; u64 rdba = ring->dma; u32 rxdctl; - u8 reg_idx = ring->reg_idx; /* disable queue to avoid issues while updating state */ rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); @@ -1710,10 +1601,6 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); - /* initialize rx_buffer_info */ - memset(ring->rx_buffer_info, 0, - sizeof(struct ixgbevf_rx_buffer) * ring->count); - /* initialize Rx descriptor 0 */ rx_desc = IXGBEVF_RX_DESC(ring, 0); rx_desc->wb.upper.length = 0; @@ -1721,16 +1608,21 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, /* reset ntu and ntc to place SW in sync with hardwdare */ ring->next_to_clean = 0; ring->next_to_use = 0; - ring->next_to_alloc = 0; - - ixgbevf_configure_srrctl(adapter, ring, reg_idx); /* RXDCTL.RLPML does not work on 82599 */ if (adapter->hw.mac.type != ixgbe_mac_82599_vf) { - rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); + u32 pkt_len = + READ_ONCE(adapter->netdev->mtu) + LIBETH_RX_LL_LEN; + + rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); + if (pkt_len <= IXGBE_RXDCTL_RLPMLMASK) { + rxdctl |= pkt_len | IXGBE_RXDCTL_RLPML_EN; + rlpml_valid = true; + } } + ixgbevf_configure_srrctl(adapter, ring, reg_idx, rlpml_valid); + rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); @@ -2126,8 +2018,6 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter) **/ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) { - u16 i = rx_ring->next_to_clean; - /* Free Rx ring sk_buff */ if (rx_ring->skb) { dev_kfree_skb(rx_ring->skb); @@ -2135,30 +2025,14 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) } /* Free all the Rx ring pages */ - while (i != rx_ring->next_to_alloc) { - struct ixgbevf_rx_buffer *rx_buffer; + for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { + const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; - rx_buffer = &rx_ring->rx_buffer_info[i]; - - /* Invalidate cache lines that may have been written to by - * device so that we avoid corrupting memory. - */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - IXGBEVF_RXBUFFER_3072, - DMA_FROM_DEVICE); - - /* free resources associated with mapping */ - ixgbevf_put_rx_buffer(rx_ring, rx_buffer); - __free_page(rx_buffer->page); - rx_buffer->page = NULL; - i++; - if (i == rx_ring->count) + libeth_rx_recycle_slow(rx_fqe->netmem); + if (unlikely(++i == rx_ring->count)) i = 0; } - rx_ring->next_to_alloc = 0; rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; } @@ -2177,7 +2051,7 @@ static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) /* Free all the Tx ring sk_buffs */ if (ring_is_xdp(tx_ring)) - page_frag_free(tx_buffer->data); + libeth_xdp_return_va(tx_buffer->data, false); else dev_kfree_skb_any(tx_buffer->skb); @@ -3259,12 +3133,25 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring) { - int size; + struct libeth_fq fq = { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_MTU, + .xdp = !!rx_ring->xdp_prog, + .buf_len = IXGBEVF_RX_PAGE_LEN(rx_ring->xdp_prog ? + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), + }; + int ret; - size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; - rx_ring->rx_buffer_info = vmalloc(size); - if (!rx_ring->rx_buffer_info) - goto err; + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + return ret; + + rx_ring->pp = fq.pp; + rx_ring->rx_fqes = fq.fqes; + rx_ring->truesize = fq.truesize; + rx_ring->rx_buf_len = fq.buf_len; u64_stats_init(&rx_ring->syncp); @@ -3272,25 +3159,31 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); - rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, + rx_ring->desc = dma_alloc_coherent(fq.pp->p.dev, rx_ring->size, &rx_ring->dma, GFP_KERNEL); - if (!rx_ring->desc) + if (!rx_ring->desc) { + ret = -ENOMEM; goto err; + } /* XDP RX-queue info */ - if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, 0) < 0) + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, + rx_ring->queue_index, 0, rx_ring->truesize); + if (ret) goto err; - rx_ring->xdp_prog = adapter->xdp_prog; + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, fq.pp); + + rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; err: - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); - return -ENOMEM; + return ret; } /** @@ -3331,17 +3224,24 @@ static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) **/ void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) { + struct libeth_fq fq = { + .fqes = rx_ring->rx_fqes, + .pp = rx_ring->pp, + }; + ixgbevf_clean_rx_ring(rx_ring); - rx_ring->xdp_prog = NULL; + rcu_assign_pointer(rx_ring->xdp_prog, NULL); + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; - dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, + dma_free_coherent(fq.pp->p.dev, rx_ring->size, rx_ring->desc, rx_ring->dma); - rx_ring->desc = NULL; + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; } /** @@ -4233,7 +4133,9 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) ixgbevf_open(dev); } else { for (i = 0; i < adapter->num_rx_queues; i++) - xchg(&adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); + rcu_assign_pointer(adapter->rx_ring[i]->xdp_prog, + adapter->xdp_prog); + synchronize_net(); } if (old_prog) From 93e53abfadb2fb0f1466213e5be794d85b04ee4f Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 2 Sep 2025 16:31:51 +0200 Subject: [PATCH 04/20] ixgbevf: support XDP multi-buffer on Rx path Implement XDP support for received fragmented packets, this requires using some helpers from libeth_xdp. Reviewed-by: Aleksandr Loktionov Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 3 +- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 97 +++++++------------ 2 files changed, 35 insertions(+), 65 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index ebf771f0caa4b3..2626af0393614b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "vf.h" @@ -105,7 +106,6 @@ struct ixgbevf_ring { struct xdp_rxq_info xdp_rxq; u64 hw_csum_rx_error; u8 __iomem *tail; - struct sk_buff *skb; /* holds the special value that gets the hardware register offset * associated with this ring, which is different for DCB and RSS modes @@ -113,6 +113,7 @@ struct ixgbevf_ring { u16 reg_idx; int queue_index; /* needed for multiqueue queue management */ u32 rx_buf_len; + struct libeth_xdp_buff_stash xdp_stash; } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index cedbf0a4d0a546..641d87f93864c2 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -84,6 +84,7 @@ MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); MODULE_IMPORT_NS("LIBETH"); +MODULE_IMPORT_NS("LIBETH_XDP"); MODULE_LICENSE("GPL v2"); #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) @@ -647,26 +648,6 @@ static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, return false; } -/** - * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff - * @rx_ring: rx descriptor ring to transact packets on - * @rx_buffer: buffer containing page to add - * @skb: sk_buff to place the data into - * @size: size of buffer to be added - * - * This function will add the data contained in rx_buffer->page to the skb. - **/ -static void ixgbevf_add_rx_frag(const struct libeth_fqe *rx_buffer, - struct sk_buff *skb, - unsigned int size) -{ - u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; - - skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, - rx_buffer->netmem, rx_buffer->offset + hr, - size, rx_buffer->truesize); -} - static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, u32 qmask) { @@ -803,16 +784,16 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbevf_adapter *adapter = q_vector->adapter; u16 cleaned_count = ixgbevf_desc_unused(rx_ring); - struct sk_buff *skb = rx_ring->skb; LIBETH_XDP_ONSTACK_BUFF(xdp); bool xdp_xmit = false; int xdp_res = 0; - xdp->base.rxq = &rx_ring->xdp_rxq; + libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; struct libeth_fqe *rx_buffer; + struct sk_buff *skb; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -833,43 +814,38 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, rmb(); rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; - libeth_rx_sync_for_cpu(rx_buffer, size); + libeth_xdp_process_buff(xdp, rx_buffer, size); - /* retrieve a buffer from the ring */ - if (!skb) { - libeth_xdp_prepare_buff(xdp, rx_buffer, size); - prefetch(xdp->data); - xdp_res = ixgbevf_run_xdp(adapter, rx_ring, xdp); - } + cleaned_count++; + /* fetch next buffer in frame if non-eop */ + if (ixgbevf_is_non_eop(rx_ring, rx_desc)) + continue; + + total_rx_packets++; + total_rx_bytes += xdp_get_buff_len(&xdp->base); + xdp_res = ixgbevf_run_xdp(adapter, rx_ring, xdp); if (xdp_res) { if (xdp_res == IXGBEVF_XDP_TX) xdp_xmit = true; - total_rx_packets++; - total_rx_bytes += size; - } else if (skb) { - ixgbevf_add_rx_frag(rx_buffer, skb, size); - } else { - skb = xdp_build_skb_from_buff(&xdp->base); + xdp->data = NULL; + continue; } + skb = xdp_build_skb_from_buff(&xdp->base); + /* exit if we failed to retrieve a buffer */ - if (unlikely(!xdp_res && !skb)) { + if (unlikely(!skb)) { libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; break; } - cleaned_count++; - - /* fetch next buffer in frame if non-eop */ - if (ixgbevf_is_non_eop(rx_ring, rx_desc)) - continue; + xdp->data = NULL; /* verify the packet layout is correct */ - if (xdp_res || - unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) { + if (unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } @@ -888,18 +864,11 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* populate checksum, VLAN, and protocol */ ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; - total_rx_packets++; - ixgbevf_rx_skb(q_vector, skb); - - /* reset skb pointer */ - skb = NULL; } /* place incomplete frames back on ring for completion */ - rx_ring->skb = skb; + libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); if (xdp_xmit) { struct ixgbevf_ring *xdp_ring = @@ -2019,10 +1988,7 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter) static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) { /* Free Rx ring sk_buff */ - if (rx_ring->skb) { - dev_kfree_skb(rx_ring->skb); - rx_ring->skb = NULL; - } + libeth_xdp_return_stash(&rx_ring->xdp_stash); /* Free all the Rx ring pages */ for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { @@ -4103,16 +4069,19 @@ ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) +static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) { - int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; + u32 frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbevf_adapter *adapter = netdev_priv(dev); struct bpf_prog *old_prog; + bool requires_mbuf; - /* verify ixgbevf ring attributes are sufficient for XDP */ - for (i = 0; i < adapter->num_rx_queues; i++) { - if (frame_size > IXGBEVF_RXBUFFER_3072) - return -EINVAL; + requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM); + if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { + NL_SET_ERR_MSG_MOD(extack, + "Configured MTU requires non-linear frames and XDP prog does not support frags"); + return -EOPNOTSUPP; } old_prog = xchg(&adapter->xdp_prog, prog); @@ -4132,7 +4101,7 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (netif_running(dev)) ixgbevf_open(dev); } else { - for (i = 0; i < adapter->num_rx_queues; i++) + for (int i = 0; i < adapter->num_rx_queues; i++) rcu_assign_pointer(adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); synchronize_net(); @@ -4148,7 +4117,7 @@ static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp) { switch (xdp->command) { case XDP_SETUP_PROG: - return ixgbevf_xdp_setup(dev, xdp->prog); + return ixgbevf_xdp_setup(dev, xdp->prog, xdp->extack); default: return -EINVAL; } @@ -4300,7 +4269,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; - netdev->xdp_features = NETDEV_XDP_ACT_BASIC; + netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; /* MTU range: 68 - 1504 or 9710 */ netdev->min_mtu = ETH_MIN_MTU; From 60a8d27dcfc7b9f88c9e8192c3bfdc2d70a096c0 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 9 Sep 2025 13:46:44 +0200 Subject: [PATCH 05/20] ixgbevf: XDP_TX in multi-buffer through libeth Use libeth to support XDP_TX action for segmented packets. Reviewed-by: Alexander Lobakin Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 14 +- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 217 +++++++++--------- .../ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h | 112 +++++++++ 3 files changed, 224 insertions(+), 119 deletions(-) create mode 100644 drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index 2626af0393614b..a27081ee764b40 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -81,20 +81,22 @@ struct ixgbevf_ring { struct net_device *netdev; struct bpf_prog __rcu *xdp_prog; union { - struct page_pool *pp; /* Rx ring */ + struct page_pool *pp; /* Rx and XDP rings */ struct device *dev; /* Tx ring */ }; void *desc; /* descriptor ring memory */ - dma_addr_t dma; /* phys. address of descriptor ring */ - unsigned int size; /* length in bytes */ - u32 truesize; /* Rx buffer full size */ + union { + u32 truesize; /* Rx buffer full size */ + u32 pending; /* Sent-not-completed descriptors */ + }; u16 count; /* amount of descriptors */ - u16 next_to_use; u16 next_to_clean; + u32 next_to_use; union { struct libeth_fqe *rx_fqes; struct ixgbevf_tx_buffer *tx_buffer_info; + struct libeth_sqe *xdp_sqes; }; unsigned long state; struct ixgbevf_stats stats; @@ -114,6 +116,8 @@ struct ixgbevf_ring { int queue_index; /* needed for multiqueue queue management */ u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; + unsigned int dma_size; /* length in bytes */ + dma_addr_t dma; /* phys. address of descriptor ring */ } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 641d87f93864c2..7073bbec872a2c 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -33,7 +33,7 @@ #include #include -#include "ixgbevf.h" +#include "ixgbevf_xdp_lib.h" const char ixgbevf_driver_name[] = "ixgbevf"; static const char ixgbevf_driver_string[] = @@ -306,10 +306,7 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, total_ipsec++; /* free the skb */ - if (ring_is_xdp(tx_ring)) - libeth_xdp_return_va(tx_buffer->data, true); - else - napi_consume_skb(tx_buffer->skb, napi_budget); + napi_consume_skb(tx_buffer->skb, napi_budget); /* unmap skb header data */ dma_unmap_single(tx_ring->dev, @@ -392,9 +389,8 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, eop_desc, (eop_desc ? eop_desc->wb.status : 0), tx_ring->tx_buffer_info[i].time_stamp, jiffies); - if (!ring_is_xdp(tx_ring)) - netif_stop_subqueue(tx_ring->netdev, - tx_ring->queue_index); + netif_stop_subqueue(tx_ring->netdev, + tx_ring->queue_index); /* schedule immediate reset if we believe we hung */ ixgbevf_tx_timeout_reset(adapter); @@ -402,9 +398,6 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, return true; } - if (ring_is_xdp(tx_ring)) - return !!budget; - #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { @@ -660,94 +653,85 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, #define IXGBEVF_XDP_CONSUMED 1 #define IXGBEVF_XDP_TX 2 -static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring, - struct xdp_buff *xdp) +static void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) { - struct ixgbevf_tx_buffer *tx_buffer; - union ixgbe_adv_tx_desc *tx_desc; - u32 len, cmd_type; - dma_addr_t dma; - u16 i; + ixgbevf_clean_xdp_num(xdp_ring, false, xdp_ring->pending); +} - len = xdp->data_end - xdp->data; +static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, + u64 priv) +{ + union ixgbe_adv_tx_desc *tx_desc = + &((union ixgbe_adv_tx_desc *)sq->descs)[i]; - if (unlikely(!ixgbevf_desc_unused(ring))) - return IXGBEVF_XDP_CONSUMED; + u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS | + desc.len; - dma = dma_map_single(ring->dev, xdp->data, len, DMA_TO_DEVICE); - if (dma_mapping_error(ring->dev, dma)) - return IXGBEVF_XDP_CONSUMED; + if (desc.flags & LIBETH_XDP_TX_LAST) + cmd_type |= IXGBE_TXD_CMD_EOP; - /* record the location of the first descriptor for this packet */ - i = ring->next_to_use; - tx_buffer = &ring->tx_buffer_info[i]; - - dma_unmap_len_set(tx_buffer, len, len); - dma_unmap_addr_set(tx_buffer, dma, dma); - tx_buffer->data = xdp->data; - tx_buffer->bytecount = len; - tx_buffer->gso_segs = 1; - tx_buffer->protocol = 0; - - /* Populate minimal context descriptor that will provide for the - * fact that we are expected to process Ethernet frames. - */ - if (!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state)) { - struct ixgbe_adv_tx_context_desc *context_desc; + if (desc.flags & LIBETH_XDP_TX_FIRST) { + struct skb_shared_info *sinfo = sq->sqes[i].sinfo; + u16 full_len = desc.len; - set_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); + if (desc.flags & LIBETH_XDP_TX_MULTI) + full_len += sinfo->xdp_frags_size; - context_desc = IXGBEVF_TX_CTXTDESC(ring, 0); - context_desc->vlan_macip_lens = - cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT); - context_desc->fceof_saidx = 0; - context_desc->type_tucmd_mlhl = - cpu_to_le32(IXGBE_TXD_CMD_DEXT | - IXGBE_ADVTXD_DTYP_CTXT); - context_desc->mss_l4len_idx = 0; - - i = 1; + tx_desc->read.olinfo_status = + cpu_to_le32((full_len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); } - /* put descriptor type bits */ - cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS; - cmd_type |= len | IXGBE_TXD_CMD; + tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); +} - tx_desc = IXGBEVF_TX_DESC(ring, i); - tx_desc->read.buffer_addr = cpu_to_le64(dma); +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xdp_flush_tx, ixgbevf_prep_xdp_sq, + ixgbevf_xdp_xmit_desc); +LIBETH_XDP_DEFINE_END(); - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - tx_desc->read.olinfo_status = - cpu_to_le32((len << IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); +static void ixgbevf_xdp_set_rs(struct ixgbevf_ring *xdp_ring, u32 cached_ntu) +{ + u32 ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; + union ixgbe_adv_tx_desc *desc; - /* Avoid any potential race with cleanup */ - smp_wmb(); + desc = IXGBEVF_TX_DESC(xdp_ring, ltu); + xdp_ring->xdp_sqes[cached_ntu].rs_idx = ltu + 1; + desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); +} - /* set next_to_watch value indicating a packet is present */ - i++; - if (i == ring->count) - i = 0; +static void ixgbevf_rx_finalize_xdp(struct libeth_xdp_tx_bulk *tx_bulk, + bool xdp_xmit, u32 cached_ntu) +{ + struct ixgbevf_ring *xdp_ring = tx_bulk->xdpsq; + + if (!xdp_xmit) + goto unlock; + + if (tx_bulk->count) + ixgbevf_xdp_flush_tx(tx_bulk, LIBETH_XDP_TX_DROP); - tx_buffer->next_to_watch = tx_desc; - ring->next_to_use = i; + ixgbevf_xdp_set_rs(xdp_ring, cached_ntu); - return IXGBEVF_XDP_TX; + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); +unlock: + rcu_read_unlock(); } -static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *rx_ring, +static int ixgbevf_run_xdp(struct libeth_xdp_tx_bulk *tx_bulk, struct libeth_xdp_buff *xdp) { int result = IXGBEVF_XDP_PASS; - struct ixgbevf_ring *xdp_ring; - struct bpf_prog *xdp_prog; + const struct bpf_prog *xdp_prog; u32 act; - xdp_prog = rcu_dereference(rx_ring->xdp_prog); - + xdp_prog = tx_bulk->prog; if (!xdp_prog) goto xdp_out; @@ -756,17 +740,16 @@ static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, case XDP_PASS: break; case XDP_TX: - xdp_ring = adapter->xdp_ring[rx_ring->queue_index]; - result = ixgbevf_xmit_xdp_ring(xdp_ring, &xdp->base); - if (result == IXGBEVF_XDP_CONSUMED) - goto out_failure; + result = IXGBEVF_XDP_TX; + if (!libeth_xdp_tx_queue_bulk(tx_bulk, xdp, + ixgbevf_xdp_flush_tx)) + result = IXGBEVF_XDP_CONSUMED; break; default: - bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); + bpf_warn_invalid_xdp_action(tx_bulk->dev, xdp_prog, act); fallthrough; case XDP_ABORTED: -out_failure: - trace_xdp_exception(rx_ring->netdev, xdp_prog, act); + trace_xdp_exception(tx_bulk->dev, xdp_prog, act); fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: result = IXGBEVF_XDP_CONSUMED; @@ -784,11 +767,19 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbevf_adapter *adapter = q_vector->adapter; u16 cleaned_count = ixgbevf_desc_unused(rx_ring); + LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); LIBETH_XDP_ONSTACK_BUFF(xdp); + u32 cached_ntu; bool xdp_xmit = false; int xdp_res = 0; libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); + libeth_xdp_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, + adapter->netdev, adapter->xdp_ring, + adapter->num_xdp_queues); + if (xdp_tx_bulk.prog) + cached_ntu = + ((struct ixgbevf_ring *)xdp_tx_bulk.xdpsq)->next_to_use; while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; @@ -824,7 +815,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, total_rx_packets++; total_rx_bytes += xdp_get_buff_len(&xdp->base); - xdp_res = ixgbevf_run_xdp(adapter, rx_ring, xdp); + xdp_res = ixgbevf_run_xdp(&xdp_tx_bulk, xdp); if (xdp_res) { if (xdp_res == IXGBEVF_XDP_TX) xdp_xmit = true; @@ -870,16 +861,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* place incomplete frames back on ring for completion */ libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); - if (xdp_xmit) { - struct ixgbevf_ring *xdp_ring = - adapter->xdp_ring[rx_ring->queue_index]; - - /* Force memory writes to complete before letting h/w - * know there are new descriptors to fetch. - */ - wmb(); - ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); - } + ixgbevf_rx_finalize_xdp(&xdp_tx_bulk, xdp_xmit, cached_ntu); u64_stats_update_begin(&rx_ring->syncp); rx_ring->stats.packets += total_rx_packets; @@ -909,6 +891,8 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) bool clean_complete = true; ixgbevf_for_each_ring(ring, q_vector->tx) { + if (ring_is_xdp(ring)) + continue; if (!ixgbevf_clean_tx_irq(q_vector, ring, budget)) clean_complete = false; } @@ -1348,6 +1332,7 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, /* reset ntu and ntc to place SW in sync with hardwdare */ ring->next_to_clean = 0; ring->next_to_use = 0; + ring->pending = 0; /* In order to avoid issues WTHRESH + PTHRESH should always be equal * to or less than the number of on chip descriptors, which is @@ -1360,8 +1345,12 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 32; /* PTHRESH = 32 */ /* reinitialize tx_buffer_info */ - memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbevf_tx_buffer) * ring->count); + if (!ring_is_xdp(ring)) + memset(ring->tx_buffer_info, 0, + sizeof(struct ixgbevf_tx_buffer) * ring->count); + else + memset(ring->xdp_sqes, 0, + sizeof(struct libeth_sqe) * ring->count); clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); @@ -2016,10 +2005,7 @@ static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) union ixgbe_adv_tx_desc *eop_desc, *tx_desc; /* Free all the Tx ring sk_buffs */ - if (ring_is_xdp(tx_ring)) - libeth_xdp_return_va(tx_buffer->data, false); - else - dev_kfree_skb_any(tx_buffer->skb); + dev_kfree_skb_any(tx_buffer->skb); /* unmap skb header data */ dma_unmap_single(tx_ring->dev, @@ -2088,7 +2074,7 @@ static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) for (i = 0; i < adapter->num_tx_queues; i++) ixgbevf_clean_tx_ring(adapter->tx_ring[i]); for (i = 0; i < adapter->num_xdp_queues; i++) - ixgbevf_clean_tx_ring(adapter->xdp_ring[i]); + ixgbevf_clean_xdp_ring(adapter->xdp_ring[i]); } void ixgbevf_down(struct ixgbevf_adapter *adapter) @@ -2834,8 +2820,6 @@ static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) if (netif_carrier_ok(adapter->netdev)) { for (i = 0; i < adapter->num_tx_queues; i++) set_check_for_tx_hang(adapter->tx_ring[i]); - for (i = 0; i < adapter->num_xdp_queues; i++) - set_check_for_tx_hang(adapter->xdp_ring[i]); } /* get one bit for every active Tx/Rx interrupt vector */ @@ -2979,7 +2963,10 @@ static void ixgbevf_service_task(struct work_struct *work) **/ void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) { - ixgbevf_clean_tx_ring(tx_ring); + if (!ring_is_xdp(tx_ring)) + ixgbevf_clean_tx_ring(tx_ring); + else + ixgbevf_clean_xdp_ring(tx_ring); vfree(tx_ring->tx_buffer_info); tx_ring->tx_buffer_info = NULL; @@ -2988,7 +2975,7 @@ void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) if (!tx_ring->desc) return; - dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, + dma_free_coherent(tx_ring->dev, tx_ring->dma_size, tx_ring->desc, tx_ring->dma); tx_ring->desc = NULL; @@ -3023,7 +3010,9 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); int size; - size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; + size = (!ring_is_xdp(tx_ring) ? sizeof(struct ixgbevf_tx_buffer) : + sizeof(struct libeth_sqe)) * tx_ring->count; + tx_ring->tx_buffer_info = vmalloc(size); if (!tx_ring->tx_buffer_info) goto err; @@ -3031,10 +3020,10 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) u64_stats_init(&tx_ring->syncp); /* round up to nearest 4K */ - tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); - tx_ring->size = ALIGN(tx_ring->size, 4096); + tx_ring->dma_size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); + tx_ring->dma_size = ALIGN(tx_ring->dma_size, 4096); - tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, + tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->dma_size, &tx_ring->dma, GFP_KERNEL); if (!tx_ring->desc) goto err; @@ -3122,10 +3111,10 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, u64_stats_init(&rx_ring->syncp); /* Round up to nearest 4K */ - rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); - rx_ring->size = ALIGN(rx_ring->size, 4096); + rx_ring->dma_size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); + rx_ring->dma_size = ALIGN(rx_ring->dma_size, 4096); - rx_ring->desc = dma_alloc_coherent(fq.pp->p.dev, rx_ring->size, + rx_ring->desc = dma_alloc_coherent(fq.pp->p.dev, rx_ring->dma_size, &rx_ring->dma, GFP_KERNEL); if (!rx_ring->desc) { @@ -3201,7 +3190,7 @@ void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - dma_free_coherent(fq.pp->p.dev, rx_ring->size, rx_ring->desc, + dma_free_coherent(fq.pp->p.dev, rx_ring->dma_size, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h new file mode 100644 index 00000000000000..629a5943eb3e27 --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2010-2026 Intel Corporation */ + +#ifndef _IXGBEVF_XDP_LIB_H_ +#define _IXGBEVF_XDP_LIB_H_ + +#include + +#include "ixgbevf.h" + +static inline u16 ixgbevf_tx_get_num_sent(struct ixgbevf_ring *xdp_ring) +{ + u16 ntc = xdp_ring->next_to_clean; + u16 to_clean = 0; + + while (likely(to_clean < xdp_ring->pending)) { + u32 idx = xdp_ring->xdp_sqes[ntc].rs_idx; + union ixgbe_adv_tx_desc *rs_desc; + + if (!idx--) + break; + + rs_desc = IXGBEVF_TX_DESC(xdp_ring, idx); + + if (!(rs_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) + break; + + xdp_ring->xdp_sqes[ntc].rs_idx = 0; + + to_clean += + (idx >= ntc ? idx : idx + xdp_ring->count) - ntc + 1; + + ntc = (idx + 1 == xdp_ring->count) ? 0 : idx + 1; + } + + return to_clean; +} + +static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, + bool in_napi, u16 to_clean) +{ + struct libeth_xdpsq_napi_stats stats = { }; + u32 ntc = xdp_ring->next_to_clean; + struct xdp_frame_bulk cbulk; + struct libeth_cq_pp cp = { + .bq = &cbulk, + .dev = xdp_ring->dev, + .xss = &stats, + .napi = in_napi, + }; + + xdp_frame_bulk_init(&cbulk); + xdp_ring->pending -= to_clean; + + while (likely(to_clean--)) { + libeth_xdp_complete_tx(&xdp_ring->xdp_sqes[ntc], &cp); + ntc++; + ntc = unlikely(ntc == xdp_ring->count) ? 0 : ntc; + } + + xdp_ring->next_to_clean = ntc; + xdp_flush_frame_bulk(&cbulk); +} + +static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) +{ + struct ixgbevf_ring *xdp_ring = xdpsq; + + if (unlikely(ixgbevf_desc_unused(xdp_ring) < LIBETH_XDP_TX_BULK)) { + u16 to_clean = ixgbevf_tx_get_num_sent(xdp_ring); + + if (likely(to_clean)) + ixgbevf_clean_xdp_num(xdp_ring, true, to_clean); + } + + if (unlikely(!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, + &xdp_ring->state))) { + struct ixgbe_adv_tx_context_desc *context_desc; + + set_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &xdp_ring->state); + + context_desc = IXGBEVF_TX_CTXTDESC(xdp_ring, 0); + context_desc->vlan_macip_lens = + cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT); + context_desc->fceof_saidx = 0; + context_desc->type_tucmd_mlhl = + cpu_to_le32(IXGBE_TXD_CMD_DEXT | + IXGBE_ADVTXD_DTYP_CTXT); + context_desc->mss_l4len_idx = 0; + + xdp_ring->next_to_use = 1; + xdp_ring->pending = 1; + + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(xdp_ring, 1); + } + + *sq = (struct libeth_xdpsq) { + .count = xdp_ring->count, + .descs = xdp_ring->desc, + .lock = NULL, + .ntu = &xdp_ring->next_to_use, + .pending = &xdp_ring->pending, + .pool = NULL, + .sqes = xdp_ring->xdp_sqes, + }; + + return ixgbevf_desc_unused(xdp_ring); +} + +#endif /* _IXGBEVF_XDP_LIB_H_ */ From 57735ac5314255e03e611e6c9d6043b5e5da4385 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 22 Sep 2025 07:14:24 +0200 Subject: [PATCH 06/20] ixgbevf: support XDP_REDIRECT and .ndo_xdp_xmit To fully support XDP_REDIRECT, utilize more libeth helpers in XDP Rx path, hence save cached_ntu in the ring structure instead of stack. ixgbevf-supported VFs usually have few queues, so use libeth_xdpsq_lock functionality for XDP queue sharing. Adjust filling-in of XDP Tx descriptors to use data from xdp frame. Otherwise, simply use libeth helpers to implement .ndo_xdp_xmit(). While at it, fix a typo in libeth docs. Reviewed-by: Aleksandr Loktionov Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 2 + .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 135 ++++++------------ .../ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h | 30 +++- include/net/libeth/xdp.h | 2 +- 4 files changed, 78 insertions(+), 91 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index a27081ee764b40..ea86679e4f81d0 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -98,6 +98,8 @@ struct ixgbevf_ring { struct ixgbevf_tx_buffer *tx_buffer_info; struct libeth_sqe *xdp_sqes; }; + struct libeth_xdpsq_lock xdpq_lock; + u32 cached_ntu; unsigned long state; struct ixgbevf_stats stats; struct u64_stats_sync syncp; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 7073bbec872a2c..f26528cadb981f 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -649,13 +649,10 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); } -#define IXGBEVF_XDP_PASS 0 -#define IXGBEVF_XDP_CONSUMED 1 -#define IXGBEVF_XDP_TX 2 - static void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) { ixgbevf_clean_xdp_num(xdp_ring, false, xdp_ring->pending); + libeth_xdpsq_put(&xdp_ring->xdpq_lock, xdp_ring->netdev); } static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, @@ -674,11 +671,16 @@ static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, cmd_type |= IXGBE_TXD_CMD_EOP; if (desc.flags & LIBETH_XDP_TX_FIRST) { - struct skb_shared_info *sinfo = sq->sqes[i].sinfo; + struct libeth_sqe *sqe = &sq->sqes[i]; + struct skb_shared_info *sinfo; u16 full_len = desc.len; - if (desc.flags & LIBETH_XDP_TX_MULTI) + if (desc.flags & LIBETH_XDP_TX_MULTI) { + sinfo = sqe->type == LIBETH_SQE_XDP_TX ? + sqe->sinfo : + xdp_get_shared_info_from_frame(sqe->xdpf); full_len += sinfo->xdp_frags_size; + } tx_desc->read.olinfo_status = cpu_to_le32((full_len << IXGBE_ADVTXD_PAYLEN_SHIFT) | @@ -692,74 +694,13 @@ static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, LIBETH_XDP_DEFINE_START(); LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xdp_flush_tx, ixgbevf_prep_xdp_sq, ixgbevf_xdp_xmit_desc); +LIBETH_XDP_DEFINE_FLUSH_XMIT(static ixgbevf_xdp_flush_xmit, ixgbevf_prep_xdp_sq, + ixgbevf_xdp_xmit_desc); +LIBETH_XDP_DEFINE_RUN_PROG(static ixgbevf_xdp_run_prog, ixgbevf_xdp_flush_tx); +LIBETH_XDP_DEFINE_FINALIZE(static ixgbevf_xdp_finalize_xdp_napi, + ixgbevf_xdp_flush_tx, ixgbevf_xdp_rs_and_bump); LIBETH_XDP_DEFINE_END(); -static void ixgbevf_xdp_set_rs(struct ixgbevf_ring *xdp_ring, u32 cached_ntu) -{ - u32 ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; - union ixgbe_adv_tx_desc *desc; - - desc = IXGBEVF_TX_DESC(xdp_ring, ltu); - xdp_ring->xdp_sqes[cached_ntu].rs_idx = ltu + 1; - desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); -} - -static void ixgbevf_rx_finalize_xdp(struct libeth_xdp_tx_bulk *tx_bulk, - bool xdp_xmit, u32 cached_ntu) -{ - struct ixgbevf_ring *xdp_ring = tx_bulk->xdpsq; - - if (!xdp_xmit) - goto unlock; - - if (tx_bulk->count) - ixgbevf_xdp_flush_tx(tx_bulk, LIBETH_XDP_TX_DROP); - - ixgbevf_xdp_set_rs(xdp_ring, cached_ntu); - - /* Finish descriptor writes before bumping tail */ - wmb(); - ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); -unlock: - rcu_read_unlock(); -} - -static int ixgbevf_run_xdp(struct libeth_xdp_tx_bulk *tx_bulk, - struct libeth_xdp_buff *xdp) -{ - int result = IXGBEVF_XDP_PASS; - const struct bpf_prog *xdp_prog; - u32 act; - - xdp_prog = tx_bulk->prog; - if (!xdp_prog) - goto xdp_out; - - act = bpf_prog_run_xdp(xdp_prog, &xdp->base); - switch (act) { - case XDP_PASS: - break; - case XDP_TX: - result = IXGBEVF_XDP_TX; - if (!libeth_xdp_tx_queue_bulk(tx_bulk, xdp, - ixgbevf_xdp_flush_tx)) - result = IXGBEVF_XDP_CONSUMED; - break; - default: - bpf_warn_invalid_xdp_action(tx_bulk->dev, xdp_prog, act); - fallthrough; - case XDP_ABORTED: - trace_xdp_exception(tx_bulk->dev, xdp_prog, act); - fallthrough; /* handle aborts by dropping packet */ - case XDP_DROP: - result = IXGBEVF_XDP_CONSUMED; - libeth_xdp_return_buff(xdp); - break; - } -xdp_out: - return result; -} - static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *rx_ring, int budget) @@ -769,17 +710,11 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, u16 cleaned_count = ixgbevf_desc_unused(rx_ring); LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); LIBETH_XDP_ONSTACK_BUFF(xdp); - u32 cached_ntu; - bool xdp_xmit = false; - int xdp_res = 0; libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); libeth_xdp_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, adapter->netdev, adapter->xdp_ring, adapter->num_xdp_queues); - if (xdp_tx_bulk.prog) - cached_ntu = - ((struct ixgbevf_ring *)xdp_tx_bulk.xdpsq)->next_to_use; while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; @@ -815,14 +750,9 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, total_rx_packets++; total_rx_bytes += xdp_get_buff_len(&xdp->base); - xdp_res = ixgbevf_run_xdp(&xdp_tx_bulk, xdp); - if (xdp_res) { - if (xdp_res == IXGBEVF_XDP_TX) - xdp_xmit = true; - - xdp->data = NULL; + if (xdp_tx_bulk.prog && + !ixgbevf_xdp_run_prog(xdp, &xdp_tx_bulk)) continue; - } skb = xdp_build_skb_from_buff(&xdp->base); @@ -861,7 +791,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* place incomplete frames back on ring for completion */ libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); - ixgbevf_rx_finalize_xdp(&xdp_tx_bulk, xdp_xmit, cached_ntu); + ixgbevf_xdp_finalize_xdp_napi(&xdp_tx_bulk); u64_stats_update_begin(&rx_ring->syncp); rx_ring->stats.packets += total_rx_packets; @@ -873,6 +803,23 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, return total_rx_packets; } +static int ixgbevf_xdp_xmit(struct net_device *dev, int n, + struct xdp_frame **frames, u32 flags) +{ + struct ixgbevf_adapter *adapter = netdev_priv(dev); + + if (unlikely(test_bit(__IXGBEVF_DOWN, &adapter->state))) + return -ENETDOWN; + + if (unlikely(!adapter->num_xdp_queues)) + return -ENXIO; + + return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, adapter->xdp_ring, + adapter->num_xdp_queues, + ixgbevf_xdp_flush_xmit, + ixgbevf_xdp_rs_and_bump); +} + /** * ixgbevf_poll - NAPI polling calback * @napi: napi struct with our devices info in it @@ -1333,6 +1280,7 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, ring->next_to_clean = 0; ring->next_to_use = 0; ring->pending = 0; + ring->cached_ntu = 0; /* In order to avoid issues WTHRESH + PTHRESH should always be equal * to or less than the number of on chip descriptors, which is @@ -1345,12 +1293,15 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 32; /* PTHRESH = 32 */ /* reinitialize tx_buffer_info */ - if (!ring_is_xdp(ring)) + if (!ring_is_xdp(ring)) { memset(ring->tx_buffer_info, 0, sizeof(struct ixgbevf_tx_buffer) * ring->count); - else + } else { memset(ring->xdp_sqes, 0, sizeof(struct libeth_sqe) * ring->count); + libeth_xdpsq_get(&ring->xdpq_lock, ring->netdev, + num_possible_cpus() > adapter->num_xdp_queues); + } clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); @@ -4077,6 +4028,8 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, /* If transitioning XDP modes reconfigure rings */ if (!!prog != !!old_prog) { + xdp_features_clear_redirect_target(dev); + /* Hardware has to reinitialize queues and interrupts to * match packet buffer alignment. Unfortunately, the * hardware is not flexible enough to do this dynamically. @@ -4096,6 +4049,9 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, synchronize_net(); } + if (prog) + xdp_features_set_redirect_target(dev, true); + if (old_prog) bpf_prog_put(old_prog); @@ -4126,6 +4082,7 @@ static const struct net_device_ops ixgbevf_netdev_ops = { .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, .ndo_features_check = ixgbevf_features_check, .ndo_bpf = ixgbevf_xdp, + .ndo_xdp_xmit = ixgbevf_xdp_xmit, }; static void ixgbevf_assign_netdev_ops(struct net_device *dev) @@ -4258,7 +4215,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; - netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; + libeth_xdp_set_features_noredir(netdev, NULL, 0, NULL); /* MTU range: 68 - 1504 or 9710 */ netdev->min_mtu = ETH_MIN_MTU; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h index 629a5943eb3e27..da4f397944ed88 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -66,6 +66,7 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) { struct ixgbevf_ring *xdp_ring = xdpsq; + libeth_xdpsq_lock(&xdp_ring->xdpq_lock); if (unlikely(ixgbevf_desc_unused(xdp_ring) < LIBETH_XDP_TX_BULK)) { u16 to_clean = ixgbevf_tx_get_num_sent(xdp_ring); @@ -99,7 +100,7 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) *sq = (struct libeth_xdpsq) { .count = xdp_ring->count, .descs = xdp_ring->desc, - .lock = NULL, + .lock = &xdp_ring->xdpq_lock, .ntu = &xdp_ring->next_to_use, .pending = &xdp_ring->pending, .pool = NULL, @@ -109,4 +110,31 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) return ixgbevf_desc_unused(xdp_ring); } +static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) +{ + struct ixgbevf_ring *xdp_ring = xdpsq; + union ixgbe_adv_tx_desc *desc; + u32 ltu; + + libeth_xdpsq_lock(&xdp_ring->xdpq_lock); + + if ((!flush && xdp_ring->pending < xdp_ring->count - 1) || + xdp_ring->cached_ntu == xdp_ring->next_to_use) + goto unlock; + + ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; + desc = IXGBEVF_TX_DESC(xdp_ring, ltu); + desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); + + xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1; + xdp_ring->cached_ntu = xdp_ring->next_to_use; + + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); + +unlock: + libeth_xdpsq_unlock(&xdp_ring->xdpq_lock); +} + #endif /* _IXGBEVF_XDP_LIB_H_ */ diff --git a/include/net/libeth/xdp.h b/include/net/libeth/xdp.h index 898723ab62e812..2e2154ccecae61 100644 --- a/include/net/libeth/xdp.h +++ b/include/net/libeth/xdp.h @@ -1094,7 +1094,7 @@ __libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq, * @xqs: array of XDPSQs driver structs * @nqs: number of active XDPSQs, the above array length * @fl: driver callback to flush an XDP xmit bulk - * @fin: driver cabback to finalize the queue + * @fin: driver callback to finalize the queue * * If the driver has active XDPSQs, perform common checks and send the frames. * Finalize the queue, if requested. From d6ec774def48d3557103786a73c1cd306e0a277a Mon Sep 17 00:00:00 2001 From: Natalia Wochtman Date: Fri, 19 Sep 2025 15:30:15 +0200 Subject: [PATCH 07/20] ixgbevf: add pseudo header split Introduce pseudo header split support in the ixgbevf driver, specifically targeting ixgbe_mac_82599_vf. On older hardware (e.g. ixgbe_mac_82599_vf), RX DMA write size can only be limited in 1K increments. This causes issues when attempting to fit multiple packets per page, as a DMA write may overwrite the headroom of the next packet. To address this, introduce pseudo header split support, where the hardware copies the full L2 header into a dedicated header buffer. This avoids the need for HR/TR alignment and allows safe skb construction from the header buffer without risking overwrites. Given that once packet is too big to fit into a single page, the behaviour is the same for all supported HW, use pseudo header split only for smaller packets. Signed-off-by: Natalia Wochtman Reviewed-by: Aleksandr Loktionov Co-developed-by: Larysa Zaremba Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 7 + .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 181 +++++++++++++++--- 2 files changed, 164 insertions(+), 24 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index ea86679e4f81d0..17958cfb4ee65b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -89,6 +89,7 @@ struct ixgbevf_ring { u32 truesize; /* Rx buffer full size */ u32 pending; /* Sent-not-completed descriptors */ }; + u32 hdr_truesize; /* Rx header buffer full size */ u16 count; /* amount of descriptors */ u16 next_to_clean; u32 next_to_use; @@ -107,6 +108,8 @@ struct ixgbevf_ring { struct ixgbevf_tx_queue_stats tx_stats; struct ixgbevf_rx_queue_stats rx_stats; }; + struct libeth_fqe *hdr_fqes; + struct page_pool *hdr_pp; struct xdp_rxq_info xdp_rxq; u64 hw_csum_rx_error; u8 __iomem *tail; @@ -151,6 +154,8 @@ struct ixgbevf_ring { #define IXGBEVF_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ IXGBE_SRRCTL_BSIZEPKT_STEP)) +#define IXGBEVF_RX_SRRCTL_BUF_SIZE(mtu) (ALIGN((mtu) + LIBETH_RX_LL_LEN, \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) #define IXGBE_TX_FLAGS_CSUM BIT(0) #define IXGBE_TX_FLAGS_VLAN BIT(1) @@ -349,6 +354,8 @@ enum ixbgevf_state_t { __IXGBEVF_QUEUE_RESET_REQUESTED, }; +#define IXGBEVF_FLAG_HSPLIT BIT(0) + enum ixgbevf_boards { board_82599_vf, board_82599_vf_hv, diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index f26528cadb981f..7080ea5512c751 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -561,6 +561,12 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, .truesize = rx_ring->truesize, .count = rx_ring->count, }; + const struct libeth_fq_fp hdr_fq = { + .pp = rx_ring->hdr_pp, + .fqes = rx_ring->hdr_fqes, + .truesize = rx_ring->hdr_truesize, + .count = rx_ring->count, + }; u16 ntu = rx_ring->next_to_use; /* nothing to do or no valid netdev defined */ @@ -578,6 +584,14 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, rx_desc->read.pkt_addr = cpu_to_le64(addr); + if (hdr_fq.pp) { + addr = libeth_rx_alloc(&hdr_fq, ntu); + if (addr == DMA_MAPPING_ERROR) { + libeth_rx_recycle_slow(fq.fqes[ntu].netmem); + break; + } + } + rx_desc++; ntu++; if (unlikely(ntu == fq.count)) { @@ -701,6 +715,32 @@ LIBETH_XDP_DEFINE_FINALIZE(static ixgbevf_xdp_finalize_xdp_napi, ixgbevf_xdp_flush_tx, ixgbevf_xdp_rs_and_bump); LIBETH_XDP_DEFINE_END(); +static u32 ixgbevf_rx_hsplit_wa(const struct libeth_fqe *hdr, + struct libeth_fqe *buf, u32 data_len) +{ + u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN; + struct page *hdr_page, *buf_page; + const void *src; + void *dst; + + if (unlikely(netmem_is_net_iov(buf->netmem)) || + !libeth_rx_sync_for_cpu(buf, copy)) + return 0; + + hdr_page = __netmem_to_page(hdr->netmem); + buf_page = __netmem_to_page(buf->netmem); + + dst = page_address(hdr_page) + hdr->offset + + pp_page_to_nmdesc(hdr_page)->pp->p.offset; + src = page_address(buf_page) + buf->offset + + pp_page_to_nmdesc(buf_page)->pp->p.offset; + + memcpy(dst, src, LARGEST_ALIGN(copy)); + buf->offset += copy; + + return copy; +} + static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *rx_ring, int budget) @@ -740,6 +780,23 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, rmb(); rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; + + if (unlikely(rx_ring->hdr_pp)) { + struct libeth_fqe *hdr_buff; + unsigned int hdr_size = 0; + + hdr_buff = &rx_ring->hdr_fqes[rx_ring->next_to_clean]; + + if (!xdp->data) { + hdr_size = ixgbevf_rx_hsplit_wa(hdr_buff, + rx_buffer, + size); + size -= hdr_size ? : size; + } + + libeth_xdp_process_buff(xdp, hdr_buff, hdr_size); + } + libeth_xdp_process_buff(xdp, rx_buffer, size); cleaned_count++; @@ -1476,6 +1533,87 @@ static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); } +static void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) +{ + struct libeth_fq fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + }; + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; + + if (!rx_ring->hdr_pp) + return; + + fq = (struct libeth_fq) { + .pp = rx_ring->hdr_pp, + .fqes = rx_ring->hdr_fqes, + }; + + libeth_rx_fq_destroy(&fq); + rx_ring->hdr_fqes = NULL; + rx_ring->hdr_pp = NULL; +} + +static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) +{ + u32 adapter_flags = rx_ring->q_vector->adapter->flags; + struct libeth_fq fq = { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_MTU, + .xdp = !!rx_ring->xdp_prog, + .buf_len = IXGBEVF_RX_PAGE_LEN(rx_ring->xdp_prog ? + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), + }; + u32 frame_size; + int ret; + + /* Some HW requires DMA write sizes to be aligned to 1K, + * which warrants fake header split usage, but this is + * not an issue if the frame size is at its maximum of 3K + */ + frame_size = + IXGBEVF_RX_SRRCTL_BUF_SIZE(READ_ONCE(rx_ring->netdev->mtu)); + fq.hsplit = (adapter_flags & IXGBEVF_FLAG_HSPLIT) && + frame_size < fq.buf_len; + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + return ret; + + rx_ring->pp = fq.pp; + rx_ring->rx_fqes = fq.fqes; + rx_ring->truesize = fq.truesize; + rx_ring->rx_buf_len = fq.buf_len; + + if (!fq.hsplit) + return 0; + + fq = (struct libeth_fq) { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_HDR, + .xdp = !!rx_ring->xdp_prog, + }; + + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + goto err; + + rx_ring->hdr_pp = fq.pp; + rx_ring->hdr_fqes = fq.fqes; + rx_ring->hdr_truesize = fq.truesize; + + return 0; + +err: + ixgbevf_rx_destroy_pp(rx_ring); + return ret; +} + static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *ring) { @@ -1933,8 +2071,13 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) /* Free all the Rx ring pages */ for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; + const struct libeth_fqe *hdr_fqe = rx_ring->hdr_fqes ? + &rx_ring->hdr_fqes[i] : + NULL; libeth_rx_recycle_slow(rx_fqe->netmem); + if (hdr_fqe) + libeth_rx_recycle_slow(hdr_fqe->netmem); if (unlikely(++i == rx_ring->count)) i = 0; } @@ -2596,6 +2739,9 @@ static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) goto out; } + if (adapter->hw.mac.type == ixgbe_mac_82599_vf) + adapter->flags |= IXGBEVF_FLAG_HSPLIT; + /* assume legacy case in which PF would only give VF 2 queues */ hw->mac.max_tx_queues = 2; hw->mac.max_rx_queues = 2; @@ -3030,42 +3176,29 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) } /** - * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) + * ixgbevf_setup_rx_resources - allocate Rx resources * @adapter: board private structure * @rx_ring: Rx descriptor ring (for a specific queue) to setup * - * Returns 0 on success, negative on failure + * Returns: 0 on success, negative on failure. **/ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring) { - struct libeth_fq fq = { - .count = rx_ring->count, - .nid = NUMA_NO_NODE, - .type = LIBETH_FQE_MTU, - .xdp = !!rx_ring->xdp_prog, - .buf_len = IXGBEVF_RX_PAGE_LEN(rx_ring->xdp_prog ? - LIBETH_XDP_HEADROOM : - LIBETH_SKB_HEADROOM), - }; int ret; - ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + ret = ixgbevf_rx_create_pp(rx_ring); if (ret) return ret; - rx_ring->pp = fq.pp; - rx_ring->rx_fqes = fq.fqes; - rx_ring->truesize = fq.truesize; - rx_ring->rx_buf_len = fq.buf_len; - u64_stats_init(&rx_ring->syncp); /* Round up to nearest 4K */ rx_ring->dma_size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->dma_size = ALIGN(rx_ring->dma_size, 4096); - rx_ring->desc = dma_alloc_coherent(fq.pp->p.dev, rx_ring->dma_size, + rx_ring->desc = dma_alloc_coherent(rx_ring->pp->p.dev, + rx_ring->dma_size, &rx_ring->dma, GFP_KERNEL); if (!rx_ring->desc) { @@ -3079,16 +3212,15 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, if (ret) goto err; - xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, fq.pp); + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; err: - libeth_rx_fq_destroy(&fq); - rx_ring->rx_fqes = NULL; - rx_ring->pp = NULL; + ixgbevf_rx_destroy_pp(rx_ring); dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); + return ret; } @@ -4017,10 +4149,11 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, struct bpf_prog *old_prog; bool requires_mbuf; - requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM); + requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM) || + adapter->flags & IXGBEVF_FLAG_HSPLIT; if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { NL_SET_ERR_MSG_MOD(extack, - "Configured MTU requires non-linear frames and XDP prog does not support frags"); + "Configured MTU or HW limitations require non-linear frames and XDP prog does not support frags"); return -EOPNOTSUPP; } From 57d29b337f36d5e91f6e4139fdc6674967cc9154 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 25 Nov 2025 16:31:14 +0100 Subject: [PATCH 08/20] ixgbevf: reconfigure page pool when reallocating buffers Currently, when MTU is changed, page pool is not reconfigured, which leads to usage of suboptimal buffer sizes. Always destroy page pool when cleaning the ring up and create it anew when we first allocate Rx buffers. Reviewed-by: Aleksandr Loktionov Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 80 ++++++++++--------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 7080ea5512c751..dcb3ebdedc6b2c 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -569,8 +569,8 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, }; u16 ntu = rx_ring->next_to_use; - /* nothing to do or no valid netdev defined */ - if (unlikely(!cleaned_count || !rx_ring->netdev)) + /* nothing to do or page pool is not present */ + if (unlikely(!cleaned_count || !fq.pp)) return; rx_desc = IXGBEVF_RX_DESC(rx_ring, ntu); @@ -1540,6 +1540,14 @@ static void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) .fqes = rx_ring->rx_fqes, }; + if (!fq.pp) + return; + + if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) { + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + } + libeth_rx_fq_destroy(&fq); rx_ring->rx_fqes = NULL; rx_ring->pp = NULL; @@ -1589,6 +1597,14 @@ static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) rx_ring->truesize = fq.truesize; rx_ring->rx_buf_len = fq.buf_len; + /* XDP RX-queue info */ + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index, 0, rx_ring->truesize); + if (ret) + goto err; + + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); + if (!fq.hsplit) return 0; @@ -1623,6 +1639,7 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, bool rlpml_valid = false; u64 rdba = ring->dma; u32 rxdctl; + int err; /* disable queue to avoid issues while updating state */ rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); @@ -1656,6 +1673,14 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, ring->next_to_clean = 0; ring->next_to_use = 0; + err = ixgbevf_rx_create_pp(ring); + if (err) { + netdev_err(ring->netdev, + "Failed to create Page Pool for buffer allocation: (%pe), RxQ %d is disabled, driver reload may be needed\n", + ERR_PTR(err), ring->queue_index); + return; + } + /* RXDCTL.RLPML does not work on 82599 */ if (adapter->hw.mac.type != ixgbe_mac_82599_vf) { u32 pkt_len = @@ -2153,8 +2178,10 @@ static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) { int i; - for (i = 0; i < adapter->num_rx_queues; i++) + for (i = 0; i < adapter->num_rx_queues; i++) { ixgbevf_clean_rx_ring(adapter->rx_ring[i]); + ixgbevf_rx_destroy_pp(adapter->rx_ring[i]); + } } /** @@ -3175,6 +3202,11 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) return err; } +static struct device *ixgbevf_dma_dev_from_ring(struct ixgbevf_ring *ring) +{ + return &ring->q_vector->adapter->pdev->dev; +} + /** * ixgbevf_setup_rx_resources - allocate Rx resources * @adapter: board private structure @@ -3185,43 +3217,25 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring) { - int ret; - - ret = ixgbevf_rx_create_pp(rx_ring); - if (ret) - return ret; - u64_stats_init(&rx_ring->syncp); /* Round up to nearest 4K */ rx_ring->dma_size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->dma_size = ALIGN(rx_ring->dma_size, 4096); - rx_ring->desc = dma_alloc_coherent(rx_ring->pp->p.dev, + rx_ring->desc = dma_alloc_coherent(ixgbevf_dma_dev_from_ring(rx_ring), rx_ring->dma_size, &rx_ring->dma, GFP_KERNEL); if (!rx_ring->desc) { - ret = -ENOMEM; - goto err; + dev_err(rx_ring->dev, + "Unable to allocate memory for the Rx descriptor ring\n"); + return -ENOMEM; } - /* XDP RX-queue info */ - ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, 0, rx_ring->truesize); - if (ret) - goto err; - - xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); - rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; -err: - ixgbevf_rx_destroy_pp(rx_ring); - dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); - - return ret; } /** @@ -3262,24 +3276,14 @@ static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) **/ void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) { - struct libeth_fq fq = { - .fqes = rx_ring->rx_fqes, - .pp = rx_ring->pp, - }; - ixgbevf_clean_rx_ring(rx_ring); - + ixgbevf_rx_destroy_pp(rx_ring); rcu_assign_pointer(rx_ring->xdp_prog, NULL); - xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); - xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - dma_free_coherent(fq.pp->p.dev, rx_ring->dma_size, rx_ring->desc, + dma_free_coherent(ixgbevf_dma_dev_from_ring(rx_ring), + rx_ring->dma_size, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; - - libeth_rx_fq_destroy(&fq); - rx_ring->rx_fqes = NULL; - rx_ring->pp = NULL; } /** From e3676d79b20b2a806c688672a6a49ac90c7b1ba8 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Thu, 8 Jan 2026 13:44:03 +0100 Subject: [PATCH 09/20] ixgbevf: allow changing MTU when XDP program is attached xskxceiver attempts to change MTU after attaching XDP program, ixgbevf rejects the request leading to test being failed. Support MTU change operation even when XDP program is already attached, perform the same frame size check as when attaching a program. Reviewed-by: Aleksandr Loktionov Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index dcb3ebdedc6b2c..c1016fde105fea 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -3969,6 +3969,18 @@ static int ixgbevf_set_mac(struct net_device *netdev, void *p) return 0; } +static bool ixgbevf_xdp_mtu_ok(const struct ixgbevf_adapter *adapter, + const struct bpf_prog *prog, unsigned int mtu) +{ + u32 frame_size = mtu + LIBETH_RX_LL_LEN; + bool requires_mbuf; + + requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM) || + adapter->flags & IXGBEVF_FLAG_HSPLIT; + + return prog->aux->xdp_has_frags || !requires_mbuf; +} + /** * ixgbevf_change_mtu - Change the Maximum Transfer Unit * @netdev: network interface device structure @@ -3984,8 +3996,10 @@ static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) int ret; /* prevent MTU being changed to a size unsupported by XDP */ - if (adapter->xdp_prog) { - dev_warn(&adapter->pdev->dev, "MTU cannot be changed while XDP program is loaded\n"); + if (adapter->xdp_prog && + !ixgbevf_xdp_mtu_ok(adapter, adapter->xdp_prog, new_mtu)) { + netdev_warn(netdev, + "MTU value provided cannot be set while current XDP program is attached\n"); return -EPERM; } @@ -4148,14 +4162,10 @@ ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev, static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, struct netlink_ext_ack *extack) { - u32 frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbevf_adapter *adapter = netdev_priv(dev); struct bpf_prog *old_prog; - bool requires_mbuf; - requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM) || - adapter->flags & IXGBEVF_FLAG_HSPLIT; - if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { + if (prog && !ixgbevf_xdp_mtu_ok(adapter, prog, READ_ONCE(dev->mtu))) { NL_SET_ERR_MSG_MOD(extack, "Configured MTU or HW limitations require non-linear frames and XDP prog does not support frags"); return -EOPNOTSUPP; From 1160e99f45068428047019d630d17790e8ddf658 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Fri, 24 Oct 2025 12:19:51 +0200 Subject: [PATCH 10/20] ixgbevf: move skb-filling code to a header AF_XDP ZC Rx path is also required to implement skb creation. Move all common functions to a header file as inlines. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 167 +----------------- .../ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h | 162 +++++++++++++++++ 2 files changed, 164 insertions(+), 165 deletions(-) create mode 100644 drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index c1016fde105fea..f301ded0067bc4 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -33,6 +33,7 @@ #include #include +#include "ixgbevf_txrx_lib.h" #include "ixgbevf_xdp_lib.h" const char ixgbevf_driver_name[] = "ixgbevf"; @@ -418,134 +419,6 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, return !!budget; } -/** - * ixgbevf_rx_skb - Helper function to determine proper Rx method - * @q_vector: structure containing interrupt and ring information - * @skb: packet to send up - **/ -static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, - struct sk_buff *skb) -{ - napi_gro_receive(&q_vector->napi, skb); -} - -#define IXGBE_RSS_L4_TYPES_MASK \ - ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ - (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ - (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ - (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) - -static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - u16 rss_type; - - if (!(ring->netdev->features & NETIF_F_RXHASH)) - return; - - rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & - IXGBE_RXDADV_RSSTYPE_MASK; - - if (!rss_type) - return; - - skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), - (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? - PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); -} - -/** - * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum - * @ring: structure containig ring specific data - * @rx_desc: current Rx descriptor being processed - * @skb: skb currently being received and modified - **/ -static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - skb_checksum_none_assert(skb); - - /* Rx csum disabled */ - if (!(ring->netdev->features & NETIF_F_RXCSUM)) - return; - - /* if IP and error */ - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && - ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { - ring->rx_stats.csum_err++; - return; - } - - if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) - return; - - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { - ring->rx_stats.csum_err++; - return; - } - - /* It must be a TCP or UDP packet with a valid checksum */ - skb->ip_summed = CHECKSUM_UNNECESSARY; -} - -/** - * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor - * @rx_ring: rx descriptor ring packet is being transacted on - * @rx_desc: pointer to the EOP Rx descriptor - * @skb: pointer to current skb being populated - * - * This function checks the ring, descriptor, and packet information in - * order to populate the checksum, VLAN, protocol, and other fields within - * the skb. - **/ -static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - ixgbevf_rx_hash(rx_ring, rx_desc, skb); - ixgbevf_rx_checksum(rx_ring, rx_desc, skb); - - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { - u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); - unsigned long *active_vlans = netdev_priv(rx_ring->netdev); - - if (test_bit(vid & VLAN_VID_MASK, active_vlans)) - __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); - } - - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP)) - ixgbevf_ipsec_rx(rx_ring, rx_desc, skb); -} - -/** - * ixgbevf_is_non_eop - process handling of non-EOP buffers - * @rx_ring: Rx ring being processed - * @rx_desc: Rx descriptor for current buffer - * - * This function updates next to clean. If the buffer is an EOP buffer - * this function exits returning false, otherwise it will place the - * sk_buff in the next buffer to be chained and return true indicating - * that this is in fact a non-EOP buffer. - **/ -static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc) -{ - u32 ntc = rx_ring->next_to_clean + 1; - - /* fetch, update, and store next to clean */ - ntc = (ntc < rx_ring->count) ? ntc : 0; - rx_ring->next_to_clean = ntc; - - prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); - - if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) - return false; - - return true; -} - /** * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on @@ -619,42 +492,6 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, } } -/** - * ixgbevf_cleanup_headers - Correct corrupted or empty headers - * @rx_ring: rx descriptor ring packet is being transacted on - * @rx_desc: pointer to the EOP Rx descriptor - * @skb: pointer to current skb being fixed - * - * Check for corrupted packet headers caused by senders on the local L2 - * embedded NIC switch not setting up their Tx Descriptors right. These - * should be very rare. - * - * Also address the case where we are pulling data in on pages only - * and as such no data is present in the skb header. - * - * In addition if skb is not at least 60 bytes we need to pad it so that - * it is large enough to qualify as a valid Ethernet frame. - * - * Returns true if an error was encountered and skb was freed. - **/ -static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - /* verify that the packet does not have any known errors */ - if (unlikely(ixgbevf_test_staterr(rx_desc, - IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { - struct net_device *netdev = rx_ring->netdev; - - if (!(netdev->features & NETIF_F_RXALL)) { - dev_kfree_skb_any(skb); - return true; - } - } - - return false; -} - static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, u32 qmask) { @@ -842,7 +679,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* populate checksum, VLAN, and protocol */ ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); - ixgbevf_rx_skb(q_vector, skb); + napi_gro_receive(&q_vector->napi, skb); } /* place incomplete frames back on ring for completion */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h new file mode 100644 index 00000000000000..851d1c054d9810 --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2010-2026 Intel Corporation */ + +#ifndef _IXGBEVF_TXRX_LIB_H_ +#define _IXGBEVF_TXRX_LIB_H_ + +#include "ixgbevf.h" + +/** + * ixgbevf_is_non_eop - process handling of non-EOP buffers + * @rx_ring: Rx ring being processed + * @rx_desc: Rx descriptor for current buffer + * + * This function updates next to clean. If the buffer is an EOP buffer + * this function exits returning false, otherwise it will place the + * sk_buff in the next buffer to be chained and return true indicating + * that this is in fact a non-EOP buffer. + **/ +static inline bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc) +{ + u32 ntc = rx_ring->next_to_clean + 1; + + /* fetch, update, and store next to clean */ + ntc = (ntc < rx_ring->count) ? ntc : 0; + rx_ring->next_to_clean = ntc; + + prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); + + if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) + return false; + + return true; +} + +/** + * ixgbevf_cleanup_headers - Correct corrupted or empty headers + * @rx_ring: rx descriptor ring packet is being transacted on + * @rx_desc: pointer to the EOP Rx descriptor + * @skb: pointer to current skb being fixed + * + * Check for corrupted packet headers caused by senders on the local L2 + * embedded NIC switch not setting up their Tx Descriptors right. These + * should be very rare. + * + * Also address the case where we are pulling data in on pages only + * and as such no data is present in the skb header. + * + * In addition if skb is not at least 60 bytes we need to pad it so that + * it is large enough to qualify as a valid Ethernet frame. + * + * Returns true if an error was encountered and skb was freed. + **/ +static inline bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + /* verify that the packet does not have any known errors */ + if (unlikely(ixgbevf_test_staterr(rx_desc, + IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { + struct net_device *netdev = rx_ring->netdev; + + if (!(netdev->features & NETIF_F_RXALL)) { + dev_kfree_skb_any(skb); + return true; + } + } + + return false; +} + +#define IXGBE_RSS_L4_TYPES_MASK \ + ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ + (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ + (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ + (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) + +static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + u16 rss_type; + + if (!(ring->netdev->features & NETIF_F_RXHASH)) + return; + + rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & + IXGBE_RXDADV_RSSTYPE_MASK; + + if (!rss_type) + return; + + skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), + (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? + PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); +} + +/** + * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum + * @ring: structure containing ring specific data + * @rx_desc: current Rx descriptor being processed + * @skb: skb currently being received and modified + **/ +static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + skb_checksum_none_assert(skb); + + /* Rx csum disabled */ + if (!(ring->netdev->features & NETIF_F_RXCSUM)) + return; + + /* if IP and error */ + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && + ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { + ring->rx_stats.csum_err++; + return; + } + + if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) + return; + + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { + ring->rx_stats.csum_err++; + return; + } + + /* It must be a TCP or UDP packet with a valid checksum */ + skb->ip_summed = CHECKSUM_UNNECESSARY; +} + +/** + * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor + * @rx_ring: rx descriptor ring packet is being transacted on + * @rx_desc: pointer to the EOP Rx descriptor + * @skb: pointer to current skb being populated + * + * This function checks the ring, descriptor, and packet information in + * order to populate the checksum, VLAN, protocol, and other fields within + * the skb. + **/ +static inline void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + ixgbevf_rx_hash(rx_ring, rx_desc, skb); + ixgbevf_rx_checksum(rx_ring, rx_desc, skb); + + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { + u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); + unsigned long *active_vlans = netdev_priv(rx_ring->netdev); + + if (test_bit(vid & VLAN_VID_MASK, active_vlans)) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); + } + + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP)) + ixgbevf_ipsec_rx(rx_ring, rx_desc, skb); +} + +#endif /* _IXGBEVF_TXRX_LIB_H_ */ From 5fb3304bd05eeb8a2fc7331cf93586bfc2e682f2 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 27 Oct 2025 13:32:49 +0100 Subject: [PATCH 11/20] ixgbevf: implement AF_XDP ZC initialization Implement xsk_buff_pool configuration and supporting functionality, such as a single queue pair reconfiguration. Also, properly initialize Rx buffers. Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/Makefile | 2 +- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 32 +++- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 155 ++++++++++++++---- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 133 +++++++++++++++ .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.h | 12 ++ 5 files changed, 299 insertions(+), 35 deletions(-) create mode 100644 drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c create mode 100644 drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h diff --git a/drivers/net/ethernet/intel/ixgbevf/Makefile b/drivers/net/ethernet/intel/ixgbevf/Makefile index 01d3e892f3fa7b..cdae62f25fd926 100644 --- a/drivers/net/ethernet/intel/ixgbevf/Makefile +++ b/drivers/net/ethernet/intel/ixgbevf/Makefile @@ -6,5 +6,5 @@ obj-$(CONFIG_IXGBEVF) += ixgbevf.o -ixgbevf-y := vf.o mbx.o ethtool.o ixgbevf_main.o +ixgbevf-y := vf.o mbx.o ethtool.o ixgbevf_main.o ixgbevf_xsk.o ixgbevf-$(CONFIG_IXGBEVF_IPSEC) += ipsec.o diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index 17958cfb4ee65b..d8f841515ca62a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -66,6 +66,7 @@ enum ixgbevf_ring_state_t { __IXGBEVF_HANG_CHECK_ARMED, __IXGBEVF_TX_XDP_RING, __IXGBEVF_TX_XDP_RING_PRIMED, + __IXGBEVF_RXTX_XSK_RING, }; #define ring_is_xdp(ring) \ @@ -75,6 +76,13 @@ enum ixgbevf_ring_state_t { #define clear_ring_xdp(ring) \ clear_bit(__IXGBEVF_TX_XDP_RING, &(ring)->state) +#define ring_is_xsk(ring) \ + test_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) +#define set_ring_xsk(ring) \ + set_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) +#define clear_ring_xsk(ring) \ + clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) + struct ixgbevf_ring { struct ixgbevf_ring *next; struct ixgbevf_q_vector *q_vector; /* backpointer to q_vector */ @@ -85,22 +93,22 @@ struct ixgbevf_ring { struct device *dev; /* Tx ring */ }; void *desc; /* descriptor ring memory */ - union { - u32 truesize; /* Rx buffer full size */ - u32 pending; /* Sent-not-completed descriptors */ - }; + u32 truesize; /* Rx buffer full size */ u32 hdr_truesize; /* Rx header buffer full size */ u16 count; /* amount of descriptors */ u16 next_to_clean; u32 next_to_use; + u32 pending; /* Sent-not-completed descriptors */ union { struct libeth_fqe *rx_fqes; + struct libeth_xdp_buff **xsk_fqes; struct ixgbevf_tx_buffer *tx_buffer_info; struct libeth_sqe *xdp_sqes; }; struct libeth_xdpsq_lock xdpq_lock; u32 cached_ntu; + u32 thresh; unsigned long state; struct ixgbevf_stats stats; struct u64_stats_sync syncp; @@ -121,8 +129,10 @@ struct ixgbevf_ring { int queue_index; /* needed for multiqueue queue management */ u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; + struct libeth_xdp_buff *xsk_xdp_head; unsigned int dma_size; /* length in bytes */ dma_addr_t dma; /* phys. address of descriptor ring */ + struct xsk_buff_pool *xsk_pool; /* AF_XDP ZC rings */ } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -399,14 +409,28 @@ int ixgbevf_open(struct net_device *netdev); int ixgbevf_close(struct net_device *netdev); void ixgbevf_up(struct ixgbevf_adapter *adapter); void ixgbevf_down(struct ixgbevf_adapter *adapter); +void ixgbevf_flush_tx_queue(struct ixgbevf_ring *ring); +void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); +void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter); void ixgbevf_reset(struct ixgbevf_adapter *adapter); void ixgbevf_set_ethtool_ops(struct net_device *netdev); int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring); +void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter); +void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); int ixgbevf_setup_tx_resources(struct ixgbevf_ring *); +void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); void ixgbevf_free_rx_resources(struct ixgbevf_ring *); +void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring); +void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring); void ixgbevf_free_tx_resources(struct ixgbevf_ring *); +void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring); +void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring); void ixgbevf_update_stats(struct ixgbevf_adapter *adapter); int ethtool_ioctl(struct ifreq *ifr); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index f301ded0067bc4..d5dc52d04d30e3 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -30,11 +30,12 @@ #include #include #include -#include +#include #include #include "ixgbevf_txrx_lib.h" #include "ixgbevf_xdp_lib.h" +#include "ixgbevf_xsk.h" const char ixgbevf_driver_name[] = "ixgbevf"; static const char ixgbevf_driver_string[] = @@ -500,7 +501,7 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); } -static void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) +void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) { ixgbevf_clean_xdp_num(xdp_ring, false, xdp_ring->pending); libeth_xdpsq_put(&xdp_ring->xdpq_lock, xdp_ring->netdev); @@ -1122,7 +1123,7 @@ static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) * ixgbevf_irq_enable - Enable default interrupt generation settings * @adapter: board private structure **/ -static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) +void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; @@ -1131,6 +1132,24 @@ static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); } +/** + * ixgbevf_xsk_pool_from_q - get ZC XSK buffer pool bound to a queue ID + * @ring: Rx or Tx ring + * + * Return: A pointer to xsk_buff_pool structure if there is a buffer pool + * attached, configured as zero-copy, and usable by this queue, NULL otherwise. + */ +static struct xsk_buff_pool *ixgbevf_xsk_pool_from_q(struct ixgbevf_ring *ring) +{ + struct xsk_buff_pool *pool = + xsk_get_pool_from_qid(ring->netdev, ring->queue_index); + + if (!READ_ONCE(ring->xdp_prog) && !ring_is_xdp(ring)) + return NULL; + + return (pool && pool->dev) ? pool : NULL; +} + /** * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset * @adapter: board private structure @@ -1138,8 +1157,8 @@ static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) * * Configure the Tx descriptor ring after a reset. **/ -static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u64 tdba = ring->dma; @@ -1197,6 +1216,12 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, num_possible_cpus() > adapter->num_xdp_queues); } + ring->xsk_pool = ixgbevf_xsk_pool_from_q(ring); + if (ring_is_xdp(ring) && ring->xsk_pool) + set_ring_xsk(ring); + else + clear_ring_xsk(ring); + clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); @@ -1266,8 +1291,8 @@ static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) } #define IXGBEVF_MAX_RX_DESC_POLL 10 -static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; @@ -1291,10 +1316,15 @@ static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, if (!wait_loop) pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", reg_idx); + + /* Specification calls for 100 usec of delay after + * RXDCTL.ENABLE is cleared + */ + usleep_range(100, 200); } -static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; @@ -1370,14 +1400,14 @@ static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); } -static void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) +void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) { struct libeth_fq fq = { .pp = rx_ring->pp, .fqes = rx_ring->rx_fqes, }; - if (!fq.pp) + if (!fq.pp && !rx_ring->xsk_fqes) return; if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) { @@ -1385,6 +1415,21 @@ static void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) xdp_rxq_info_unreg(&rx_ring->xdp_rxq); } + if (test_and_clear_bit(__IXGBEVF_RXTX_XSK_RING, &rx_ring->state)) { + struct libeth_xskfq xskfq = { + .fqes = rx_ring->xsk_fqes, + }; + + libeth_xskfq_destroy(&xskfq); + rx_ring->xsk_fqes = NULL; + rx_ring->pending = xskfq.pending; + rx_ring->thresh = xskfq.thresh; + rx_ring->rx_buf_len = xskfq.buf_len; + rx_ring->xsk_pool = NULL; + + return; + } + libeth_rx_fq_destroy(&fq); rx_ring->rx_fqes = NULL; rx_ring->pp = NULL; @@ -1414,9 +1459,44 @@ static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) LIBETH_XDP_HEADROOM : LIBETH_SKB_HEADROOM), }; + struct xsk_buff_pool *pool; u32 frame_size; int ret; + pool = ixgbevf_xsk_pool_from_q(rx_ring); + if (pool) { + u32 frag_sz = xsk_pool_get_rx_frag_step(pool); + struct libeth_xskfq xskfq = { + .nid = numa_node_id(), + .count = rx_ring->count, + .pool = pool, + }; + + ret = libeth_xskfq_create(&xskfq); + if (ret) + return ret; + + rx_ring->xsk_pool = xskfq.pool; + rx_ring->xsk_fqes = xskfq.fqes; + rx_ring->pending = xskfq.count - 1; + rx_ring->thresh = xskfq.thresh; + rx_ring->rx_buf_len = xskfq.buf_len; + set_ring_xsk(rx_ring); + + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index, 0, frag_sz); + if (ret) + goto err; + + ret = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + rx_ring->xsk_pool); + if (ret) + goto err; + + return 0; + } + /* Some HW requires DMA write sizes to be aligned to 1K, * which warrants fake header split usage, but this is * not an issue if the frame size is at its maximum of 3K @@ -1467,8 +1547,8 @@ static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) return ret; } -static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; union ixgbe_adv_rx_desc *rx_desc; @@ -1509,6 +1589,7 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, /* reset ntu and ntc to place SW in sync with hardwdare */ ring->next_to_clean = 0; ring->next_to_use = 0; + ring->pending = ixgbevf_desc_unused(ring); err = ixgbevf_rx_create_pp(ring); if (err) { @@ -1526,7 +1607,8 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); if (pkt_len <= IXGBE_RXDCTL_RLPMLMASK) { rxdctl |= pkt_len | IXGBE_RXDCTL_RLPML_EN; - rlpml_valid = true; + if (pkt_len <= ring->rx_buf_len) + rlpml_valid = true; } } @@ -1536,7 +1618,11 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); ixgbevf_rx_desc_queue_enable(adapter, ring); - ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); + + if (ring_is_xsk(ring)) + ixgbevf_xsk_alloc_rx_bufs(ring, ring->pending); + else + ixgbevf_alloc_rx_buffers(ring, ring->pending); } /** @@ -1925,8 +2011,13 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter) * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue * @rx_ring: ring to free buffers from **/ -static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) +void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) { + if (ring_is_xsk(rx_ring)) { + ixgbevf_rx_xsk_ring_free_buffs(rx_ring); + goto reset; + } + /* Free Rx ring sk_buff */ libeth_xdp_return_stash(&rx_ring->xdp_stash); @@ -1944,15 +2035,17 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) i = 0; } +reset: rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; + rx_ring->pending = 0; } /** * ixgbevf_clean_tx_ring - Free Tx Buffers * @tx_ring: ring to be cleaned **/ -static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) +void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) { u16 i = tx_ring->next_to_clean; struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; @@ -2035,10 +2128,17 @@ static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) ixgbevf_clean_xdp_ring(adapter->xdp_ring[i]); } +void ixgbevf_flush_tx_queue(struct ixgbevf_ring *ring) +{ + u8 reg_idx = ring->reg_idx; + + IXGBE_WRITE_REG(&ring->q_vector->adapter->hw, IXGBE_VFTXDCTL(reg_idx), + IXGBE_TXDCTL_SWFLSH); +} + void ixgbevf_down(struct ixgbevf_adapter *adapter) { struct net_device *netdev = adapter->netdev; - struct ixgbe_hw *hw = &adapter->hw; int i; /* signal that we are down to the interrupt handler */ @@ -2064,19 +2164,11 @@ void ixgbevf_down(struct ixgbevf_adapter *adapter) timer_delete_sync(&adapter->service_timer); /* disable transmits in the hardware now that interrupts are off */ - for (i = 0; i < adapter->num_tx_queues; i++) { - u8 reg_idx = adapter->tx_ring[i]->reg_idx; - - IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), - IXGBE_TXDCTL_SWFLSH); - } - - for (i = 0; i < adapter->num_xdp_queues; i++) { - u8 reg_idx = adapter->xdp_ring[i]->reg_idx; + for (i = 0; i < adapter->num_tx_queues; i++) + ixgbevf_flush_tx_queue(adapter->tx_ring[i]); - IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), - IXGBE_TXDCTL_SWFLSH); - } + for (i = 0; i < adapter->num_xdp_queues; i++) + ixgbevf_flush_tx_queue(adapter->xdp_ring[i]); if (!pci_channel_offline(adapter->pdev)) ixgbevf_reset(adapter); @@ -4047,6 +4139,9 @@ static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp) switch (xdp->command) { case XDP_SETUP_PROG: return ixgbevf_xdp_setup(dev, xdp->prog, xdp->extack); + case XDP_SETUP_XSK_POOL: + return ixgbevf_setup_xsk_pool(netdev_priv(dev), xdp->xsk.pool, + xdp->xsk.queue_id); default: return -EINVAL; } diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c new file mode 100644 index 00000000000000..21b1cbd25bc9ed --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2026 Intel Corporation */ + +#include + +#include "ixgbevf.h" +#include "ixgbevf_xsk.h" + +/** + * ixgbevf_single_irq_disable - Mask off interrupt generation on a single vector + * @adapter: board private structure + * @vidx: vector id + **/ +static void ixgbevf_single_irq_disable(struct ixgbevf_adapter *adapter, + u16 vidx) +{ + struct ixgbe_hw *hw = &adapter->hw; + + IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, + adapter->eims_enable_mask & ~BIT(vidx)); + IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, BIT(vidx)); + IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, + adapter->eims_enable_mask & ~BIT(vidx)); + + IXGBE_WRITE_FLUSH(hw); + + synchronize_irq(adapter->msix_entries[vidx].vector); +} + +static void ixgbevf_qp_dis(struct ixgbevf_adapter *adapter, u16 qid) +{ + struct ixgbevf_ring *tx_ring, *rx_ring = adapter->rx_ring[qid]; + struct ixgbevf_q_vector *q_vector = rx_ring->q_vector; + + netif_stop_subqueue(adapter->netdev, qid); + ixgbevf_single_irq_disable(adapter, q_vector->v_idx); + napi_disable(&q_vector->napi); + + synchronize_net(); + + ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[qid]); + ixgbevf_clean_rx_ring(rx_ring); + ixgbevf_rx_destroy_pp(rx_ring); + + /* Clean both XDP and normal Tx queue */ + ixgbevf_for_each_ring(tx_ring, q_vector->tx) { + ixgbevf_flush_tx_queue(tx_ring); + if (ring_is_xdp(tx_ring)) + ixgbevf_clean_xdp_ring(tx_ring); + else + ixgbevf_clean_tx_ring(tx_ring); + } +} + +static void ixgbevf_qp_ena(struct ixgbevf_adapter *adapter, u16 qid) +{ + struct ixgbevf_ring *tx_ring, *rx_ring = adapter->rx_ring[qid]; + struct ixgbevf_q_vector *q_vector = rx_ring->q_vector; + + ixgbevf_configure_rx_ring(adapter, rx_ring); + ixgbevf_for_each_ring(tx_ring, q_vector->tx) + ixgbevf_configure_tx_ring(adapter, tx_ring); + + napi_enable(&q_vector->napi); + ixgbevf_irq_enable(adapter); + netif_start_subqueue(adapter->netdev, qid); +} + +int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, + struct xsk_buff_pool *pool, u16 qid) +{ + bool running = !test_bit(__IXGBEVF_DOWN, &adapter->state) && + adapter->xdp_prog; + int err; + + if (running) + ixgbevf_qp_dis(adapter, qid); + + err = libeth_xsk_setup_pool(adapter->netdev, qid, !!pool); + + if (running) + ixgbevf_qp_ena(adapter, qid); + + return err; +} + +static void ixgbevf_fill_rx_xsk_desc(const struct libeth_xskfq_fp *fq, u32 i) +{ + union ixgbe_adv_rx_desc *rx_desc = + &((union ixgbe_adv_rx_desc *)fq->descs)[i]; + + rx_desc->read.pkt_addr = + cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); + rx_desc->wb.upper.length = 0; +} + +void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) +{ + struct libeth_xskfq_fp fq = { + .count = rx_ring->count, + .descs = rx_ring->desc, + .fqes = rx_ring->xsk_fqes, + .ntu = rx_ring->next_to_use, + .pool = rx_ring->xsk_pool, + }; + u32 done; + + done = libeth_xskfqe_alloc(&fq, num, ixgbevf_fill_rx_xsk_desc); + if (likely(done)) { + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(rx_ring, fq.ntu); + } + + rx_ring->next_to_use = fq.ntu; + rx_ring->pending -= done; +} + +void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) +{ + u32 ntc = rx_ring->next_to_clean; + + if (rx_ring->xsk_xdp_head) + xsk_buff_free(&rx_ring->xsk_xdp_head->base); + + rx_ring->xsk_xdp_head = NULL; + + while (ntc != rx_ring->next_to_use) { + xsk_buff_free(&rx_ring->xsk_fqes[ntc]->base); + ntc++; + ntc = ntc == rx_ring->count ? 0 : ntc; + } +} diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h new file mode 100644 index 00000000000000..1cbcea803509b6 --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef _IXGBEVF_XSK_H_ +#define _IXGBEVF_XSK_H_ + +int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, + struct xsk_buff_pool *pool, u16 qid); +void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num); +void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring); + +#endif /* _IXGBEVF_XSK_H_ */ From c6dde36722abf540b1303e03928c47e3a80dd92a Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 27 Oct 2025 14:21:11 +0100 Subject: [PATCH 12/20] ixgbevf: implement AF_XDP zero-copy Tx Add code that handles Tx ZC queues inside of napi_poll(), utilize libeth. As NIC's multiple buffer conventions do not play nicely with AF_XDP's, leave handling of segments for later. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 14 +++++--- .../ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h | 20 ++++++++--- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 34 ++++++++++++++++++- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.h | 5 +++ 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index d5dc52d04d30e3..ab7223aa70cd9a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -733,10 +733,13 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) bool clean_complete = true; ixgbevf_for_each_ring(ring, q_vector->tx) { - if (ring_is_xdp(ring)) - continue; - if (!ixgbevf_clean_tx_irq(q_vector, ring, budget)) - clean_complete = false; + if (ring_is_xsk(ring)) + clean_complete &= + ixgbevf_clean_xsk_tx_irq(q_vector, ring, + budget); + else if (!ring_is_xdp(ring)) + clean_complete &= + ixgbevf_clean_tx_irq(q_vector, ring, budget); } if (budget <= 0) @@ -1222,6 +1225,9 @@ void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, else clear_ring_xsk(ring); + ring->thresh = ring_is_xsk(ring) ? IXGBEVF_XSK_TX_CLEAN_THRESH(ring) : + XDP_BULK_QUEUE_SIZE; + clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h index da4f397944ed88..08b0c427d1a3b5 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -4,7 +4,7 @@ #ifndef _IXGBEVF_XDP_LIB_H_ #define _IXGBEVF_XDP_LIB_H_ -#include +#include #include "ixgbevf.h" @@ -40,6 +40,7 @@ static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, bool in_napi, u16 to_clean) { struct libeth_xdpsq_napi_stats stats = { }; + bool xsk_ring = ring_is_xsk(xdp_ring); u32 ntc = xdp_ring->next_to_clean; struct xdp_frame_bulk cbulk; struct libeth_cq_pp cp = { @@ -48,11 +49,14 @@ static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, .xss = &stats, .napi = in_napi, }; + u32 xsk_frames = 0; xdp_frame_bulk_init(&cbulk); xdp_ring->pending -= to_clean; while (likely(to_clean--)) { + xsk_frames += xsk_ring && + likely(!xdp_ring->xdp_sqes[ntc].type) ? 1 : 0; libeth_xdp_complete_tx(&xdp_ring->xdp_sqes[ntc], &cp); ntc++; ntc = unlikely(ntc == xdp_ring->count) ? 0 : ntc; @@ -60,6 +64,8 @@ static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, xdp_ring->next_to_clean = ntc; xdp_flush_frame_bulk(&cbulk); + if (xsk_frames) + xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); } static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) @@ -67,8 +73,8 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) struct ixgbevf_ring *xdp_ring = xdpsq; libeth_xdpsq_lock(&xdp_ring->xdpq_lock); - if (unlikely(ixgbevf_desc_unused(xdp_ring) < LIBETH_XDP_TX_BULK)) { - u16 to_clean = ixgbevf_tx_get_num_sent(xdp_ring); + if (unlikely(ixgbevf_desc_unused(xdp_ring) < xdp_ring->thresh)) { + u16 to_clean = ixgbevf_tx_get_num_sent(xdpsq); if (likely(to_clean)) ixgbevf_clean_xdp_num(xdp_ring, true, to_clean); @@ -91,6 +97,7 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) xdp_ring->next_to_use = 1; xdp_ring->pending = 1; + xdp_ring->xdp_sqes[0].type = LIBETH_SQE_CTX; /* Finish descriptor writes before bumping tail */ wmb(); @@ -103,7 +110,7 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) .lock = &xdp_ring->xdpq_lock, .ntu = &xdp_ring->next_to_use, .pending = &xdp_ring->pending, - .pool = NULL, + .pool = xdp_ring->xsk_pool, .sqes = xdp_ring->xdp_sqes, }; @@ -123,6 +130,11 @@ static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) goto unlock; ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; + + /* We will not get DD on a context descriptor */ + if (unlikely(xdp_ring->xdp_sqes[ltu].type == LIBETH_SQE_CTX)) + goto unlock; + desc = IXGBEVF_TX_DESC(xdp_ring, ltu); desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c index 21b1cbd25bc9ed..3566857cebc723 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -3,7 +3,8 @@ #include -#include "ixgbevf.h" +#include "ixgbevf_txrx_lib.h" +#include "ixgbevf_xdp_lib.h" #include "ixgbevf_xsk.h" /** @@ -131,3 +132,34 @@ void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) ntc = ntc == rx_ring->count ? 0 : ntc; } } + +static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, u64 priv) +{ + union ixgbe_adv_tx_desc *tx_desc = + &((union ixgbe_adv_tx_desc *)sq->descs)[i]; + + u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS | + IXGBE_TXD_CMD_EOP | + desc.len; + + tx_desc->read.olinfo_status = + cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + + tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); +} + +bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *tx_ring, int napi_budget) +{ + u32 budget = min_t(u32, napi_budget, tx_ring->thresh); + + return libeth_xsk_xmit_do_bulk(tx_ring->xsk_pool, tx_ring, budget, + NULL, ixgbevf_prep_xdp_sq, + ixgbevf_xsk_xmit_desc, + ixgbevf_xdp_rs_and_bump); +} diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h index 1cbcea803509b6..eda3e9b9554763 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -4,9 +4,14 @@ #ifndef _IXGBEVF_XSK_H_ #define _IXGBEVF_XSK_H_ +/* Process completions as soon as possible */ +#define IXGBEVF_XSK_TX_CLEAN_THRESH(r) ((r)->count - 1) + int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, struct xsk_buff_pool *pool, u16 qid); void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num); void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring); +bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *tx_ring, int napi_budget); #endif /* _IXGBEVF_XSK_H_ */ From da809e83c55255d96fdc89ce480ba4b13342fcf1 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 27 Oct 2025 14:24:57 +0100 Subject: [PATCH 13/20] ixgbevf: implement AF_XDP zero-copy Rx Add code that handles AF_XDP ZC Rx queues inside of napi_poll(), utilize libeth helpers. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 5 +- .../ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h | 1 + .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 118 +++++++++++++++++- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.h | 4 +- 4 files changed, 125 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index ab7223aa70cd9a..6d074825217cba 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -754,7 +754,10 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) per_ring_budget = budget; ixgbevf_for_each_ring(ring, q_vector->rx) { - int cleaned = ixgbevf_clean_rx_irq(q_vector, ring, + int cleaned = ring_is_xsk(ring) ? + ixgbevf_clean_xsk_rx_irq(q_vector, ring, + per_ring_budget) : + ixgbevf_clean_rx_irq(q_vector, ring, per_ring_budget); work_done += cleaned; if (cleaned >= per_ring_budget) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h index 851d1c054d9810..ac36fac9f79717 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h @@ -24,6 +24,7 @@ static inline bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, /* fetch, update, and store next to clean */ ntc = (ntc < rx_ring->count) ? ntc : 0; rx_ring->next_to_clean = ntc; + rx_ring->pending++; prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c index 3566857cebc723..549eb8c999bd78 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -65,6 +65,7 @@ static void ixgbevf_qp_ena(struct ixgbevf_adapter *adapter, u16 qid) napi_enable(&q_vector->napi); ixgbevf_irq_enable(adapter); netif_start_subqueue(adapter->netdev, qid); + napi_schedule(&q_vector->napi); } int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, @@ -95,7 +96,7 @@ static void ixgbevf_fill_rx_xsk_desc(const struct libeth_xskfq_fp *fq, u32 i) rx_desc->wb.upper.length = 0; } -void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) +bool ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) { struct libeth_xskfq_fp fq = { .count = rx_ring->count, @@ -115,6 +116,8 @@ void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) rx_ring->next_to_use = fq.ntu; rx_ring->pending -= done; + + return done == num; } void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) @@ -153,6 +156,119 @@ static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); } +LIBETH_XDP_DEFINE_START(); +LIBETH_XSK_DEFINE_FLUSH_TX(static ixgbevf_xsk_flush_tx, ixgbevf_prep_xdp_sq, + ixgbevf_xsk_xmit_desc); +LIBETH_XSK_DEFINE_RUN_PROG(static ixgbevf_xsk_run_prog, ixgbevf_xsk_flush_tx); +LIBETH_XSK_DEFINE_FINALIZE(static ixgbevf_xsk_finalize_xdp_napi, + ixgbevf_xsk_flush_tx, ixgbevf_xdp_rs_and_bump); +LIBETH_XDP_DEFINE_END(); + +u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *rx_ring, int budget) +{ + struct ixgbevf_adapter *adapter = q_vector->adapter; + u32 total_rx_bytes = 0, total_rx_packets = 0; + LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); + struct libeth_xdp_buff *head_xdp; + bool failure = false, wake; + struct sk_buff *skb; + + wake = xsk_uses_need_wakeup(rx_ring->xsk_pool); + if (wake) + xsk_clear_rx_need_wakeup(rx_ring->xsk_pool); + + head_xdp = rx_ring->xsk_xdp_head; + libeth_xsk_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, + adapter->netdev, adapter->xdp_ring, + adapter->num_xdp_queues); + + while (likely(total_rx_packets < budget)) { + union ixgbe_adv_rx_desc *rx_desc; + struct libeth_xdp_buff *rx_buffer; + unsigned int size; + u32 xdp_result; + + rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); + size = le16_to_cpu(rx_desc->wb.upper.length); + if (unlikely(!size)) + break; + + /* Avoid reading other descriptor fields before checking size */ + rmb(); + + rx_buffer = rx_ring->xsk_fqes[rx_ring->next_to_clean]; + head_xdp = libeth_xsk_process_buff(head_xdp, rx_buffer, size); + if (unlikely(!head_xdp) || ixgbevf_is_non_eop(rx_ring, rx_desc)) + continue; + + total_rx_packets++; + total_rx_bytes += xdp_get_buff_len(&head_xdp->base); + + xdp_result = ixgbevf_xsk_run_prog(head_xdp, &xdp_tx_bulk); + if (xdp_result) { + head_xdp = NULL; + if (likely(xdp_result != LIBETH_XDP_ABORTED)) + continue; + failure = true; + break; + } + + skb = xdp_build_skb_from_zc(&head_xdp->base); + + if (unlikely(!skb)) { + libeth_xdp_return_buff_slow(head_xdp); + head_xdp = NULL; + rx_ring->rx_stats.alloc_rx_buff_failed++; + break; + } + + head_xdp = NULL; + + if (unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) { + skb = NULL; + continue; + } + + if (unlikely((skb->pkt_type == PACKET_BROADCAST || + skb->pkt_type == PACKET_MULTICAST) && + ether_addr_equal(rx_ring->netdev->dev_addr, + eth_hdr(skb)->h_source))) { + dev_kfree_skb_irq(skb); + continue; + } + + /* populate checksum, VLAN, and protocol */ + ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); + + napi_gro_receive(&q_vector->napi, skb); + } + + if (rx_ring->pending >= rx_ring->thresh) + failure |= !ixgbevf_xsk_alloc_rx_bufs(rx_ring, + rx_ring->pending); + + /* place incomplete frames back on ring for completion */ + rx_ring->xsk_xdp_head = head_xdp; + + ixgbevf_xsk_finalize_xdp_napi(&xdp_tx_bulk); + + u64_stats_update_begin(&rx_ring->syncp); + rx_ring->stats.packets += total_rx_packets; + rx_ring->stats.bytes += total_rx_bytes; + u64_stats_update_end(&rx_ring->syncp); + q_vector->rx.total_packets += total_rx_packets; + q_vector->rx.total_bytes += total_rx_bytes; + + if (likely(!failure)) + return total_rx_packets; + + if (wake) + xsk_set_rx_need_wakeup(rx_ring->xsk_pool); + + return budget; +} + bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *tx_ring, int napi_budget) { diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h index eda3e9b9554763..042a90cfa9131c 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -9,8 +9,10 @@ int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, struct xsk_buff_pool *pool, u16 qid); -void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num); +bool ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num); void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring); +u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *rx_ring, int budget); bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *tx_ring, int napi_budget); From 98845da5530932c6bb1635c85508b12a136fa723 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 27 Oct 2025 14:26:46 +0100 Subject: [PATCH 14/20] ixgbevf: implement .ndo_xsk_wakeup() and set features To finalize basic AF_XDP implementation, set features and add .ndo_xsk_wakeup() handler. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 3 ++- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 24 +++++++++++++++++++ .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.h | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 6d074825217cba..4e99c36c6b574d 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -4171,6 +4171,7 @@ static const struct net_device_ops ixgbevf_netdev_ops = { .ndo_features_check = ixgbevf_features_check, .ndo_bpf = ixgbevf_xdp, .ndo_xdp_xmit = ixgbevf_xdp_xmit, + .ndo_xsk_wakeup = ixgbevf_xsk_wakeup, }; static void ixgbevf_assign_netdev_ops(struct net_device *dev) @@ -4303,7 +4304,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; - libeth_xdp_set_features_noredir(netdev, NULL, 0, NULL); + libeth_xdp_set_features_noredir(netdev, NULL, 1, NULL); /* MTU range: 68 - 1504 or 9710 */ netdev->min_mtu = ETH_MIN_MTU; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c index 549eb8c999bd78..be849f83c12361 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -279,3 +279,27 @@ bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, ixgbevf_xsk_xmit_desc, ixgbevf_xdp_rs_and_bump); } + +int ixgbevf_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) +{ + struct ixgbevf_adapter *adapter = netdev_priv(dev); + struct ixgbevf_q_vector *q_vector; + struct ixgbevf_ring *rx_ring; + + if (unlikely(test_bit(__IXGBEVF_DOWN, &adapter->state))) + return -ENETDOWN; + + if (unlikely(queue_id >= adapter->num_xdp_queues)) + return -EINVAL; + + rx_ring = adapter->rx_ring[queue_id]; + if (unlikely(!ring_is_xsk(rx_ring))) + return -EINVAL; + + q_vector = rx_ring->q_vector; + if (!napi_if_scheduled_mark_missed(&q_vector->napi)) + IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, + BIT(q_vector->v_idx)); + + return 0; +} diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h index 042a90cfa9131c..7af14c78ead978 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -15,5 +15,6 @@ u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *rx_ring, int budget); bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *tx_ring, int napi_budget); +int ixgbevf_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags); #endif /* _IXGBEVF_XSK_H_ */ From d9c60d3b1fad594edeb9eb8a2c85b90de67e5359 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Fri, 19 Dec 2025 14:48:57 +0100 Subject: [PATCH 15/20] ixgbevf: multi-buffer AF_XDP Tx Transmitting multi-buffer AF_XDP packets is not very straightforward given HW limitations in ixgbevf, namely that the first data descriptor must contain the length of the whole packet. Use private data of an sqe to store the length of an unfinished packet so far and the first descriptor index. Once EoP zero-copy descriptor is processed, write the accumulated length into the saved first descriptor. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 2 +- .../ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h | 3 + .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 65 ++++++++++++++++--- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.h | 1 + 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 4e99c36c6b574d..3006d6dae7147d 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -4304,7 +4304,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; - libeth_xdp_set_features_noredir(netdev, NULL, 1, NULL); + libeth_xdp_set_features_noredir(netdev, NULL, IXGBEVF_XSK_MAX_ZC_FRAGS); /* MTU range: 68 - 1504 or 9710 */ netdev->min_mtu = ETH_MIN_MTU; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h index 08b0c427d1a3b5..ad1e09ae6aa1c3 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -141,6 +141,9 @@ static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1; xdp_ring->cached_ntu = xdp_ring->next_to_use; + /* In case the packet was interrupted, discard it */ + xdp_ring->xdp_sqes[ltu].priv = 0; + /* Finish descriptor writes before bumping tail */ wmb(); ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c index be849f83c12361..9bb4477f43e979 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -136,24 +136,73 @@ void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) } } +struct ixgbevf_zc_sqe_priv { + u16 first_desc; + u16 len; +}; + +static_assert(sizeof(struct ixgbevf_zc_sqe_priv) <= + sizeof_field(struct libeth_sqe, priv)); + static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, const struct libeth_xdpsq *sq, u64 priv) { - union ixgbe_adv_tx_desc *tx_desc = - &((union ixgbe_adv_tx_desc *)sq->descs)[i]; + union ixgbe_adv_tx_desc *descs = sq->descs, *tx_desc = &descs[i]; + u32 ltu = (i ? : sq->count) - 1; u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DCMD_IFCS | - IXGBE_TXD_CMD_EOP | desc.len; - tx_desc->read.olinfo_status = - cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); - tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + + if (likely((desc.flags & LIBETH_XDP_TX_LAST) && !sq->sqes[ltu].priv)) { + tx_desc->read.olinfo_status = + cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + tx_desc->read.cmd_type_len = + cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP); + return; + } + + /* No previous packet */ + if (!sq->sqes[ltu].priv) { + struct ixgbevf_zc_sqe_priv *sqe_priv = + (void *)&sq->sqes[i].priv; + + sqe_priv->first_desc = i; + sqe_priv->len = desc.len; + + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + + return; + } + + if (sq->sqes[ltu].priv) { + struct ixgbevf_zc_sqe_priv *sqe_priv = + (void *)&sq->sqes[i].priv; + + sq->sqes[i].priv = sq->sqes[ltu].priv; + sq->sqes[ltu].priv = 0; + sqe_priv->len += desc.len; + + if (desc.flags & LIBETH_XDP_TX_LAST) { + union ixgbe_adv_tx_desc *first_desc = + &descs[sqe_priv->first_desc]; + + first_desc->read.olinfo_status = + cpu_to_le32((sqe_priv->len << + IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + tx_desc->read.cmd_type_len = + cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP); + cmd_type |= IXGBE_TXD_CMD_EOP; + sq->sqes[i].priv = 0; + } + + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + } } LIBETH_XDP_DEFINE_START(); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h index 7af14c78ead978..2bb39735b10efd 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -6,6 +6,7 @@ /* Process completions as soon as possible */ #define IXGBEVF_XSK_TX_CLEAN_THRESH(r) ((r)->count - 1) +#define IXGBEVF_XSK_MAX_ZC_FRAGS min(18, MAX_SKB_FRAGS) int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, struct xsk_buff_pool *pool, u16 qid); From 56628ce4a0e14f01d19ffc740c21d69f13913fd8 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Mon, 13 Apr 2026 12:12:32 +0200 Subject: [PATCH 16/20] ixgbe: remove legacy rx Similarly, like for virtual function in ixgbevf, remove ixgbe_construct_skb(), the legacy-rx private flag, and all ring_uses_build_skb() conditionals. build_skb is now the only RX code path. This is a preparation for the conversion to libeth and page_pool based Rx buffer management. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 18 +-- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 13 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 132 +++--------------- 3 files changed, 25 insertions(+), 138 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index dce4936708eb44..940d43c6ccf820 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -130,12 +130,8 @@ static inline int ixgbe_skb_pad(void) #endif /* - * NOTE: netdev_alloc_skb reserves up to 64 bytes, NET_IP_ALIGN means we - * reserve 64 more, and skb_shared_info adds an additional 320 bytes more, - * this adds up to 448 bytes of extra data. - * - * Since netdev_alloc_skb now allocates a page fragment we can use a value - * of 256 and the resultant skb will have a truesize of 960 or less. + * RX header buffer size for page-based allocation using napi_build_skb. + * Allows efficient header/data split without excessive overhead. */ #define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256 @@ -312,7 +308,6 @@ struct ixgbe_rx_queue_stats { enum ixgbe_ring_state_t { __IXGBE_RX_3K_BUFFER, - __IXGBE_RX_BUILD_SKB_ENABLED, __IXGBE_RX_RSC_ENABLED, __IXGBE_RX_CSUM_UDP_ZERO_ERR, __IXGBE_RX_FCOE, @@ -324,9 +319,6 @@ enum ixgbe_ring_state_t { __IXGBE_TX_DISABLED, }; -#define ring_uses_build_skb(ring) \ - test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &(ring)->state) - struct ixgbe_fwd_adapter { unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; struct net_device *netdev; @@ -456,8 +448,7 @@ static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring) if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) return IXGBE_RXBUFFER_3K; #if (PAGE_SIZE < 8192) - if (ring_uses_build_skb(ring)) - return IXGBE_MAX_2K_FRAME_BUILD_SKB; + return IXGBE_MAX_2K_FRAME_BUILD_SKB; #endif return IXGBE_RXBUFFER_2K; } @@ -471,7 +462,6 @@ static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring) return 0; } #define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring)) - #define IXGBE_ITR_ADAPTIVE_MIN_INC 2 #define IXGBE_ITR_ADAPTIVE_MIN_USECS 10 #define IXGBE_ITR_ADAPTIVE_MAX_USECS 126 @@ -672,7 +662,7 @@ struct ixgbe_adapter { #define IXGBE_FLAG2_VLAN_PROMISC BIT(13) #define IXGBE_FLAG2_EEE_CAPABLE BIT(14) #define IXGBE_FLAG2_EEE_ENABLED BIT(15) -#define IXGBE_FLAG2_RX_LEGACY BIT(16) +/* BIT16 used to be reserved for legacy RX flag */ #define IXGBE_FLAG2_IPSEC_ENABLED BIT(17) #define IXGBE_FLAG2_VF_IPSEC_ENABLED BIT(18) #define IXGBE_FLAG2_AUTO_DISABLE_VF BIT(19) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 2ad81f687a844a..eaf50ce47fe5e1 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -132,11 +132,9 @@ static const char ixgbe_gstrings_test[][ETH_GSTRING_LEN] = { #define IXGBE_TEST_LEN sizeof(ixgbe_gstrings_test) / ETH_GSTRING_LEN static const char ixgbe_priv_flags_strings[][ETH_GSTRING_LEN] = { -#define IXGBE_PRIV_FLAGS_LEGACY_RX BIT(0) - "legacy-rx", -#define IXGBE_PRIV_FLAGS_VF_IPSEC_EN BIT(1) +#define IXGBE_PRIV_FLAGS_VF_IPSEC_EN BIT(0) "vf-ipsec", -#define IXGBE_PRIV_FLAGS_AUTO_DISABLE_VF BIT(2) +#define IXGBE_PRIV_FLAGS_AUTO_DISABLE_VF BIT(1) "mdd-disable-vf", }; @@ -3667,9 +3665,6 @@ static u32 ixgbe_get_priv_flags(struct net_device *netdev) struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); u32 priv_flags = 0; - if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) - priv_flags |= IXGBE_PRIV_FLAGS_LEGACY_RX; - if (adapter->flags2 & IXGBE_FLAG2_VF_IPSEC_ENABLED) priv_flags |= IXGBE_PRIV_FLAGS_VF_IPSEC_EN; @@ -3685,10 +3680,6 @@ static int ixgbe_set_priv_flags(struct net_device *netdev, u32 priv_flags) unsigned int flags2 = adapter->flags2; unsigned int i; - flags2 &= ~IXGBE_FLAG2_RX_LEGACY; - if (priv_flags & IXGBE_PRIV_FLAGS_LEGACY_RX) - flags2 |= IXGBE_FLAG2_RX_LEGACY; - flags2 &= ~IXGBE_FLAG2_VF_IPSEC_ENABLED; if (priv_flags & IXGBE_PRIV_FLAGS_VF_IPSEC_EN) flags2 |= IXGBE_FLAG2_VF_IPSEC_ENABLED; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index c58051e4350be2..d6b8fb050a2f45 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1730,19 +1730,14 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, } } -static unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring) -{ - return ring_uses_build_skb(rx_ring) ? IXGBE_SKB_PAD : 0; -} - static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *bi) { - struct page *page = bi->page; + struct page *page; dma_addr_t dma; /* since we are recycling buffers we should seldom need to alloc */ - if (likely(page)) + if (likely(bi->page)) return true; /* alloc new page for storage */ @@ -2034,24 +2029,12 @@ static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring, struct sk_buff *skb) { - if (ring_uses_build_skb(rx_ring)) { - unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1; - unsigned long offset = (unsigned long)(skb->data) & mask; + unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1; + unsigned long offset = (unsigned long)(skb->data) & mask; - dma_sync_single_range_for_cpu(rx_ring->dev, - IXGBE_CB(skb)->dma, - offset, - skb_headlen(skb), - DMA_FROM_DEVICE); - } else { - skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; - - dma_sync_single_range_for_cpu(rx_ring->dev, - IXGBE_CB(skb)->dma, - skb_frag_off(frag), - skb_frag_size(frag), - DMA_FROM_DEVICE); - } + dma_sync_single_range_for_cpu(rx_ring->dev, IXGBE_CB(skb)->dma, + offset, skb_headlen(skb), + DMA_FROM_DEVICE); /* If the page was released, just unmap it. */ if (unlikely(IXGBE_CB(skb)->page_released)) { @@ -2192,13 +2175,9 @@ static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer, * @skb: sk_buff to place the data into * @size: size of data in rx_buffer * - * This function will add the data contained in rx_buffer->page to the skb. - * This is done either through a direct copy if the data in the buffer is - * less than the skb header size, otherwise it will just attach the page as - * a frag to the skb. - * - * The function will then update the page offset if necessary and return - * true if the buffer can be reused by the adapter. + * This function attaches the page contained in rx_buffer to the skb as + * a fragment. It then updates the page offset based on the configured + * PAGE_SIZE to prepare the buffer for the next packet. **/ static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *rx_buffer, @@ -2208,9 +2187,7 @@ static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, #if (PAGE_SIZE < 8192) unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; #else - unsigned int truesize = rx_ring->rx_offset ? - SKB_DATA_ALIGN(rx_ring->rx_offset + size) : - SKB_DATA_ALIGN(size); + unsigned int truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size); #endif skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, rx_buffer->page_offset, size, truesize); @@ -2291,65 +2268,6 @@ static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring, rx_buffer->skb = NULL; } -static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int size = xdp->data_end - xdp->data; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); -#endif - struct sk_buff *skb; - - /* prefetch first cache line of first page */ - net_prefetch(xdp->data); - - /* Note, we get here by enabling legacy-rx via: - * - * ethtool --set-priv-flags legacy-rx on - * - * In this mode, we currently get 0 extra XDP headroom as - * opposed to having legacy-rx off, where we process XDP - * packets going to stack via ixgbe_build_skb(). The latter - * provides us currently with 192 bytes of headroom. - * - * For ixgbe_construct_skb() mode it means that the - * xdp->data_meta will always point to xdp->data, since - * the helper cannot expand the head. Should this ever - * change in future for legacy-rx mode on, then lets also - * add xdp->data_meta handling here. - */ - - /* allocate a skb to store the frags */ - skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE); - if (unlikely(!skb)) - return NULL; - - if (size > IXGBE_RX_HDR_SIZE) { - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) - IXGBE_CB(skb)->dma = rx_buffer->dma; - - skb_add_rx_frag(skb, 0, rx_buffer->page, - xdp->data - page_address(rx_buffer->page), - size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - } else { - memcpy(__skb_put(skb, size), - xdp->data, ALIGN(size, sizeof(long))); - rx_buffer->pagecnt_bias++; - } - - return skb; -} - static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *rx_buffer, struct xdp_buff *xdp, @@ -2460,10 +2378,8 @@ static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring, #if (PAGE_SIZE < 8192) truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ #else - truesize = rx_ring->rx_offset ? - SKB_DATA_ALIGN(rx_ring->rx_offset + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : - SKB_DATA_ALIGN(size); + truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); #endif return truesize; } @@ -2504,7 +2420,6 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, unsigned int mss = 0; #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); - unsigned int offset = rx_ring->rx_offset; unsigned int xdp_xmit = 0; struct xdp_buff xdp; int xdp_res = 0; @@ -2543,6 +2458,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* retrieve a buffer from the ring */ if (!skb) { + unsigned int offset = rx_ring->rx_offset; unsigned char *hard_start; hard_start = page_address(rx_buffer->page) + @@ -2567,12 +2483,9 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_bytes += size; } else if (skb) { ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size); - } else if (ring_uses_build_skb(rx_ring)) { + } else { skb = ixgbe_build_skb(rx_ring, rx_buffer, &xdp, rx_desc); - } else { - skb = ixgbe_construct_skb(rx_ring, rx_buffer, - &xdp, rx_desc); } /* exit if we failed to retrieve a buffer */ @@ -4557,14 +4470,13 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, * This can happen in SRIOV mode when the MTU of the VF is * higher than the MTU of the PF. */ - if (ring_uses_build_skb(ring) && - !test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) + if (!test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB | IXGBE_RXDCTL_RLPML_EN; #endif } - ring->rx_offset = ixgbe_rx_offset(ring); + ring->rx_offset = IXGBE_SKB_PAD; if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); @@ -4734,7 +4646,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) clear_ring_rsc_enabled(rx_ring); clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - clear_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state); if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) set_ring_rsc_enabled(rx_ring); @@ -4742,11 +4653,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state)) set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) - continue; - - set_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state); - #if (PAGE_SIZE < 8192) if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); @@ -7199,7 +7105,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, rx_ring->queue_index, ixgbe_rx_napi_id(rx_ring)) < 0) goto err; - WRITE_ONCE(rx_ring->xdp_prog, adapter->xdp_prog); + rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; err: @@ -7335,7 +7241,7 @@ static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) */ static int ixgbe_max_xdp_frame_size(struct ixgbe_adapter *adapter) { - if (PAGE_SIZE >= 8192 || adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) + if (PAGE_SIZE >= 8192) return IXGBE_RXBUFFER_2K; else return IXGBE_RXBUFFER_3K; From 8d9a520667a94114aa4201752c2655a7318ee901 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Thu, 7 May 2026 23:33:54 +0200 Subject: [PATCH 17/20] ixgbe: do not share pages between packets Stop sharing pages between multiple Rx buffers. Convert ixgbe Rx path from page-sharing/recycling to a simple one-page-per-packet model. Remove logic that refers to page-sharing buffer management. This is a preparatory step before converting the Rx path to page_pool and libeth. Fixed XDP_DROP/ABORTED: reaches ixgbe_put_rx_buffer() but page is never freed. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 91 +--- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 8 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 417 ++++-------------- 3 files changed, 87 insertions(+), 429 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index 940d43c6ccf820..ec9390cd457f84 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -69,65 +69,13 @@ /* Supported Rx Buffer Sizes */ #define IXGBE_RXBUFFER_256 256 /* Used for skb receive header */ -#define IXGBE_RXBUFFER_1536 1536 -#define IXGBE_RXBUFFER_2K 2048 #define IXGBE_RXBUFFER_3K 3072 #define IXGBE_RXBUFFER_4K 4096 #define IXGBE_MAX_RXBUFFER 16384 /* largest size for a single descriptor */ #define IXGBE_PKT_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2)) -/* Attempt to maximize the headroom available for incoming frames. We - * use a 2K buffer for receives and need 1536/1534 to store the data for - * the frame. This leaves us with 512 bytes of room. From that we need - * to deduct the space needed for the shared info and the padding needed - * to IP align the frame. - * - * Note: For cache line sizes 256 or larger this value is going to end - * up negative. In these cases we should fall back to the 3K - * buffers. - */ -#if (PAGE_SIZE < 8192) -#define IXGBE_MAX_2K_FRAME_BUILD_SKB (IXGBE_RXBUFFER_1536 - NET_IP_ALIGN) -#define IXGBE_2K_TOO_SMALL_WITH_PADDING \ -((NET_SKB_PAD + IXGBE_RXBUFFER_1536) > SKB_WITH_OVERHEAD(IXGBE_RXBUFFER_2K)) - -static inline int ixgbe_compute_pad(int rx_buf_len) -{ - int page_size, pad_size; - - page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2); - pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len; - - return pad_size; -} - -static inline int ixgbe_skb_pad(void) -{ - int rx_buf_len; - - /* If a 2K buffer cannot handle a standard Ethernet frame then - * optimize padding for a 3K buffer instead of a 1.5K buffer. - * - * For a 3K buffer we need to add enough padding to allow for - * tailroom due to NET_IP_ALIGN possibly shifting us out of - * cache-line alignment. - */ - if (IXGBE_2K_TOO_SMALL_WITH_PADDING) - rx_buf_len = IXGBE_RXBUFFER_3K + SKB_DATA_ALIGN(NET_IP_ALIGN); - else - rx_buf_len = IXGBE_RXBUFFER_1536; - - /* if needed make room for NET_IP_ALIGN */ - rx_buf_len -= NET_IP_ALIGN; - - return ixgbe_compute_pad(rx_buf_len); -} - -#define IXGBE_SKB_PAD ixgbe_skb_pad() -#else #define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) -#endif /* * RX header buffer size for page-based allocation using napi_build_skb. @@ -270,11 +218,9 @@ struct ixgbe_tx_buffer { struct ixgbe_rx_buffer { union { struct { - struct sk_buff *skb; dma_addr_t dma; struct page *page; __u32 page_offset; - __u16 pagecnt_bias; }; struct { bool discard; @@ -307,7 +253,6 @@ struct ixgbe_rx_queue_stats { #define IXGBE_TS_HDR_LEN 8 enum ixgbe_ring_state_t { - __IXGBE_RX_3K_BUFFER, __IXGBE_RX_RSC_ENABLED, __IXGBE_RX_CSUM_UDP_ZERO_ERR, __IXGBE_RX_FCOE, @@ -374,12 +319,9 @@ struct ixgbe_ring { unsigned long last_rx_timestamp; - union { - u16 next_to_alloc; - struct { - u8 atr_sample_rate; - u8 atr_count; - }; + struct { + u8 atr_sample_rate; + u8 atr_count; }; u8 dcb_tc; @@ -389,7 +331,7 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; - u16 rx_offset; + struct sk_buff *skb; /* partial Rx skb across non-EOP */ struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; @@ -438,30 +380,6 @@ struct ixgbe_ring_feature { #define IXGBE_82599_VMDQ_4Q_MASK 0x7C #define IXGBE_82599_VMDQ_2Q_MASK 0x7E -/* - * FCoE requires that all Rx buffers be over 2200 bytes in length. Since - * this is twice the size of a half page we need to double the page order - * for FCoE enabled Rx queues. - */ -static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring) -{ - if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - return IXGBE_RXBUFFER_3K; -#if (PAGE_SIZE < 8192) - return IXGBE_MAX_2K_FRAME_BUILD_SKB; -#endif - return IXGBE_RXBUFFER_2K; -} - -static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring) -{ -#if (PAGE_SIZE < 8192) - if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - return 1; -#endif - return 0; -} -#define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring)) #define IXGBE_ITR_ADAPTIVE_MIN_INC 2 #define IXGBE_ITR_ADAPTIVE_MIN_USECS 10 #define IXGBE_ITR_ADAPTIVE_MAX_USECS 126 @@ -909,7 +827,6 @@ struct ixgbe_cb { }; dma_addr_t dma; u16 append_cnt; - bool page_released; }; #define IXGBE_CB(skb) ((struct ixgbe_cb *)(skb)->cb) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index eaf50ce47fe5e1..b9db6ee9668e93 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1961,6 +1961,10 @@ static int ixgbe_setup_desc_rings(struct ixgbe_adapter *adapter) hw->mac.ops.disable_rx(hw); + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + ixgbe_configure_rx_ring(adapter, rx_ring); rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXCTRL); @@ -2119,7 +2123,7 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, /* sync Rx buffer for CPU read */ dma_sync_single_for_cpu(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_bufsz(rx_ring), + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* verify contents of skb */ @@ -2131,7 +2135,7 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, /* sync Rx buffer for device write */ dma_sync_single_for_device(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_bufsz(rx_ring), + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* increment Rx next to clean counter */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index d6b8fb050a2f45..7d18cefb77c987 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -592,7 +592,6 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) struct my_u0 { u64 a; u64 b; } *u0; struct ixgbe_ring *rx_ring; union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *rx_buffer_info; int i = 0; if (!netif_msg_hw(adapter)) @@ -790,14 +789,12 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) pr_info("------------------------------------\n"); pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); pr_info("------------------------------------\n"); - pr_info("%s%s%s\n", + pr_info("%s%s\n", "R [desc] [ PktBuf A0] ", - "[ HeadBuf DD] [bi->dma ] [bi->skb ] ", - "<-- Adv Rx Read format"); - pr_info("%s%s%s\n", + "[ HeadBuf DD] <-- Adv Rx Read format"); + pr_info("%s%s\n", "RWB[desc] [PcsmIpSHl PtRs] ", - "[vl er S cks ln] ---------------- [bi->skb ] ", - "<-- Adv Rx Write-Back format"); + "[vl er S cks ln] <-- Adv Rx Write-Back format"); for (i = 0; i < rx_ring->count; i++) { const char *ring_desc; @@ -809,34 +806,30 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) else ring_desc = ""; - rx_buffer_info = &rx_ring->rx_buffer_info[i]; rx_desc = IXGBE_RX_DESC(rx_ring, i); u0 = (struct my_u0 *)rx_desc; if (rx_desc->wb.upper.length) { - /* Descriptor Done */ - pr_info("RWB[0x%03X] %016llX %016llX ---------------- %p%s\n", + pr_info("RWB[0x%03X] %016llX %016llX%s\n", i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - rx_buffer_info->skb, ring_desc); } else { - pr_info("R [0x%03X] %016llX %016llX %016llX %p%s\n", + pr_info("R [0x%03X] %016llX %016llX %016llX %s\n", i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - (u64)rx_buffer_info->dma, - rx_buffer_info->skb, + (u64)rx_ring->rx_buffer_info[i].dma, ring_desc); + } - if (netif_msg_pktdata(adapter) && - rx_buffer_info->dma) { - print_hex_dump(KERN_INFO, "", - DUMP_PREFIX_ADDRESS, 16, 1, - page_address(rx_buffer_info->page) + - rx_buffer_info->page_offset, - ixgbe_rx_bufsz(rx_ring), true); - } + if (netif_msg_pktdata(adapter) && + rx_ring->rx_buffer_info[i].dma) { + print_hex_dump(KERN_INFO, "", + DUMP_PREFIX_ADDRESS, 16, 1, + page_address(rx_ring->rx_buffer_info[i].page) + + rx_ring->rx_buffer_info[i].page_offset, + IXGBE_RXBUFFER_3K, true); } } } @@ -1736,29 +1729,19 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, struct page *page; dma_addr_t dma; - /* since we are recycling buffers we should seldom need to alloc */ - if (likely(bi->page)) - return true; - /* alloc new page for storage */ - page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring)); + page = dev_alloc_page(); if (unlikely(!page)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; } /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); + dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, + DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); - /* - * if mapping failed free memory back to system since - * there isn't much point in holding memory we can't use - */ if (dma_mapping_error(rx_ring->dev, dma)) { - __free_pages(page, ixgbe_rx_pg_order(rx_ring)); + __free_page(page); rx_ring->rx_stats.alloc_rx_page_failed++; return false; @@ -1766,9 +1749,8 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, bi->dma = dma; bi->page = page; - bi->page_offset = rx_ring->rx_offset; - page_ref_add(page, USHRT_MAX - 1); - bi->pagecnt_bias = USHRT_MAX; + bi->page_offset = rx_ring->xdp_prog ? XDP_PACKET_HEADROOM : + IXGBE_SKB_PAD; rx_ring->rx_stats.alloc_rx_page++; return true; @@ -1784,7 +1766,6 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *bi; u16 i = rx_ring->next_to_use; - u16 bufsz; /* nothing to do */ if (!cleaned_count) @@ -1794,15 +1775,14 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) bi = &rx_ring->rx_buffer_info[i]; i -= rx_ring->count; - bufsz = ixgbe_rx_bufsz(rx_ring); - do { if (!ixgbe_alloc_mapped_page(rx_ring, bi)) break; /* sync the buffer for use by the device */ dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, bufsz, + bi->page_offset, + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* @@ -1831,9 +1811,6 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) if (rx_ring->next_to_use != i) { rx_ring->next_to_use = i; - /* update next to alloc since we have filled the ring */ - rx_ring->next_to_alloc = i; - /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only * applicable for weak-ordered memory model archs, @@ -1967,8 +1944,6 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) return false; - /* place skb in next buffer to be received */ - rx_ring->rx_buffer_info[ntc].skb = skb; rx_ring->rx_stats.non_eop_descs++; return true; @@ -2016,34 +1991,6 @@ static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, skb->tail += pull_len; } -/** - * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB - * @rx_ring: rx descriptor ring packet is being transacted on - * @skb: pointer to current skb being updated - * - * This function provides a basic DMA sync up for the first fragment of an - * skb. The reason for doing this is that the first fragment cannot be - * unmapped until we have reached the end of packet descriptor for a buffer - * chain. - */ -static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring, - struct sk_buff *skb) -{ - unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1; - unsigned long offset = (unsigned long)(skb->data) & mask; - - dma_sync_single_range_for_cpu(rx_ring->dev, IXGBE_CB(skb)->dma, - offset, skb_headlen(skb), - DMA_FROM_DEVICE); - - /* If the page was released, just unmap it. */ - if (unlikely(IXGBE_CB(skb)->page_released)) { - dma_unmap_page_attrs(rx_ring->dev, IXGBE_CB(skb)->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - } -} /** * ixgbe_cleanup_headers - Correct corrupted or empty headers @@ -2094,80 +2041,10 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return false; #endif - /* if eth_skb_pad returns an error the skb was freed */ - if (eth_skb_pad(skb)) - return true; return false; } -/** - * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring - * @rx_ring: rx descriptor ring to store buffers on - * @old_buff: donor buffer to have page reused - * - * Synchronizes page for reuse by the adapter - **/ -static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *old_buff) -{ - struct ixgbe_rx_buffer *new_buff; - u16 nta = rx_ring->next_to_alloc; - - new_buff = &rx_ring->rx_buffer_info[nta]; - - /* update, and store next to alloc */ - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - /* Transfer page from old buffer to new buffer. - * Move each member individually to avoid possible store - * forwarding stalls and unnecessary copy of skb. - */ - new_buff->dma = old_buff->dma; - new_buff->page = old_buff->page; - new_buff->page_offset = old_buff->page_offset; - new_buff->pagecnt_bias = old_buff->pagecnt_bias; -} - -static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer, - int rx_buffer_pgcnt) -{ - unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; - struct page *page = rx_buffer->page; - - /* avoid re-using remote and pfmemalloc pages */ - if (!dev_page_is_reusable(page)) - return false; - -#if (PAGE_SIZE < 8192) - /* if we are only owner of page we can reuse it */ - if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1)) - return false; -#else - /* The last offset is a bit aggressive in that we assume the - * worst case of FCoE being enabled and using a 3K buffer. - * However this should have minimal impact as the 1K extra is - * still less than one buffer in size. - */ -#define IXGBE_LAST_OFFSET \ - (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K) - if (rx_buffer->page_offset > IXGBE_LAST_OFFSET) - return false; -#endif - - /* If we have drained the page fragment pool we need to update - * the pagecnt_bias and page count so that we fully restock the - * number of references the driver holds. - */ - if (unlikely(pagecnt_bias == 1)) { - page_ref_add(page, USHRT_MAX - 1); - rx_buffer->pagecnt_bias = USHRT_MAX; - } - - return true; -} - /** * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff * @rx_ring: rx descriptor ring to transact packets on @@ -2175,97 +2052,42 @@ static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer, * @skb: sk_buff to place the data into * @size: size of data in rx_buffer * - * This function attaches the page contained in rx_buffer to the skb as - * a fragment. It then updates the page offset based on the configured - * PAGE_SIZE to prepare the buffer for the next packet. + * This function will add the data contained in rx_buffer->page to the skb. **/ static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *rx_buffer, struct sk_buff *skb, unsigned int size) { -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else unsigned int truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size); -#endif + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, rx_buffer->page_offset, size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif } static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff **skb, - const unsigned int size, - int *rx_buffer_pgcnt) + const unsigned int size) { struct ixgbe_rx_buffer *rx_buffer; rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - *rx_buffer_pgcnt = -#if (PAGE_SIZE < 8192) - page_count(rx_buffer->page); -#else - 0; -#endif prefetchw(rx_buffer->page); - *skb = rx_buffer->skb; - - /* Delay unmapping of the first packet. It carries the header - * information, HW may still access the header after the writeback. - * Only unmap it when EOP is reached - */ - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) { - if (!*skb) - goto skip_sync; - } else { - if (*skb) - ixgbe_dma_sync_frag(rx_ring, *skb); - } - /* we are reusing so sync this buffer for CPU use */ + /* sync this buffer for CPU use */ dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, size, DMA_FROM_DEVICE); -skip_sync: - rx_buffer->pagecnt_bias--; return rx_buffer; } static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct sk_buff *skb, - int rx_buffer_pgcnt) + struct ixgbe_rx_buffer *rx_buffer) { - if (ixgbe_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) { - /* hand second half of page back to the ring */ - ixgbe_reuse_rx_page(rx_ring, rx_buffer); - } else { - if (skb && IXGBE_CB(skb)->dma == rx_buffer->dma) { - /* the page has been released from the ring */ - IXGBE_CB(skb)->page_released = true; - } else { - /* we are not reusing the buffer so unmap it */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - } - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); - } - - /* clear contents of rx_buffer */ - rx_buffer->page = NULL; - rx_buffer->skb = NULL; + dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE, + DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); } static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, @@ -2274,13 +2096,9 @@ static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc) { unsigned int metasize = xdp->data - xdp->data_meta; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + SKB_DATA_ALIGN(xdp->data_end - xdp->data_hard_start); -#endif struct sk_buff *skb; /* Prefetch first cache line of first page. If xdp->data_meta @@ -2305,13 +2123,6 @@ static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) IXGBE_CB(skb)->dma = rx_buffer->dma; - /* update buffer offset */ -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - return skb; } @@ -2370,31 +2181,6 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, return result; } -static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring, - unsigned int size) -{ - unsigned int truesize; - -#if (PAGE_SIZE < 8192) - truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ -#else - truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); -#endif - return truesize; -} - -static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - unsigned int size) -{ - unsigned int truesize = ixgbe_rx_frame_truesize(rx_ring, size); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif -} /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf @@ -2413,7 +2199,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *rx_ring, const int budget) { - unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; #ifdef IXGBE_FCOE int ddp_bytes; @@ -2422,19 +2208,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, u16 cleaned_count = ixgbe_desc_unused(rx_ring); unsigned int xdp_xmit = 0; struct xdp_buff xdp; + struct sk_buff *skb = rx_ring->skb; int xdp_res = 0; - /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ -#if (PAGE_SIZE < 8192) - frame_sz = ixgbe_rx_frame_truesize(rx_ring, 0); -#endif - xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); + xdp_init_buff(&xdp, IXGBE_RXBUFFER_3K, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *rx_buffer; - struct sk_buff *skb; - int rx_buffer_pgcnt; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -2454,31 +2235,24 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size, &rx_buffer_pgcnt); + rx_buffer = ixgbe_get_rx_buffer(rx_ring, size); /* retrieve a buffer from the ring */ if (!skb) { - unsigned int offset = rx_ring->rx_offset; + unsigned int offset = IXGBE_SKB_PAD; unsigned char *hard_start; hard_start = page_address(rx_buffer->page) + rx_buffer->page_offset - offset; xdp_prepare_buff(&xdp, hard_start, offset, size, true); xdp_buff_clear_frags_flag(&xdp); -#if (PAGE_SIZE > 4096) - /* At larger PAGE_SIZE, frame_sz depend on len size */ - xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, size); -#endif xdp_res = ixgbe_run_xdp(adapter, rx_ring, &xdp); } if (xdp_res) { - if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) { + if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) xdp_xmit |= xdp_res; - ixgbe_rx_buffer_flip(rx_ring, rx_buffer, size); - } else { - rx_buffer->pagecnt_bias++; - } + total_rx_packets++; total_rx_bytes += size; } else if (skb) { @@ -2491,11 +2265,13 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* exit if we failed to retrieve a buffer */ if (!xdp_res && !skb) { rx_ring->rx_stats.alloc_rx_buff_failed++; - rx_buffer->pagecnt_bias++; + cleaned_count++; break; } - ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt); + ixgbe_put_rx_buffer(rx_ring, rx_buffer); + if (xdp_res == IXGBE_XDP_CONSUMED) + __free_page(rx_buffer->page); cleaned_count++; /* place incomplete frames back on ring for completion */ @@ -2543,6 +2319,8 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_packets++; } + rx_ring->skb = skb; + if (xdp_xmit & IXGBE_XDP_REDIR) xdp_do_flush(); @@ -4086,10 +3864,8 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; else srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) { - srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; } else { - srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; + srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; } /* configure descriptor type */ @@ -4465,19 +4241,8 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, } else if (hw->mac.type != ixgbe_mac_82599EB) { rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); - - /* Limit the maximum frame size so we don't overrun the skb. - * This can happen in SRIOV mode when the MTU of the VF is - * higher than the MTU of the PF. - */ - if (!test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB | - IXGBE_RXDCTL_RLPML_EN; -#endif } - - ring->rx_offset = IXGBE_SKB_PAD; - +#endif if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); @@ -4487,11 +4252,9 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, ring->rx_buf_len = xsk_buf_len; } - /* initialize rx_buffer_info */ memset(ring->rx_buffer_info, 0, - sizeof(struct ixgbe_rx_buffer) * ring->count); - + sizeof(struct ixgbe_rx_buffer) * ring->count); /* initialize Rx descriptor 0 */ rx_desc = IXGBE_RX_DESC(ring, 0); rx_desc->wb.upper.length = 0; @@ -4608,8 +4371,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) struct ixgbe_hw *hw = &adapter->hw; struct net_device *netdev = adapter->netdev; int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - struct ixgbe_ring *rx_ring; - int i; u32 mhadd, hlreg0; #ifdef IXGBE_FCOE @@ -4617,7 +4378,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; - #endif /* IXGBE_FCOE */ /* adjust max frame to be at least the size of a standard frame */ @@ -4633,35 +4393,11 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) } hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); - /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */ + /* set jumbo enable since MHADD.MFS is keeping size locked at + * max_frame + */ hlreg0 |= IXGBE_HLREG0_JUMBOEN; IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); - - /* - * Setup the HW Rx Head and Tail Descriptor Pointers and - * the Base and Length of the Rx Descriptor Ring - */ - for (i = 0; i < adapter->num_rx_queues; i++) { - rx_ring = adapter->rx_ring[i]; - - clear_ring_rsc_enabled(rx_ring); - clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - - if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) - set_ring_rsc_enabled(rx_ring); - - if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state)) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - -#if (PAGE_SIZE < 8192) - if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - - if (IXGBE_2K_TOO_SMALL_WITH_PADDING || - (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN))) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); -#endif - } } static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) @@ -4746,8 +4482,15 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) * Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ - for (i = 0; i < adapter->num_rx_queues; i++) - ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]); + for (i = 0; i < adapter->num_rx_queues; i++) { + struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; + + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + + ixgbe_configure_rx_ring(adapter, rx_ring); + } rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); /* disable drop enable for 82598 parts */ @@ -5653,35 +5396,28 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) goto skip_free; } - /* Free all the Rx ring sk_buffs */ - while (i != rx_ring->next_to_alloc) { - if (rx_buffer->skb) { - struct sk_buff *skb = rx_buffer->skb; - if (IXGBE_CB(skb)->page_released) - dma_unmap_page_attrs(rx_ring->dev, - IXGBE_CB(skb)->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - dev_kfree_skb(skb); - } + /* Free Rx ring sk_buff */ + if (rx_ring->skb) { + dev_kfree_skb(rx_ring->skb); + rx_ring->skb = NULL; + } + /* Free all the Rx ring pages */ + while (i != rx_ring->next_to_use) { /* Invalidate cache lines that may have been written to by * device so that we avoid corrupting memory. */ dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, - ixgbe_rx_bufsz(rx_ring), + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* free resources associated with mapping */ dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, + PAGE_SIZE, DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); + __free_page(rx_buffer->page); i++; rx_buffer++; @@ -5692,7 +5428,6 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) } skip_free: - rx_ring->next_to_alloc = 0; rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; } @@ -7241,10 +6976,7 @@ static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) */ static int ixgbe_max_xdp_frame_size(struct ixgbe_adapter *adapter) { - if (PAGE_SIZE >= 8192) - return IXGBE_RXBUFFER_2K; - else - return IXGBE_RXBUFFER_3K; + return IXGBE_RXBUFFER_3K; } /** @@ -10809,11 +10541,11 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (ring_is_rsc_enabled(ring)) return -EINVAL; - - if (frame_size > ixgbe_rx_bufsz(ring)) - return -EINVAL; } + if (frame_size > IXGBE_RXBUFFER_3K) + return -EINVAL; + /* if the number of cpus is much larger than the maximum of queues, * we should stop it and then return with ENOMEM like before. */ @@ -11179,6 +10911,11 @@ void ixgbe_txrx_ring_enable(struct ixgbe_adapter *adapter, int ring) ixgbe_configure_tx_ring(adapter, tx_ring); if (xdp_ring) ixgbe_configure_tx_ring(adapter, xdp_ring); + + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + ixgbe_configure_rx_ring(adapter, rx_ring); clear_bit(__IXGBE_TX_DISABLED, &tx_ring->state); From 87fdded1d393157e83652d4854b2005311f45a04 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Thu, 7 May 2026 23:34:48 +0200 Subject: [PATCH 18/20] ixgbe: use libeth in Rx processing Convert ixgbe Rx path from manual page-per-packet DMA management to libeth/page_pool infrastructure, and integrates libeth XDP helpers. Each packet gets its own buffer instead of sharing pages between packets. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/Kconfig | 1 + drivers/net/ethernet/intel/ixgbe/ixgbe.h | 35 +- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 23 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 1134 ++++++++--------- drivers/net/ethernet/intel/ixgbe/ixgbe_type.h | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 19 +- 6 files changed, 552 insertions(+), 662 deletions(-) diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig index b513baf3cbb296..072133b4a4416d 100644 --- a/drivers/net/ethernet/intel/Kconfig +++ b/drivers/net/ethernet/intel/Kconfig @@ -146,6 +146,7 @@ config IXGBE tristate "Intel(R) 10GbE PCI Express adapters support" depends on PCI depends on PTP_1588_CLOCK_OPTIONAL + select LIBETH_XDP select LIBIE_FWLOG if DEBUG_FS select MDIO select NET_DEVLINK diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index ec9390cd457f84..e228baef36acea 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -33,6 +33,7 @@ #include "ixgbe_ipsec.h" #include +#include /* common prefix used by pr_<> macros */ #undef pr_fmt @@ -73,9 +74,8 @@ #define IXGBE_RXBUFFER_4K 4096 #define IXGBE_MAX_RXBUFFER 16384 /* largest size for a single descriptor */ -#define IXGBE_PKT_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2)) - -#define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) +#define IXGBE_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) /* * RX header buffer size for page-based allocation using napi_build_skb. @@ -215,18 +215,9 @@ struct ixgbe_tx_buffer { u32 tx_flags; }; -struct ixgbe_rx_buffer { - union { - struct { - dma_addr_t dma; - struct page *page; - __u32 page_offset; - }; - struct { - bool discard; - struct xdp_buff *xdp; - }; - }; +struct ixgbe_xsk_rx_buffer { + bool discard; + struct xdp_buff *xdp; }; struct ixgbe_queue_stats { @@ -294,17 +285,22 @@ struct ixgbe_ring { struct ixgbe_ring *next; /* pointer to next ring in q_vector */ struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */ struct net_device *netdev; /* netdev ring belongs to */ - struct bpf_prog *xdp_prog; - struct device *dev; /* device for DMA mapping */ + struct bpf_prog __rcu *xdp_prog; + union { + struct page_pool *pp; /* Rx ring */ + struct device *dev; /* Tx ring */ + }; void *desc; /* descriptor ring memory */ union { + struct libeth_fqe *rx_fqes; + struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; struct ixgbe_tx_buffer *tx_buffer_info; - struct ixgbe_rx_buffer *rx_buffer_info; }; unsigned long state; u8 __iomem *tail; dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ + u32 truesize; u16 count; /* amount of descriptors */ @@ -336,7 +332,7 @@ struct ixgbe_ring { spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ - u16 rx_buf_len; + u32 rx_buf_len; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { @@ -380,6 +376,7 @@ struct ixgbe_ring_feature { #define IXGBE_82599_VMDQ_4Q_MASK 0x7C #define IXGBE_82599_VMDQ_2Q_MASK 0x7E + #define IXGBE_ITR_ADAPTIVE_MIN_INC 2 #define IXGBE_ITR_ADAPTIVE_MIN_USECS 10 #define IXGBE_ITR_ADAPTIVE_MAX_USECS 126 diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index b9db6ee9668e93..a2cc9b39eb634c 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1949,7 +1949,6 @@ static int ixgbe_setup_desc_rings(struct ixgbe_adapter *adapter) /* Setup Rx Descriptor ring and Rx buffers */ rx_ring->count = IXGBE_DEFAULT_RXD; rx_ring->queue_index = 0; - rx_ring->dev = &adapter->pdev->dev; rx_ring->netdev = adapter->netdev; rx_ring->reg_idx = adapter->rx_ring[0]->reg_idx; @@ -2060,14 +2059,16 @@ static void ixgbe_create_lbtest_frame(struct sk_buff *skb, skb->data[frame_size + 12] = 0xAF; } -static bool ixgbe_check_lbtest_frame(struct ixgbe_rx_buffer *rx_buffer, +static bool ixgbe_check_lbtest_frame(const struct libeth_fqe *rx_buffer, unsigned int frame_size) { + u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; unsigned char *data; frame_size >>= 1; - data = page_address(rx_buffer->page) + rx_buffer->page_offset; + data = page_address(__netmem_to_page(rx_buffer->netmem)) + + rx_buffer->offset + hr; return data[3] == 0xFF && data[frame_size + 10] == 0xBE && data[frame_size + 12] == 0xAF; @@ -2115,16 +2116,13 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, } while (rx_desc->wb.upper.length) { - struct ixgbe_rx_buffer *rx_buffer; + struct libeth_fqe *rx_buffer; /* check Rx buffer */ - rx_buffer = &rx_ring->rx_buffer_info[rx_ntc]; + rx_buffer = &rx_ring->rx_fqes[rx_ntc]; /* sync Rx buffer for CPU read */ - dma_sync_single_for_cpu(rx_ring->dev, - rx_buffer->dma, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); + libeth_rx_sync_for_cpu(rx_buffer, rx_ring->rx_buf_len); /* verify contents of skb */ if (ixgbe_check_lbtest_frame(rx_buffer, size)) @@ -2132,11 +2130,8 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, else break; - /* sync Rx buffer for device write */ - dma_sync_single_for_device(rx_ring->dev, - rx_buffer->dma, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); + /* recycle the page back to the pool */ + libeth_rx_recycle_slow(rx_buffer->netmem); /* increment Rx next to clean counter */ rx_ntc++; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 7d18cefb77c987..db2b2a5800904e 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -54,13 +55,13 @@ char ixgbe_driver_name[] = "ixgbe"; static const char ixgbe_driver_string[] = - "Intel(R) 10 Gigabit PCI Express Network Driver"; + "Intel(R) 10 Gigabit PCI Express Network Driver"; #ifdef IXGBE_FCOE char ixgbe_default_device_descr[] = - "Intel(R) 10 Gigabit Network Connection"; + "Intel(R) 10 Gigabit Network Connection"; #else static char ixgbe_default_device_descr[] = - "Intel(R) 10 Gigabit Network Connection"; + "Intel(R) 10 Gigabit Network Connection"; #endif static const char ixgbe_copyright[] = "Copyright (c) 1999-2016 Intel Corporation."; @@ -147,7 +148,7 @@ MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl); #ifdef CONFIG_IXGBE_DCA static int ixgbe_notify_dca(struct notifier_block *, unsigned long event, - void *p); + void *p); static struct notifier_block dca_notifier = { .notifier_call = ixgbe_notify_dca, .next = NULL, @@ -174,6 +175,7 @@ MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); MODULE_IMPORT_NS("LIBIE_FWLOG"); MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); +MODULE_IMPORT_NS("LIBETH"); MODULE_LICENSE("GPL v2"); DEFINE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key); @@ -212,7 +214,7 @@ static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter, pcie_capability_read_word(parent_dev, reg, value); if (*value == IXGBE_FAILED_READ_CFG_WORD && - ixgbe_check_cfg_remove(&adapter->hw, parent_dev)) + ixgbe_check_cfg_remove(&adapter->hw, parent_dev)) return -1; return 0; } @@ -264,7 +266,7 @@ static bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw) } static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter, - int expected_gts) + int expected_gts) { struct ixgbe_hw *hw = &adapter->hw; struct pci_dev *pdev; @@ -288,8 +290,8 @@ static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter, static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter) { if (!test_bit(__IXGBE_DOWN, &adapter->state) && - !test_bit(__IXGBE_REMOVING, &adapter->state) && - !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) + !test_bit(__IXGBE_REMOVING, &adapter->state) && + !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) queue_work(ixgbe_wq, &adapter->service_task); } @@ -354,7 +356,7 @@ u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg) if (ixgbe_removed(reg_addr)) return IXGBE_FAILED_READ_REG; if (unlikely(hw->phy.nw_mng_if_sel & - IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) { + IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) { struct ixgbe_adapter *adapter; int i; @@ -401,7 +403,7 @@ u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg) return IXGBE_FAILED_READ_CFG_WORD; pci_read_config_word(adapter->pdev, reg, &value); if (value == IXGBE_FAILED_READ_CFG_WORD && - ixgbe_check_cfg_remove(hw, adapter->pdev)) + ixgbe_check_cfg_remove(hw, adapter->pdev)) return IXGBE_FAILED_READ_CFG_WORD; return value; } @@ -416,7 +418,7 @@ static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg) return IXGBE_FAILED_READ_CFG_DWORD; pci_read_config_dword(adapter->pdev, reg, &value); if (value == IXGBE_FAILED_READ_CFG_DWORD && - ixgbe_check_cfg_remove(hw, adapter->pdev)) + ixgbe_check_cfg_remove(hw, adapter->pdev)) return IXGBE_FAILED_READ_CFG_DWORD; return value; } @@ -612,7 +614,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) dev_info(&adapter->pdev->dev, "Register Dump\n"); pr_info(" Register Name Value\n"); for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl; - reginfo->name; reginfo++) { + reginfo->name; reginfo++) { ixgbe_regdump(hw, reginfo); } @@ -693,7 +695,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) const char *ring_desc; if (i == ring->next_to_use && - i == ring->next_to_clean) + i == ring->next_to_clean) ring_desc = " NTC/U"; else if (i == ring->next_to_use) ring_desc = " NTU"; @@ -713,7 +715,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) ring_desc); if (netif_msg_pktdata(adapter) && - tx_buffer->skb) + tx_buffer->skb) print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 16, 1, tx_buffer->skb->data, @@ -910,7 +912,7 @@ static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction, } void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter, - u64 qmask) + u64 qmask) { u32 mask; @@ -943,7 +945,7 @@ static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter) u32 data; if ((hw->fc.current_mode != ixgbe_fc_full) && - (hw->fc.current_mode != ixgbe_fc_rx_pause)) + (hw->fc.current_mode != ixgbe_fc_rx_pause)) return; switch (hw->mac.type) { @@ -1127,7 +1129,7 @@ static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter) * @maxrate: desired maximum transmit bitrate **/ static int ixgbe_tx_maxrate(struct net_device *netdev, - int queue_index, u32 maxrate) + int queue_index, u32 maxrate) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -1199,7 +1201,7 @@ void ixgbe_update_rx_ring_stats(struct ixgbe_ring *rx_ring, * Prints a message containing details about the tx hang. */ static void ixgbe_pf_handle_tx_hang(struct ixgbe_ring *tx_ring, - unsigned int next) + unsigned int next) { struct ixgbe_adapter *adapter = netdev_priv(tx_ring->netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -1212,14 +1214,14 @@ static void ixgbe_pf_handle_tx_hang(struct ixgbe_ring *tx_ring, "tx_buffer_info[next_to_clean]\n" " time_stamp <%lx>\n" " jiffies <%lx>\n", - tx_ring->queue_index, - IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), - IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), - tx_ring->next_to_use, next, - tx_ring->tx_buffer_info[next].time_stamp, jiffies); + tx_ring->queue_index, + IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), + IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), + tx_ring->next_to_use, next, + tx_ring->tx_buffer_info[next].time_stamp, jiffies); netif_stop_subqueue(tx_ring->netdev, - tx_ring->queue_index); + tx_ring->queue_index); } /** @@ -1238,8 +1240,8 @@ static void ixgbe_vf_handle_tx_hang(struct ixgbe_adapter *adapter, u16 vf) return; e_warn(drv, - "Malicious Driver Detection tx hang detected on PF %d VF %d MAC: %pM", - hw->bus.func, vf, adapter->vfinfo[vf].vf_mac_addresses); + "Malicious Driver Detection tx hang detected on PF %d VF %d MAC: %pM", + hw->bus.func, vf, adapter->vfinfo[vf].vf_mac_addresses); adapter->tx_hang_count[vf]++; if (adapter->tx_hang_count[vf] == IXGBE_MAX_TX_VF_HANGS) { @@ -1266,7 +1268,7 @@ static u32 ixgbe_poll_tx_icache(struct ixgbe_hw *hw, u16 queue, u16 idx) * Returns: true if queue contain illegal packet. */ static bool ixgbe_check_illegal_queue(struct ixgbe_adapter *adapter, - u16 queue) + u16 queue) { u32 hdr_len_reg, mss_len_reg, type_reg; struct ixgbe_hw *hw = &adapter->hw; @@ -1289,13 +1291,13 @@ static bool ixgbe_check_illegal_queue(struct ixgbe_adapter *adapter, /* check if Advanced Context Descriptor */ if (FIELD_GET(IXGBE_ADVTXD_DTYP_MASK, type_reg) != - IXGBE_ADVTXD_DTYP_CTXT) + IXGBE_ADVTXD_DTYP_CTXT) continue; /* check for illegal MSS and Header length */ mss_len = FIELD_GET(IXGBE_ADVTXD_MSS_MASK, mss_len_reg); header_len = FIELD_GET(IXGBE_ADVTXD_HEADER_LEN_MASK, - hdr_len_reg); + hdr_len_reg); if ((mss_len + header_len) > SZ_16K) { e_warn(probe, "mss len + header len too long\n"); return true; @@ -1326,7 +1328,7 @@ static void ixgbe_handle_mdd_event(struct ixgbe_adapter *adapter, /* malicious vf not detected */ for (q = 0; q < IXGBE_MAX_TX_QUEUES; q++) { if (ixgbe_check_illegal_queue(adapter, q) && - !ixgbe_get_vf_idx(adapter, q, &vf)) + !ixgbe_get_vf_idx(adapter, q, &vf)) /* illegal queue detected */ ixgbe_vf_handle_tx_hang(adapter, vf); } @@ -1340,7 +1342,7 @@ static void ixgbe_handle_mdd_event(struct ixgbe_adapter *adapter, * @napi_budget: Used to determine if we are in netpoll **/ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring *tx_ring, int napi_budget) + struct ixgbe_ring *tx_ring, int napi_budget) { struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_tx_buffer *tx_buffer; @@ -1409,9 +1411,9 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, /* unmap any remaining paged data */ if (dma_unmap_len(tx_buffer, len)) { dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buffer, len, 0); } } @@ -1449,7 +1451,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, ixgbe_pf_handle_tx_hang(tx_ring, i); e_info(probe, - "tx hang %d detected on queue %d, resetting adapter\n", + "tx hang %d detected on queue %d, resetting adapter\n", adapter->tx_timeout_count + 1, tx_ring->queue_index); /* schedule immediate reset if we believe we hung */ @@ -1518,7 +1520,7 @@ static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, u8 reg_idx = rx_ring->reg_idx; if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) - rxctrl = dca3_get_tag(rx_ring->dev, cpu); + rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu); switch (hw->mac.type) { case ixgbe_mac_82599EB: @@ -1636,8 +1638,8 @@ static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, return; skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), - (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? - PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); + (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? + PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); } #ifdef IXGBE_FCOE @@ -1649,14 +1651,14 @@ static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, * Returns : true if it is FCoE pkt */ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, - union ixgbe_adv_rx_desc *rx_desc) + union ixgbe_adv_rx_desc *rx_desc) { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; return test_bit(__IXGBE_RX_FCOE, &ring->state) && - ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == + ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE << - IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); + IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); } #endif /* IXGBE_FCOE */ @@ -1667,8 +1669,8 @@ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, * @skb: skb currently being received and modified **/ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; bool encap_pkt = false; @@ -1687,7 +1689,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, /* if IP and error */ if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && - ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { + ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { ring->rx_stats.csum_err++; return; } @@ -1701,7 +1703,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, * checksum errors. */ if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) && - test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) + test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) return; ring->rx_stats.csum_err++; @@ -1723,39 +1725,6 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, } } -static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *bi) -{ - struct page *page; - dma_addr_t dma; - - /* alloc new page for storage */ - page = dev_alloc_page(); - if (unlikely(!page)) { - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, - DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); - - if (dma_mapping_error(rx_ring->dev, dma)) { - __free_page(page); - - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - bi->dma = dma; - bi->page = page; - bi->page_offset = rx_ring->xdp_prog ? XDP_PACKET_HEADROOM : - IXGBE_SKB_PAD; - rx_ring->rx_stats.alloc_rx_page++; - - return true; -} - /** * ixgbe_alloc_rx_buffers - Replace used receive buffers * @rx_ring: ring to place buffers on @@ -1764,40 +1733,34 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; - u16 i = rx_ring->next_to_use; + const struct libeth_fq_fp fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + .truesize = rx_ring->truesize, + .count = rx_ring->count, + }; + u16 ntu = rx_ring->next_to_use; /* nothing to do */ if (!cleaned_count) return; - - rx_desc = IXGBE_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; - i -= rx_ring->count; + rx_desc = IXGBE_RX_DESC(rx_ring, ntu); do { - if (!ixgbe_alloc_mapped_page(rx_ring, bi)) - break; + dma_addr_t addr; - /* sync the buffer for use by the device */ - dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); + addr = libeth_rx_alloc(&fq, ntu); + if (addr == DMA_MAPPING_ERROR) + break; - /* - * Refresh the desc even if buffer_addrs didn't change - * because each write-back erases this info. - */ - rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); + rx_desc->read.pkt_addr = cpu_to_le64(addr); rx_desc++; - bi++; - i++; - if (unlikely(!i)) { + ntu++; + + if (unlikely(ntu == rx_ring->count)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; - i -= rx_ring->count; + ntu = 0; } /* clear the length for the next_to_use descriptor */ @@ -1806,18 +1769,16 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) cleaned_count--; } while (cleaned_count); - i += rx_ring->count; - - if (rx_ring->next_to_use != i) { - rx_ring->next_to_use = i; - + if (likely(rx_ring->next_to_use != ntu)) { + /* record the next descriptor to use */ + rx_ring->next_to_use = ntu; /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only * applicable for weak-ordered memory model archs, * such as IA-64). */ wmb(); - writel(i, rx_ring->tail); + writel(ntu, rx_ring->tail); } } @@ -1859,8 +1820,8 @@ static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring, * other fields within the skb. **/ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { struct net_device *dev = rx_ring->netdev; u32 flags = rx_ring->q_vector->adapter->flags; @@ -1875,7 +1836,7 @@ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb); if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && - ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { + ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); } @@ -1889,8 +1850,6 @@ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, else macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true, false); - - skb->protocol = eth_type_trans(skb, dev); } void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, @@ -1911,8 +1870,8 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, * that this is in fact a non-EOP buffer. **/ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { u32 ntc = rx_ring->next_to_clean + 1; @@ -1925,7 +1884,7 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, /* update RSC append count if present */ if (ring_is_rsc_enabled(rx_ring)) { __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data & - cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); + cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); if (unlikely(rsc_enabled)) { u32 rsc_cnt = le32_to_cpu(rsc_enabled); @@ -1962,7 +1921,7 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, * truesize for the skb. */ static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, - struct sk_buff *skb) + struct sk_buff *skb) { skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; unsigned char *va; @@ -1991,7 +1950,6 @@ static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, skb->tail += pull_len; } - /** * ixgbe_cleanup_headers - Correct corrupted or empty headers * @rx_ring: rx descriptor ring packet is being transacted on @@ -2024,9 +1982,9 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, * errors that would be unacceptable to the netdev. */ if (!netdev || - (unlikely(ixgbe_test_staterr(rx_desc, + (unlikely(ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_FRAME_ERR_MASK) && - !(netdev->features & NETIF_F_RXALL)))) { + !(netdev->features & NETIF_F_RXALL)))) { dev_kfree_skb_any(skb); return true; } @@ -2053,82 +2011,28 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, * @size: size of data in rx_buffer * * This function will add the data contained in rx_buffer->page to the skb. + * This is done either through a direct copy if the data in the buffer is + * less than the skb header size, otherwise it will just attach the page as + * a frag to the skb. + * + * The function will then update the page offset if necessary and return + * true if the buffer can be reused by the adapter. **/ -static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct sk_buff *skb, - unsigned int size) +static void ixgbe_add_rx_frag(const struct libeth_fqe *rx_buffer, + struct sk_buff *skb, + unsigned int size) { - unsigned int truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size); + u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, - rx_buffer->page_offset, size, truesize); + skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, + rx_buffer->netmem, rx_buffer->offset + hr, + size, rx_buffer->truesize); } -static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring, - const unsigned int size) -{ - struct ixgbe_rx_buffer *rx_buffer; - - rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - prefetchw(rx_buffer->page); - - /* sync this buffer for CPU use */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - size, - DMA_FROM_DEVICE); - - return rx_buffer; -} - -static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer) -{ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE, - DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); -} - -static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int metasize = xdp->data - xdp->data_meta; - unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + - SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); - struct sk_buff *skb; - - /* Prefetch first cache line of first page. If xdp->data_meta - * is unused, this points exactly as xdp->data, otherwise we - * likely have a consumer accessing first few bytes of meta - * data, and then actual data. - */ - net_prefetch(xdp->data_meta); - - /* build an skb to around the page buffer */ - skb = napi_build_skb(xdp->data_hard_start, truesize); - if (unlikely(!skb)) - return NULL; - - /* update pointers within the skb to store the data */ - skb_reserve(skb, xdp->data - xdp->data_hard_start); - __skb_put(skb, xdp->data_end - xdp->data); - if (metasize) - skb_metadata_set(skb, metasize); - - /* record DMA address if this is the start of a chain of buffers */ - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) - IXGBE_CB(skb)->dma = rx_buffer->dma; - - return skb; -} static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, - struct xdp_buff *xdp) + struct libeth_xdp_buff *xdp) { int err, result = IXGBE_XDP_PASS; struct bpf_prog *xdp_prog; @@ -2141,14 +2045,14 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, if (!xdp_prog) goto xdp_out; - prefetchw(xdp->data_hard_start); /* xdp_frame write */ + prefetchw(xdp->base.data_hard_start); /* xdp_frame write */ - act = bpf_prog_run_xdp(xdp_prog, xdp); + act = bpf_prog_run_xdp(xdp_prog, &xdp->base); switch (act) { case XDP_PASS: break; case XDP_TX: - xdpf = xdp_convert_buff_to_frame(xdp); + xdpf = xdp_convert_buff_to_frame(&xdp->base); if (unlikely(!xdpf)) goto out_failure; ring = ixgbe_determine_xdp_ring(adapter); @@ -2161,7 +2065,7 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, goto out_failure; break; case XDP_REDIRECT: - err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog); + err = xdp_do_redirect(adapter->netdev, &xdp->base, xdp_prog); if (err) goto out_failure; result = IXGBE_XDP_REDIR; @@ -2175,13 +2079,13 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: result = IXGBE_XDP_CONSUMED; + libeth_xdp_return_buff(xdp); break; } xdp_out: return result; } - /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @q_vector: structure containing interrupt and ring information @@ -2196,8 +2100,8 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, * Returns amount of work completed **/ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring *rx_ring, - const int budget) + struct ixgbe_ring *rx_ring, + const int budget) { unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; @@ -2206,16 +2110,17 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, unsigned int mss = 0; #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); - unsigned int xdp_xmit = 0; - struct xdp_buff xdp; + LIBETH_XDP_ONSTACK_BUFF(xdp); struct sk_buff *skb = rx_ring->skb; + unsigned int xdp_xmit = 0; int xdp_res = 0; - xdp_init_buff(&xdp, IXGBE_RXBUFFER_3K, &rx_ring->xdp_rxq); + xdp->data = NULL; + xdp->base.rxq = &rx_ring->xdp_rxq; while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *rx_buffer; + struct libeth_fqe *rx_buffer; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -2226,7 +2131,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); - if (!size) + if (unlikely(!size)) break; /* This memory barrier is needed to keep us from reading @@ -2235,20 +2140,17 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - rx_buffer = ixgbe_get_rx_buffer(rx_ring, size); + rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; /* retrieve a buffer from the ring */ if (!skb) { - unsigned int offset = IXGBE_SKB_PAD; - unsigned char *hard_start; - - hard_start = page_address(rx_buffer->page) + - rx_buffer->page_offset - offset; - xdp_prepare_buff(&xdp, hard_start, offset, size, true); - xdp_buff_clear_frags_flag(&xdp); - xdp_res = ixgbe_run_xdp(adapter, rx_ring, &xdp); + libeth_xdp_process_buff(xdp, rx_buffer, size); + xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); + } else { + libeth_rx_sync_for_cpu(rx_buffer, size); } + if (xdp_res) { if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) xdp_xmit |= xdp_res; @@ -2256,22 +2158,19 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_packets++; total_rx_bytes += size; } else if (skb) { - ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size); + ixgbe_add_rx_frag(rx_buffer, skb, size); } else { - skb = ixgbe_build_skb(rx_ring, rx_buffer, - &xdp, rx_desc); + skb = xdp_build_skb_from_buff(&xdp->base); } /* exit if we failed to retrieve a buffer */ - if (!xdp_res && !skb) { + if (unlikely(!xdp_res && !skb)) { + libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; cleaned_count++; break; } - ixgbe_put_rx_buffer(rx_ring, rx_buffer); - if (xdp_res == IXGBE_XDP_CONSUMED) - __free_page(rx_buffer->page); cleaned_count++; /* place incomplete frames back on ring for completion */ @@ -2279,8 +2178,10 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, continue; /* verify the packet layout is correct */ - if (xdp_res || ixgbe_cleanup_headers(rx_ring, rx_desc, skb)) + if (xdp_res || unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { + skb = NULL; continue; + } /* probably a little skewed due to removing CRC */ total_rx_bytes += skb->len; @@ -2308,12 +2209,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, } if (!ddp_bytes) { dev_kfree_skb_any(skb); + skb = NULL; continue; } } #endif /* IXGBE_FCOE */ ixgbe_rx_skb(q_vector, skb); + skb = NULL; /* update budget accounting */ total_rx_packets++; @@ -2375,7 +2278,7 @@ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX, - v_idx); + v_idx); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: @@ -2416,7 +2319,7 @@ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) * while increasing bulk throughput. **/ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring_container *ring_container) + struct ixgbe_ring_container *ring_container) { unsigned int itr = IXGBE_ITR_ADAPTIVE_MIN_USECS | IXGBE_ITR_ADAPTIVE_LATENCY; @@ -2565,8 +2468,8 @@ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, case IXGBE_LINK_SPEED_100_FULL: default: itr += DIV_ROUND_UP(avg_wire_size, - IXGBE_ITR_ADAPTIVE_MIN_INC * 256) * - IXGBE_ITR_ADAPTIVE_MIN_INC; + IXGBE_ITR_ADAPTIVE_MIN_INC * 256) * + IXGBE_ITR_ADAPTIVE_MIN_INC; break; case IXGBE_LINK_SPEED_2_5GB_FULL: case IXGBE_LINK_SPEED_1GB_FULL: @@ -2574,8 +2477,8 @@ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, if (avg_wire_size > 8064) avg_wire_size = 8064; itr += DIV_ROUND_UP(avg_wire_size, - IXGBE_ITR_ADAPTIVE_MIN_INC * 64) * - IXGBE_ITR_ADAPTIVE_MIN_INC; + IXGBE_ITR_ADAPTIVE_MIN_INC * 64) * + IXGBE_ITR_ADAPTIVE_MIN_INC; break; } @@ -2677,7 +2580,7 @@ static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter) * check if we got a LSC */ if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) && - !(eicr & IXGBE_EICR_LSC)) + !(eicr & IXGBE_EICR_LSC)) return; if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) { @@ -2717,7 +2620,7 @@ static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr) struct ixgbe_hw *hw = &adapter->hw; if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && - (eicr & IXGBE_EICR_GPI_SDP1(hw))) { + (eicr & IXGBE_EICR_GPI_SDP1(hw))) { e_crit(probe, "Fan has stopped, replace the adapter\n"); /* write to clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw)); @@ -2738,8 +2641,8 @@ static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr) * on service task */ if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) || - (eicr & IXGBE_EICR_LSC)) && - (!test_bit(__IXGBE_DOWN, &adapter->state))) { + (eicr & IXGBE_EICR_LSC)) && + (!test_bit(__IXGBE_DOWN, &adapter->state))) { adapter->interrupt_event = eicr; adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT; ixgbe_service_event_schedule(adapter); @@ -2814,7 +2717,7 @@ static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr) } if (adapter->hw.mac.type == ixgbe_mac_82599EB && - (eicr & IXGBE_EICR_GPI_SDP1(hw))) { + (eicr & IXGBE_EICR_GPI_SDP1(hw))) { /* Clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw)); if (!test_bit(__IXGBE_DOWN, &adapter->state)) { @@ -2846,7 +2749,7 @@ static void ixgbe_check_lsc(struct ixgbe_adapter *adapter) * Check if external PHY FW load failed and print an error message if it did. */ static void ixgbe_check_phy_fw_load(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { if (!(link_cfg_err & IXGBE_ACI_LINK_EXTERNAL_PHY_LOAD_FAILURE)) { adapter->flags2 &= ~IXGBE_FLAG2_PHY_FW_LOAD_FAILED; @@ -2871,11 +2774,11 @@ static void ixgbe_check_phy_fw_load(struct ixgbe_adapter *adapter, * and print error messages if module power level is not supported. */ static void ixgbe_check_module_power(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { /* If module power level is supported, clear the flag. */ if (!(link_cfg_err & (IXGBE_ACI_LINK_INVAL_MAX_POWER_LIMIT | - IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED))) { + IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED))) { adapter->flags2 &= ~IXGBE_FLAG2_MOD_POWER_UNSUPPORTED; return; } @@ -2904,7 +2807,7 @@ static void ixgbe_check_module_power(struct ixgbe_adapter *adapter, * link_cfg_err parameter in the link info structure. */ static void ixgbe_check_link_cfg_err(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { ixgbe_check_module_power(adapter, link_cfg_err); ixgbe_check_phy_fw_load(adapter, link_cfg_err); @@ -2943,12 +2846,12 @@ ixgbe_process_link_status_event(struct ixgbe_adapter *adapter, bool link_up, /* Turn off PHY if media was removed. */ if (!(adapter->flags2 & IXGBE_FLAG2_NO_MEDIA) && - !(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE)) + !(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE)) adapter->flags2 |= IXGBE_FLAG2_NO_MEDIA; if (link_up == adapter->link_up && - link_up == netif_carrier_ok(adapter->netdev) && - link_speed == adapter->link_speed) + link_up == netif_carrier_ok(adapter->netdev) && + link_speed == adapter->link_speed) return 0; adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; @@ -2970,7 +2873,7 @@ ixgbe_process_link_status_event(struct ixgbe_adapter *adapter, bool link_up, */ static void ixgbe_handle_link_status_event(struct ixgbe_adapter *adapter, - struct ixgbe_aci_event *e) + struct ixgbe_aci_event *e) { struct ixgbe_aci_cmd_get_link_status_data *link_data; u16 link_speed; @@ -2995,8 +2898,8 @@ ixgbe_handle_link_status_event(struct ixgbe_adapter *adapter, static void ixgbe_schedule_fw_event(struct ixgbe_adapter *adapter) { if (!test_bit(__IXGBE_DOWN, &adapter->state) && - !test_bit(__IXGBE_REMOVING, &adapter->state) && - !test_bit(__IXGBE_RESETTING, &adapter->state)) { + !test_bit(__IXGBE_REMOVING, &adapter->state) && + !test_bit(__IXGBE_RESETTING, &adapter->state)) { adapter->flags2 |= IXGBE_FLAG2_FW_ASYNC_EVENT; ixgbe_service_event_schedule(adapter); } @@ -3049,7 +2952,7 @@ static void ixgbe_handle_fw_event(struct ixgbe_adapter *adapter) break; case libie_aqc_opc_fw_logs_event: libie_get_fwlog_data(&hw->fwlog, event.msg_buf, - le16_to_cpu(event.desc.datalen)); + le16_to_cpu(event.desc.datalen)); break; default: e_warn(hw, "unknown FW async event captured\n"); @@ -3095,7 +2998,7 @@ static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter, * @flush: flush register write **/ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, - bool flush) + bool flush) { struct ixgbe_hw *hw = &adapter->hw; u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); @@ -3133,8 +3036,8 @@ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, fallthrough; case ixgbe_mac_x550em_a: if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP || - adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP || - adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) + adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP || + adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw); if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t) mask |= IXGBE_EICR_GPI_SDP0_X540; @@ -3146,7 +3049,7 @@ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, } if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && - !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) + !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) mask |= IXGBE_EIMS_FLOW_DIR; IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask); @@ -3198,7 +3101,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data) case ixgbe_mac_x550em_a: case ixgbe_mac_e610: if (hw->phy.type == ixgbe_phy_x550em_ext_t && - (eicr & IXGBE_EICR_GPI_SDP0_X540)) { + (eicr & IXGBE_EICR_GPI_SDP0_X540)) { adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT; ixgbe_service_event_schedule(adapter); IXGBE_WRITE_REG(hw, IXGBE_EICR, @@ -3217,7 +3120,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data) for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *ring = adapter->tx_ring[i]; if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE, - &ring->state)) + &ring->state)) reinit_count++; } if (reinit_count) { @@ -3301,9 +3204,9 @@ int ixgbe_poll(struct napi_struct *napi, int budget) ixgbe_for_each_ring(ring, q_vector->rx) { int cleaned = ring->xsk_pool ? - ixgbe_clean_rx_irq_zc(q_vector, ring, - per_ring_budget) : - ixgbe_clean_rx_irq(q_vector, ring, + ixgbe_clean_rx_irq_zc(q_vector, ring, + per_ring_budget) : + ixgbe_clean_rx_irq(q_vector, ring, per_ring_budget); work_done += cleaned; @@ -3362,7 +3265,7 @@ static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) q_vector->name, q_vector); if (err) { e_err(probe, "request_irq failed for MSIX interrupt " - "Error: %d\n", err); + "Error: %d\n", err); goto free_queue_irqs; } /* If Flow Director is enabled, set interrupt affinity */ @@ -3592,7 +3495,7 @@ static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) * Configure the Tx descriptor ring after a reset. **/ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u64 tdba = ring->dma; @@ -3654,22 +3557,22 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, if (q_vector) netif_set_xps_queue(ring->netdev, - &q_vector->affinity_mask, - ring->queue_index); + &q_vector->affinity_mask, + ring->queue_index); } clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state); /* reinitialize tx_buffer_info */ memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbe_tx_buffer) * ring->count); + sizeof(struct ixgbe_tx_buffer) * ring->count); /* enable queue */ IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* poll to verify queue is enabled */ @@ -3814,7 +3717,7 @@ static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) * and performance reasons. */ if (adapter->num_vfs || (adapter->num_rx_queues > 1 && - !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { + !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { for (i = 0; i < adapter->num_rx_queues; i++) ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]); } else { @@ -3829,11 +3732,11 @@ static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u32 srrctl; - u8 reg_idx = rx_ring->reg_idx; + u8 reg_idx = ring->reg_idx; if (hw->mac.type == ixgbe_mac_82598EB) { u16 mask = adapter->ring_feature[RING_F_RSS].mask; @@ -3848,25 +3751,12 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, /* configure header buffer length, needed for RSC */ srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - /* configure the packet buffer length */ - if (rx_ring->xsk_pool) { - u32 xsk_buf_len = xsk_pool_get_rx_frame_size(rx_ring->xsk_pool); - - /* If the MAC support setting RXDCTL.RLPML, the - * SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and - * RXDCTL.RLPML is set to the actual UMEM buffer - * size. If not, then we are stuck with a 1k buffer - * size resolution. In this case frames larger than - * the UMEM buffer size viewed in a 1k resolution will - * be dropped. - */ - if (hw->mac.type != ixgbe_mac_82599EB) - srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - else - srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } else { - srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } + if (ring->xsk_pool) + srrctl |= DIV_ROUND_UP(xsk_pool_get_rx_frame_size(ring->xsk_pool), + IXGBE_SRRCTL_BSIZEPKT_STEP); + else + srrctl |= DIV_ROUND_UP(ring->rx_buf_len, + IXGBE_SRRCTL_BSIZEPKT_STEP); /* configure descriptor type */ srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; @@ -4097,9 +3987,9 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) /* Perform hash on these packet types */ rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 | - IXGBE_MRQC_RSS_FIELD_IPV4_TCP | - IXGBE_MRQC_RSS_FIELD_IPV6 | - IXGBE_MRQC_RSS_FIELD_IPV6_TCP; + IXGBE_MRQC_RSS_FIELD_IPV4_TCP | + IXGBE_MRQC_RSS_FIELD_IPV6 | + IXGBE_MRQC_RSS_FIELD_IPV6_TCP; if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP) rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; @@ -4107,7 +3997,7 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; if ((hw->mac.type >= ixgbe_mac_X550) && - (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) { + (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) { u16 pool = adapter->num_rx_pools; /* Enable VF RSS mode */ @@ -4158,7 +4048,7 @@ static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, #define IXGBE_MAX_RX_DESC_POLL 10 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBE_MAX_RX_DESC_POLL; @@ -4169,7 +4059,7 @@ static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, return; /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; do { @@ -4179,12 +4069,12 @@ static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, if (!wait_loop) { e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within " - "the polling period\n", reg_idx); + "the polling period\n", reg_idx); } } void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; union ixgbe_adv_rx_desc *rx_desc; @@ -4200,8 +4090,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, NULL)); xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); } else { - WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_PAGE_SHARED, NULL)); + xdp_rxq_info_attach_page_pool(&ring->xdp_rxq, ring->pp); } /* disable queue to avoid use of these values while updating state */ @@ -4223,7 +4112,6 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0); ring->tail = adapter->io_addr + IXGBE_RDT(reg_idx); - ixgbe_configure_srrctl(adapter, ring); ixgbe_configure_rscctl(adapter, ring); if (hw->mac.type == ixgbe_mac_82598EB) { @@ -4240,21 +4128,19 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, /* RXDCTL.RLPML does not work on 82599 */ } else if (hw->mac.type != ixgbe_mac_82599EB) { rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); + IXGBE_RXDCTL_RLPML_EN); } #endif if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { - u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); - - rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); - rxdctl |= xsk_buf_len | IXGBE_RXDCTL_RLPML_EN; + u32 pkt_len = + READ_ONCE(adapter->netdev->mtu) + LIBETH_RX_LL_LEN; - ring->rx_buf_len = xsk_buf_len; + rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); + if (pkt_len <= IXGBE_RXDCTL_RLPMLMASK) + rxdctl |= pkt_len | IXGBE_RXDCTL_RLPML_EN; } - /* initialize rx_buffer_info */ - memset(ring->rx_buffer_info, 0, - sizeof(struct ixgbe_rx_buffer) * ring->count); + + ixgbe_configure_srrctl(adapter, ring); /* initialize Rx descriptor 0 */ rx_desc = IXGBE_RX_DESC(ring, 0); rx_desc->wb.upper.length = 0; @@ -4278,10 +4164,10 @@ static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter) /* PSRTYPE must be initialized in non 82598 adapters */ u32 psrtype = IXGBE_PSRTYPE_TCPHDR | - IXGBE_PSRTYPE_UDPHDR | - IXGBE_PSRTYPE_IPV4HDR | - IXGBE_PSRTYPE_L2HDR | - IXGBE_PSRTYPE_IPV6HDR; + IXGBE_PSRTYPE_UDPHDR | + IXGBE_PSRTYPE_IPV4HDR | + IXGBE_PSRTYPE_L2HDR | + IXGBE_PSRTYPE_IPV6HDR; if (hw->mac.type == ixgbe_mac_82598EB) return; @@ -4366,40 +4252,6 @@ static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter) } } -static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) -{ - struct ixgbe_hw *hw = &adapter->hw; - struct net_device *netdev = adapter->netdev; - int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - u32 mhadd, hlreg0; - -#ifdef IXGBE_FCOE - /* adjust max frame to be able to do baby jumbo for FCoE */ - if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && - (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) - max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; -#endif /* IXGBE_FCOE */ - - /* adjust max frame to be at least the size of a standard frame */ - if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) - max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); - - mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); - if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { - mhadd &= ~IXGBE_MHADD_MFS_MASK; - mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; - - IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); - } - - hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); - /* set jumbo enable since MHADD.MFS is keeping size locked at - * max_frame - */ - hlreg0 |= IXGBE_HLREG0_JUMBOEN; - IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); -} - static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; @@ -4453,8 +4305,12 @@ static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; + struct net_device *netdev = adapter->netdev; + int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; + struct ixgbe_ring *rx_ring; int i; u32 rxctrl, rfctl; + u32 mhadd, hlreg0; /* disable receives while setting up the descriptors */ hw->mac.ops.disable_rx(hw); @@ -4475,15 +4331,38 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) /* Program registers for the distribution of queues */ ixgbe_setup_mrqc(adapter); - /* set_rx_buffer_len must be called before ring initialization */ - ixgbe_set_rx_buffer_len(adapter); +#ifdef IXGBE_FCOE + /* adjust max frame to be able to do baby jumbo for FCoE */ + if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && + (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) + max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; +#endif /* IXGBE_FCOE */ + + /* adjust max frame to be at least the size of a standard frame */ + if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) + max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); + + mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); + if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { + mhadd &= ~IXGBE_MHADD_MFS_MASK; + mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; + + IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); + } + + /* set jumbo enable since MHADD.MFS is keeping size locked at + * max_frame + */ + hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); + hlreg0 |= IXGBE_HLREG0_JUMBOEN; + IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); /* * Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ for (i = 0; i < adapter->num_rx_queues; i++) { - struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; + rx_ring = adapter->rx_ring[i]; clear_ring_rsc_enabled(rx_ring); if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) @@ -4754,7 +4633,7 @@ static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) || - hw->mac.type == ixgbe_mac_82598EB) + hw->mac.type == ixgbe_mac_82598EB) return; /* We are not in VLAN promisc, nothing to do */ @@ -4819,9 +4698,9 @@ void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter) if (mac_table->state & IXGBE_MAC_STATE_IN_USE) hw->mac.ops.set_rar(hw, i, - mac_table->addr, - mac_table->pool, - IXGBE_RAH_AV); + mac_table->addr, + mac_table->pool, + IXGBE_RAH_AV); else hw->mac.ops.clear_rar(hw, i); } @@ -4842,9 +4721,9 @@ static void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter) if (mac_table->state & IXGBE_MAC_STATE_IN_USE) hw->mac.ops.set_rar(hw, i, - mac_table->addr, - mac_table->pool, - IXGBE_RAH_AV); + mac_table->addr, + mac_table->pool, + IXGBE_RAH_AV); else hw->mac.ops.clear_rar(hw, i); } @@ -4899,7 +4778,7 @@ static void ixgbe_mac_set_default_filter(struct ixgbe_adapter *adapter) mac_table->state = IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE; hw->mac.ops.set_rar(hw, 0, mac_table->addr, mac_table->pool, - IXGBE_RAH_AV); + IXGBE_RAH_AV); } int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, @@ -4920,7 +4799,7 @@ int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, mac_table->pool = pool; mac_table->state |= IXGBE_MAC_STATE_MODIFIED | - IXGBE_MAC_STATE_IN_USE; + IXGBE_MAC_STATE_IN_USE; ixgbe_sync_mac_table(adapter); @@ -5211,7 +5090,7 @@ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && - (pb == ixgbe_fcoe_get_tc(adapter))) + (pb == ixgbe_fcoe_get_tc(adapter))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -5245,8 +5124,8 @@ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) */ if (marker < 0) { e_warn(drv, "Packet Buffer(%i) can not provide enough" - "headroom to support flow control." - "Decrease MTU or number of traffic classes\n", pb); + "headroom to support flow control." + "Decrease MTU or number of traffic classes\n", pb); marker = tc + 1; } @@ -5272,7 +5151,7 @@ static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && - (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) + (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -5326,7 +5205,7 @@ static void ixgbe_configure_pb(struct ixgbe_adapter *adapter) u8 tc = adapter->hw_tcs; if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE || - adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) + adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) hdrm = 32 << adapter->fdir_pballoc; else hdrm = 0; @@ -5357,13 +5236,13 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) if (!vf && (ring >= adapter->num_rx_queues)) { e_err(drv, "FDIR restore failed without VF, ring: %u\n", - ring); + ring); continue; } else if (vf && ((vf > adapter->num_vfs) || - ring >= adapter->num_rx_queues_per_pool)) { + ring >= adapter->num_rx_queues_per_pool)) { e_err(drv, "FDIR restore failed with VF, vf: %hhu, ring: %u\n", - vf, ring); + vf, ring); continue; } @@ -5388,9 +5267,6 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) **/ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) { - u16 i = rx_ring->next_to_clean; - struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i]; - if (rx_ring->xsk_pool) { ixgbe_xsk_clean_rx_ring(rx_ring); goto skip_free; @@ -5402,29 +5278,12 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) rx_ring->skb = NULL; } - /* Free all the Rx ring pages */ - while (i != rx_ring->next_to_use) { - /* Invalidate cache lines that may have been written to by - * device so that we avoid corrupting memory. - */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); - - /* free resources associated with mapping */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - PAGE_SIZE, DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - __free_page(rx_buffer->page); + for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { + const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; - i++; - rx_buffer++; - if (i == rx_ring->count) { + libeth_rx_recycle_slow(rx_fqe->netmem); + if (unlikely(++i == rx_ring->count)) i = 0; - rx_buffer = rx_ring->rx_buffer_info; - } } skip_free: @@ -5433,7 +5292,7 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) } static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, - struct ixgbe_fwd_adapter *accel) + struct ixgbe_fwd_adapter *accel) { u16 rss_i = adapter->ring_feature[RING_F_RSS].indices; int num_tc = netdev_get_num_tc(adapter->netdev); @@ -5451,7 +5310,7 @@ static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, /* record configuration for macvlan interface in vdev */ for (i = 0; i < num_tc; i++) netdev_bind_sb_channel_queue(adapter->netdev, vdev, - i, rss_i, baseq + (rss_i * i)); + i, rss_i, baseq + (rss_i * i)); for (i = 0; i < adapter->num_rx_queues_per_pool; i++) adapter->rx_ring[baseq + i]->netdev = vdev; @@ -5488,7 +5347,7 @@ static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, } static int ixgbe_macvlan_up(struct net_device *vdev, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data; struct ixgbe_fwd_adapter *accel; @@ -5512,7 +5371,7 @@ static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter) }; netdev_walk_all_upper_dev_rcu(adapter->netdev, - ixgbe_macvlan_up, &priv); + ixgbe_macvlan_up, &priv); } static void ixgbe_configure(struct ixgbe_adapter *adapter) @@ -5547,7 +5406,7 @@ static void ixgbe_configure(struct ixgbe_adapter *adapter) adapter->fdir_pballoc); } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { ixgbe_init_fdir_perfect_82599(&adapter->hw, - adapter->fdir_pballoc); + adapter->fdir_pballoc); ixgbe_fdir_filter_restore(adapter); } @@ -5648,7 +5507,7 @@ static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter) static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw) { struct ixgbe_adapter *adapter = container_of(hw, struct ixgbe_adapter, - hw); + hw); u16 mask = ~((u16)(IXGBE_ACI_LINK_EVENT_UPDOWN | IXGBE_ACI_LINK_EVENT_MEDIA_NA | IXGBE_ACI_LINK_EVENT_MODULE_QUAL_FAIL | @@ -5772,7 +5631,7 @@ static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter) if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | - IXGBE_GPIE_OCD; + IXGBE_GPIE_OCD; gpie |= IXGBE_GPIE_EIAME; /* * use EIAM to auto-mask when MSI-X interrupt is asserted @@ -6006,7 +5865,7 @@ void ixgbe_disable_rx(struct ixgbe_adapter *adapter) /* RXDCTL.EN may not change on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* Determine our minimum delay interval. We will increase this value @@ -6047,7 +5906,7 @@ void ixgbe_disable_rx(struct ixgbe_adapter *adapter) } e_err(drv, - "RXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); + "RXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); } void ixgbe_disable_tx(struct ixgbe_adapter *adapter) @@ -6128,7 +5987,7 @@ void ixgbe_disable_tx(struct ixgbe_adapter *adapter) } e_err(drv, - "TXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); + "TXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); dma_engine_disable: /* Disable the Tx DMA engine on 82599 and later MAC */ @@ -6162,7 +6021,7 @@ void ixgbe_reset(struct ixgbe_adapter *adapter) /* clear all SFP and link config related flags while holding SFP_INIT */ adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP | - IXGBE_FLAG2_SFP_NEEDS_RESET); + IXGBE_FLAG2_SFP_NEEDS_RESET); adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; err = hw->mac.ops.init_hw(hw); @@ -6258,9 +6117,9 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) /* unmap any remaining paged data */ if (dma_unmap_len(tx_buffer, len)) dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); } /* move us one more past the eop_desc for start of next pkt */ @@ -6474,7 +6333,7 @@ static void ixgbe_init_dcb(struct ixgbe_adapter *adapter) if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE) adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE; memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg, - sizeof(adapter->temp_dcb_cfg)); + sizeof(adapter->temp_dcb_cfg)); } #endif @@ -6546,8 +6405,8 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter, adapter->jump_tables[i] = NULL; adapter->mac_table = kcalloc(hw->mac.num_rar_entries, - sizeof(struct ixgbe_mac_addr), - GFP_KERNEL); + sizeof(struct ixgbe_mac_addr), + GFP_KERNEL); if (!adapter->mac_table) return -ENOMEM; @@ -6798,56 +6657,82 @@ static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) * Returns 0 on success, negative on failure **/ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) + struct ixgbe_ring *rx_ring) { - struct device *dev = rx_ring->dev; + struct libeth_fq fq = { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_MTU, + .xdp = !!rx_ring->xdp_prog, + .buf_len = IXGBE_RX_PAGE_LEN(rx_ring->xdp_prog ? + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), + }; + struct device *dev = &adapter->pdev->dev; int orig_node = dev_to_node(dev); + void *napi_dev; int ring_node = NUMA_NO_NODE; - int size; - - size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; + int ret; - if (rx_ring->q_vector) + if (rx_ring->q_vector) { + fq.nid = rx_ring->q_vector->numa_node; ring_node = rx_ring->q_vector->numa_node; + napi_dev = &rx_ring->q_vector->napi; + } else { + fq.no_napi = true; + napi_dev = &adapter->pdev->dev; + } - rx_ring->rx_buffer_info = vmalloc_node(size, ring_node); - if (!rx_ring->rx_buffer_info) - rx_ring->rx_buffer_info = vmalloc(size); - if (!rx_ring->rx_buffer_info) - goto err; + ret = libeth_rx_fq_create(&fq, napi_dev); + if (ret) + return ret; + + rx_ring->pp = fq.pp; + rx_ring->rx_fqes = fq.fqes; + rx_ring->truesize = fq.truesize; + rx_ring->rx_buf_len = fq.buf_len; /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); set_dev_node(dev, ring_node); - rx_ring->desc = dma_alloc_coherent(dev, - rx_ring->size, - &rx_ring->dma, - GFP_KERNEL); + rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, + &rx_ring->dma, GFP_KERNEL); set_dev_node(dev, orig_node); if (!rx_ring->desc) rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, - &rx_ring->dma, GFP_KERNEL); - if (!rx_ring->desc) - goto err; + &rx_ring->dma, + GFP_KERNEL); + if (!rx_ring->desc) { + ret = -ENOMEM; + goto err_destroy_fq; + } rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; /* XDP RX-queue info */ - if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, ixgbe_rx_napi_id(rx_ring)) < 0) - goto err; - + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, + rx_ring->queue_index, 0, rx_ring->truesize); + if (ret < 0) + goto err_free_desc; + + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, fq.pp); rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; -err: - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; + +err_free_desc: + dma_free_coherent(dev, rx_ring->size, + rx_ring->desc, rx_ring->dma); + rx_ring->desc = NULL; +err_destroy_fq: dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); - return -ENOMEM; + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; + return ret; } /** @@ -6934,21 +6819,32 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) **/ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { + struct libeth_fq fq = { + .fqes = rx_ring->rx_fqes, + .pp = rx_ring->pp, + }; + ixgbe_clean_rx_ring(rx_ring); - rx_ring->xdp_prog = NULL; + rcu_assign_pointer(rx_ring->xdp_prog, NULL); + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; /* if not set, then don't free */ if (!rx_ring->desc) return; - dma_free_coherent(rx_ring->dev, rx_ring->size, - rx_ring->desc, rx_ring->dma); + dma_free_coherent(fq.pp->p.dev, rx_ring->size, rx_ring->desc, + rx_ring->dma); rx_ring->desc = NULL; + + kvfree(rx_ring->rx_xsk_buffer_info); + rx_ring->rx_xsk_buffer_info = NULL; + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; } /** @@ -6991,7 +6887,7 @@ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); if (ixgbe_enabled_xdp_adapter(adapter)) { - int new_frame_size = new_mtu + IXGBE_PKT_HDR_PAD; + int new_frame_size = new_mtu + LIBETH_RX_LL_LEN; if (new_frame_size > ixgbe_max_xdp_frame_size(adapter)) { e_warn(probe, "Requested MTU size is not supported with XDP\n"); @@ -7005,8 +6901,8 @@ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) * warning that legacy VFs will be disabled. */ if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && - (adapter->hw.mac.type == ixgbe_mac_82599EB) && - (new_mtu > ETH_DATA_LEN)) + (adapter->hw.mac.type == ixgbe_mac_82599EB) && + (new_mtu > ETH_DATA_LEN)) e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n"); netdev_dbg(netdev, "changing MTU from %d to %d\n", @@ -7304,7 +7200,7 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) u64 bytes = 0, packets = 0, hw_csum_rx_error = 0; if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_RESETTING, &adapter->state)) return; if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { @@ -7405,11 +7301,11 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); if (hw->mac.type == ixgbe_mac_82599EB || - hw->mac.type == ixgbe_mac_X540 || - hw->mac.type == ixgbe_mac_X550 || - hw->mac.type == ixgbe_mac_X550EM_x || - hw->mac.type == ixgbe_mac_x550em_a || - hw->mac.type == ixgbe_mac_e610) { + hw->mac.type == ixgbe_mac_X540 || + hw->mac.type == ixgbe_mac_X550 || + hw->mac.type == ixgbe_mac_X550EM_x || + hw->mac.type == ixgbe_mac_x550em_a || + hw->mac.type == ixgbe_mac_e610) { hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */ hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); @@ -7445,7 +7341,7 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) case ixgbe_mac_82599EB: for (i = 0; i < 16; i++) adapter->hw_rx_no_dma_resources += - IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); + IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */ hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); @@ -7592,7 +7488,7 @@ static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); } else { e_err(probe, "failed to finish FDIR re-initialization, " - "ignored adding FDIR ATR filters\n"); + "ignored adding FDIR ATR filters\n"); } } @@ -7613,8 +7509,8 @@ static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter) /* If we're down, removing or resetting, just bail */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) return; /* Force detection of hung controller */ @@ -7674,7 +7570,7 @@ static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter) } if (link_up || - time_after(jiffies, (adapter->link_check_timeout + + time_after(jiffies, (adapter->link_check_timeout + IXGBE_TRY_LINK_TIMEOUT))) { adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC); @@ -7690,9 +7586,9 @@ static void ixgbe_update_default_up(struct ixgbe_adapter *adapter) #ifdef CONFIG_IXGBE_DCB struct net_device *netdev = adapter->netdev; struct dcb_app app = { - .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE, - .protocol = 0, - }; + .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE, + .protocol = 0, + }; u8 up = 0; if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE) @@ -7776,9 +7672,9 @@ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter) break; } e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str, - ((flow_rx && flow_tx) ? "RX/TX" : - (flow_rx ? "RX" : - (flow_tx ? "TX" : "None")))); + ((flow_rx && flow_tx) ? "RX/TX" : + (flow_rx ? "RX" : + (flow_tx ? "TX" : "None")))); netif_carrier_on(netdev); ixgbe_check_vf_rate_limit(adapter); @@ -7881,7 +7777,7 @@ static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter) { if (!netif_carrier_ok(adapter->netdev)) { if (ixgbe_ring_tx_pending(adapter) || - ixgbe_vf_tx_pending(adapter)) { + ixgbe_vf_tx_pending(adapter)) { /* We've lost link, so the controller stops DMA, * but we've got queued Tx work that's never going * to get done, so reset controller to flush Tx. @@ -7899,18 +7795,18 @@ static void ixgbe_bad_vf_abort(struct ixgbe_adapter *adapter, u32 vf) struct ixgbe_hw *hw = &adapter->hw; if (adapter->hw.mac.type == ixgbe_mac_82599EB && - adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) { + adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) { adapter->vfinfo[vf].primary_abort_count++; if (adapter->vfinfo[vf].primary_abort_count == - IXGBE_PRIMARY_ABORT_LIMIT) { + IXGBE_PRIMARY_ABORT_LIMIT) { ixgbe_set_vf_link_state(adapter, vf, IFLA_VF_LINK_STATE_DISABLE); adapter->vfinfo[vf].primary_abort_count = 0; e_info(drv, - "Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on", - hw->bus.func, vf, - adapter->vfinfo[vf].vf_mac_addresses); + "Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on", + hw->bus.func, vf, + adapter->vfinfo[vf].vf_mac_addresses); } } } @@ -7946,7 +7842,7 @@ static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter) continue; pci_read_config_word(vfdev, PCI_STATUS, &status_reg); if (status_reg != IXGBE_FAILED_READ_CFG_WORD && - status_reg & PCI_STATUS_REC_MASTER_ABORT) { + status_reg & PCI_STATUS_REC_MASTER_ABORT) { ixgbe_bad_vf_abort(adapter, vf); pcie_flr(vfdev); } @@ -7959,7 +7855,7 @@ static void ixgbe_spoof_check(struct ixgbe_adapter *adapter) /* Do not perform spoof check for 82598 or if not in IOV mode */ if (adapter->hw.mac.type == ixgbe_mac_82598EB || - adapter->num_vfs == 0) + adapter->num_vfs == 0) return; ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC); @@ -7993,8 +7889,8 @@ static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter) { /* if interface is down, removing or resetting, do nothing */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) return; ixgbe_watchdog_update_link(adapter); @@ -8022,11 +7918,11 @@ static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) /* not searching for SFP so there is nothing to do here */ if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) && - !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) + !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) return; if (adapter->sfp_poll_time && - time_after(adapter->sfp_poll_time, jiffies)) + time_after(adapter->sfp_poll_time, jiffies)) return; /* If not yet time to poll for SFP */ /* someone else is in init, wait until next service event */ @@ -8075,7 +7971,7 @@ static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); if (err == -EOPNOTSUPP && - adapter->netdev->reg_state == NETREG_REGISTERED) { + adapter->netdev->reg_state == NETREG_REGISTERED) { e_dev_err("failed to initialize because an unsupported " "SFP+ module type was detected.\n"); e_dev_err("Reload the driver after installing a " @@ -8111,7 +8007,7 @@ static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter) speed = IXGBE_LINK_SPEED_10GB_FULL; else speed = cap_speed & (IXGBE_LINK_SPEED_10GB_FULL | - IXGBE_LINK_SPEED_1GB_FULL); + IXGBE_LINK_SPEED_1GB_FULL); if (hw->mac.ops.setup_link) hw->mac.ops.setup_link(hw, speed, true); @@ -8169,8 +8065,8 @@ static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter) rtnl_lock(); /* If we're already down, removing or resetting, just bail */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) { + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) { rtnl_unlock(); return; } @@ -8228,7 +8124,7 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) /* skip if E610's FW is reloading, warning in that case may be misleading */ if (fwsm & IXGBE_FWSM_EXT_ERR_IND_MASK || - (!(fwsm & IXGBE_FWSM_FW_VAL_BIT) && !(hw->mac.type == ixgbe_mac_e610))) + (!(fwsm & IXGBE_FWSM_FW_VAL_BIT) && !(hw->mac.type == ixgbe_mac_e610))) e_dev_warn("Warning firmware error detected FWSM: 0x%08X\n", fwsm); @@ -8254,7 +8150,7 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) goto no_version; if (hw->mac.ops.get_nvm_ver && - hw->mac.ops.get_nvm_ver(hw, nvm_info)) + hw->mac.ops.get_nvm_ver(hw, nvm_info)) goto no_version; snprintf(ver_buff, sizeof(ver_buff), @@ -8274,8 +8170,8 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) static void ixgbe_recovery_service_task(struct work_struct *work) { struct ixgbe_adapter *adapter = container_of(work, - struct ixgbe_adapter, - service_task); + struct ixgbe_adapter, + service_task); ixgbe_handle_fw_event(adapter); ixgbe_service_event_complete(adapter); @@ -8290,8 +8186,8 @@ static void ixgbe_recovery_service_task(struct work_struct *work) static void ixgbe_service_task(struct work_struct *work) { struct ixgbe_adapter *adapter = container_of(work, - struct ixgbe_adapter, - service_task); + struct ixgbe_adapter, + service_task); if (ixgbe_removed(adapter->hw.hw_addr)) { if (!test_bit(__IXGBE_DOWN, &adapter->state)) { rtnl_lock(); @@ -8337,9 +8233,9 @@ static void ixgbe_service_task(struct work_struct *work) } static int ixgbe_tso(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, - u8 *hdr_len, - struct ixgbe_ipsec_tx_data *itd) + struct ixgbe_tx_buffer *first, + u8 *hdr_len, + struct ixgbe_ipsec_tx_data *itd) { u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; struct sk_buff *skb = first->skb; @@ -8375,7 +8271,7 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring, /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ? - IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP; + IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP; /* initialize outer IP header fields */ if (ip.v4->version == 4) { @@ -8412,12 +8308,12 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring, /* compute length of segmentation header */ *hdr_len = (l4.tcp->doff * 4) + l4_offset; csum_replace_by_diff(&l4.tcp->check, - (__force __wsum)htonl(paylen)); + (__force __wsum)htonl(paylen)); } else { /* compute length of segmentation header */ *hdr_len = sizeof(*l4.udp) + l4_offset; csum_replace_by_diff(&l4.udp->check, - (__force __wsum)htonl(paylen)); + (__force __wsum)htonl(paylen)); } /* update gso size and bytecount with header size */ @@ -8501,8 +8397,8 @@ static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) { /* set type for advanced descriptor with frame checksum insertion */ u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS; + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS; /* set HW vlan bit if vlan is present */ cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN, @@ -8556,7 +8452,7 @@ static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) { if (!netif_subqueue_try_stop(tx_ring->netdev, tx_ring->queue_index, - ixgbe_desc_unused(tx_ring), size)) + ixgbe_desc_unused(tx_ring), size)) return -EBUSY; ++tx_ring->tx_stats.restart_queue; @@ -8656,7 +8552,7 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, data_len -= size; dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, - DMA_TO_DEVICE); + DMA_TO_DEVICE); tx_buffer = &tx_ring->tx_buffer_info[i]; } @@ -8706,9 +8602,9 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, tx_buffer = &tx_ring->tx_buffer_info[i]; if (dma_unmap_len(tx_buffer, len)) dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buffer, len, 0); if (tx_buffer == first) break; @@ -8726,7 +8622,7 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, } static void ixgbe_atr(struct ixgbe_ring *ring, - struct ixgbe_tx_buffer *first) + struct ixgbe_tx_buffer *first) { struct ixgbe_q_vector *q_vector = ring->q_vector; union ixgbe_atr_hash_dword input = { .dword = 0 }; @@ -8754,7 +8650,7 @@ static void ixgbe_atr(struct ixgbe_ring *ring, /* currently only IPv4/IPv6 with TCP is supported */ if ((first->protocol != htons(ETH_P_IP)) && - (first->protocol != htons(ETH_P_IPV6))) + (first->protocol != htons(ETH_P_IPV6))) return; /* snag network header to get L4 type and address */ @@ -8763,21 +8659,21 @@ static void ixgbe_atr(struct ixgbe_ring *ring, if (unlikely(hdr.network <= skb->data)) return; if (skb->encapsulation && - first->protocol == htons(ETH_P_IP) && - hdr.ipv4->protocol == IPPROTO_UDP) { + first->protocol == htons(ETH_P_IP) && + hdr.ipv4->protocol == IPPROTO_UDP) { struct ixgbe_adapter *adapter = q_vector->adapter; if (unlikely(skb_tail_pointer(skb) < hdr.network + - vxlan_headroom(0))) + vxlan_headroom(0))) return; /* verify the port is recognized as VXLAN */ if (adapter->vxlan_port && - udp_hdr(skb)->dest == adapter->vxlan_port) + udp_hdr(skb)->dest == adapter->vxlan_port) hdr.network = skb_inner_network_header(skb); if (adapter->geneve_port && - udp_hdr(skb)->dest == adapter->geneve_port) + udp_hdr(skb)->dest == adapter->geneve_port) hdr.network = skb_inner_network_header(skb); } @@ -8807,7 +8703,7 @@ static void ixgbe_atr(struct ixgbe_ring *ring, return; if (unlikely(skb_tail_pointer(skb) < hdr.network + - hlen + sizeof(struct tcphdr))) + hlen + sizeof(struct tcphdr))) return; th = (struct tcphdr *)(hdr.network + hlen); @@ -8852,13 +8748,13 @@ static void ixgbe_atr(struct ixgbe_ring *ring, case 6: input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6; common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^ - hdr.ipv6->saddr.s6_addr32[1] ^ - hdr.ipv6->saddr.s6_addr32[2] ^ - hdr.ipv6->saddr.s6_addr32[3] ^ - hdr.ipv6->daddr.s6_addr32[0] ^ - hdr.ipv6->daddr.s6_addr32[1] ^ - hdr.ipv6->daddr.s6_addr32[2] ^ - hdr.ipv6->daddr.s6_addr32[3]; + hdr.ipv6->saddr.s6_addr32[1] ^ + hdr.ipv6->saddr.s6_addr32[2] ^ + hdr.ipv6->saddr.s6_addr32[3] ^ + hdr.ipv6->daddr.s6_addr32[0] ^ + hdr.ipv6->daddr.s6_addr32[1] ^ + hdr.ipv6->daddr.s6_addr32[2] ^ + hdr.ipv6->daddr.s6_addr32[3]; break; default: break; @@ -8869,12 +8765,12 @@ static void ixgbe_atr(struct ixgbe_ring *ring, /* This assumes the Rx queue and Tx queue are bound to the same CPU */ ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw, - input, common, ring->queue_index); + input, common, ring->queue_index); } #ifdef IXGBE_FCOE static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev) + struct net_device *sb_dev) { struct ixgbe_adapter *adapter; struct ixgbe_ring_feature *f; @@ -8987,8 +8883,8 @@ int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, tx_buff = &ring->tx_buffer_info[index]; if (dma_unmap_len(tx_buff, len)) dma_unmap_page(ring->dev, dma_unmap_addr(tx_buff, dma), - dma_unmap_len(tx_buff, len), - DMA_TO_DEVICE); + dma_unmap_len(tx_buff, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buff, len, 0); if (tx_buff == tx_head) break; @@ -9054,9 +8950,9 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, protocol = vlan_get_protocol(skb); if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && - adapter->ptp_clock) { + adapter->ptp_clock) { if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && - !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS, + !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS, &adapter->state)) { skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; tx_flags |= IXGBE_TX_FLAGS_TSTAMP; @@ -9081,8 +8977,8 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #endif /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */ if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && - ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || - (skb->priority != TC_PRIO_CONTROL))) { + ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || + (skb->priority != TC_PRIO_CONTROL))) { tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK; tx_flags |= (skb->priority & 0x7) << IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT; @@ -9106,7 +9002,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #ifdef IXGBE_FCOE /* setup tx offload for FCoE */ if ((protocol == htons(ETH_P_FCOE)) && - (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { + (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { tso = ixgbe_fso(tx_ring, first, &hdr_len); if (tso < 0) goto out_drop; @@ -9118,7 +9014,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #ifdef CONFIG_IXGBE_IPSEC if (xfrm_offload(skb) && - !ixgbe_ipsec_tx(tx_ring, first, &ipsec_tx)) + !ixgbe_ipsec_tx(tx_ring, first, &ipsec_tx)) goto out_drop; #endif tso = ixgbe_tso(tx_ring, first, &hdr_len, &ipsec_tx); @@ -9154,8 +9050,8 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, } static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb, - struct net_device *netdev, - struct ixgbe_ring *ring) + struct net_device *netdev, + struct ixgbe_ring *ring) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_ring *tx_ring; @@ -9175,7 +9071,7 @@ static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb, } static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, - struct net_device *netdev) + struct net_device *netdev) { return __ixgbe_xmit_frame(skb, netdev, NULL); } @@ -9231,7 +9127,7 @@ ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr) } static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad, - u16 addr, u16 value) + u16 addr, u16 value) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -9328,7 +9224,7 @@ static void ixgbe_get_ring_stats64(struct rtnl_link_stats64 *stats, } static void ixgbe_get_stats64(struct net_device *netdev, - struct rtnl_link_stats64 *stats) + struct rtnl_link_stats64 *stats) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); int i; @@ -9452,7 +9348,7 @@ static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter) #endif /* CONFIG_IXGBE_DCB */ static int ixgbe_reassign_macvlan_pool(struct net_device *vdev, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data; struct ixgbe_fwd_adapter *accel; @@ -9500,7 +9396,7 @@ static void ixgbe_defrag_macvlan_pools(struct net_device *dev) /* walk through upper devices reassigning pools */ netdev_walk_all_upper_dev_rcu(dev, ixgbe_reassign_macvlan_pool, - &priv); + &priv); } /** @@ -9580,7 +9476,7 @@ int ixgbe_setup_tc(struct net_device *dev, u8 tc) } static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 hdl = cls->knode.handle; u32 uhtid = TC_U32_USERHTID(cls->knode.handle); @@ -9616,8 +9512,8 @@ static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, continue; spin_lock(&adapter->fdir_perfect_lock); err = ixgbe_update_ethtool_fdir_entry(adapter, - NULL, - j + 1); + NULL, + j + 1); spin_unlock(&adapter->fdir_perfect_lock); clear_bit(j, jump->child_loc_map); } @@ -9637,7 +9533,7 @@ static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, } static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 uhtid = TC_U32_USERHTID(cls->hnode.handle); @@ -9655,7 +9551,7 @@ static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter, } static int ixgbe_configure_clsu32_del_hnode(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 uhtid = TC_U32_USERHTID(cls->hnode.handle); @@ -9675,7 +9571,7 @@ struct upper_walk_data { }; static int get_macvlan_queue(struct net_device *upper, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { if (netif_is_macvlan(upper)) { struct ixgbe_fwd_adapter *vadapter = macvlan_accel_priv(upper); @@ -9734,7 +9630,7 @@ static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex, } static int parse_tc_actions(struct ixgbe_adapter *adapter, - struct tcf_exts *exts, u64 *action, u8 *queue) + struct tcf_exts *exts, u64 *action, u8 *queue) { const struct tc_action *a; int i; @@ -9757,7 +9653,7 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter, if (!dev) return -EINVAL; return handle_redirect_action(adapter, dev->ifindex, - queue, action); + queue, action); } return -EINVAL; @@ -9767,17 +9663,17 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter, } #else static int parse_tc_actions(struct ixgbe_adapter *adapter, - struct tcf_exts *exts, u64 *action, u8 *queue) + struct tcf_exts *exts, u64 *action, u8 *queue) { return -EINVAL; } #endif /* CONFIG_NET_CLS_ACT */ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, - union ixgbe_atr_input *mask, - struct tc_cls_u32_offload *cls, - struct ixgbe_mat_field *field_ptr, - struct ixgbe_nexthdr *nexthdr) + union ixgbe_atr_input *mask, + struct tc_cls_u32_offload *cls, + struct ixgbe_mat_field *field_ptr, + struct ixgbe_nexthdr *nexthdr) { int i, j, off; __be32 val, m; @@ -9800,10 +9696,10 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, } if (nexthdr) { if (nexthdr->off == cls->knode.sel->keys[i].off && - nexthdr->val == - (__force u32)cls->knode.sel->keys[i].val && - nexthdr->mask == - (__force u32)cls->knode.sel->keys[i].mask) + nexthdr->val == + (__force u32)cls->knode.sel->keys[i].val && + nexthdr->mask == + (__force u32)cls->knode.sel->keys[i].mask) found_jump_field = true; else continue; @@ -9817,7 +9713,7 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, return 0; mask->formatted.flow_type = IXGBE_ATR_L4TYPE_IPV6_MASK | - IXGBE_ATR_L4TYPE_MASK; + IXGBE_ATR_L4TYPE_MASK; if (input->filter.formatted.flow_type == IXGBE_ATR_FLOW_TYPE_IPV4) mask->formatted.flow_type &= IXGBE_ATR_L4TYPE_IPV6_MASK; @@ -9898,17 +9794,17 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, * and link to it. */ if (adapter->jump_tables[link_uhtid] && - (adapter->jump_tables[link_uhtid])->link_hdl) { + (adapter->jump_tables[link_uhtid])->link_hdl) { e_err(drv, "Link filter exists for link: %x\n", - link_uhtid); + link_uhtid); return err; } for (i = 0; nexthdr[i].jump; i++) { if (nexthdr[i].o != cls->knode.sel->offoff || - nexthdr[i].s != cls->knode.sel->offshift || - nexthdr[i].m != - (__force u32)cls->knode.sel->offmask) + nexthdr[i].s != cls->knode.sel->offshift || + nexthdr[i].m != + (__force u32)cls->knode.sel->offmask) return err; jump = kzalloc(sizeof(*jump), GFP_KERNEL); @@ -9929,7 +9825,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, jump->link_hdl = cls->knode.handle; err = ixgbe_clsu32_build_input(input, mask, cls, - field_ptr, &nexthdr[i]); + field_ptr, &nexthdr[i]); if (!err) { jump->mat = nexthdr[i].jump; adapter->jump_tables[link_uhtid] = jump; @@ -9955,10 +9851,10 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, if ((uhtid != 0x800) && (adapter->jump_tables[uhtid])) { if ((adapter->jump_tables[uhtid])->input) memcpy(input, (adapter->jump_tables[uhtid])->input, - sizeof(*input)); + sizeof(*input)); if ((adapter->jump_tables[uhtid])->mask) memcpy(mask, (adapter->jump_tables[uhtid])->mask, - sizeof(*mask)); + sizeof(*mask)); /* Lookup in all child hash tables if this location is already * filled with a filter @@ -9968,7 +9864,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, if (link && (test_bit(loc - 1, link->child_loc_map))) { e_err(drv, "Filter exists in location: %x\n", - loc); + loc); err = -EINVAL; goto err_out; } @@ -9979,7 +9875,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, goto err_out; err = parse_tc_actions(adapter, cls->knode.exts, &input->action, - &queue); + &queue); if (err < 0) goto err_out; @@ -9999,7 +9895,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, ixgbe_atr_compute_perfect_hash_82599(&input->filter, mask); err = ixgbe_fdir_write_perfect_filter_82599(hw, &input->filter, - input->sw_idx, queue); + input->sw_idx, queue); if (err) goto err_out_w_lock; @@ -10067,7 +9963,7 @@ static int ixgbe_setup_tc_mqprio(struct net_device *dev, static LIST_HEAD(ixgbe_block_cb_list); static int __ixgbe_setup_tc(struct net_device *dev, enum tc_setup_type type, - void *type_data) + void *type_data) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); @@ -10106,7 +10002,7 @@ void ixgbe_do_reset(struct net_device *netdev) } static netdev_features_t ixgbe_fix_features(struct net_device *netdev, - netdev_features_t features) + netdev_features_t features) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); @@ -10134,7 +10030,7 @@ static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter) /* go back to full RSS if we're not running SR-IOV */ if (!adapter->ring_feature[RING_F_VMDQ].offset) adapter->flags &= ~(IXGBE_FLAG_VMDQ_ENABLED | - IXGBE_FLAG_SRIOV_ENABLED); + IXGBE_FLAG_SRIOV_ENABLED); adapter->ring_feature[RING_F_RSS].limit = rss; adapter->ring_feature[RING_F_VMDQ].limit = 1; @@ -10143,7 +10039,7 @@ static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter) } static int ixgbe_set_features(struct net_device *netdev, - netdev_features_t features) + netdev_features_t features) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); netdev_features_t changed = netdev->features ^ features; @@ -10157,12 +10053,12 @@ static int ixgbe_set_features(struct net_device *netdev, } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) && !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) { if (adapter->rx_itr_setting == 1 || - adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { + adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; need_reset = true; } else if ((changed ^ features) & NETIF_F_LRO) { e_info(probe, "rx-usecs set too low, " - "disabling RSC\n"); + "disabling RSC\n"); } } @@ -10186,12 +10082,12 @@ static int ixgbe_set_features(struct net_device *netdev, /* We cannot enable ATR if SR-IOV is enabled */ if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED || - /* We cannot enable ATR if we have 2 or more tcs */ - (adapter->hw_tcs > 1) || - /* We cannot enable ATR if RSS is disabled */ - (adapter->ring_feature[RING_F_RSS].limit <= 1) || - /* A sample rate of 0 indicates ATR disabled */ - (!adapter->atr_sample_rate)) + /* We cannot enable ATR if we have 2 or more tcs */ + (adapter->hw_tcs > 1) || + /* We cannot enable ATR if RSS is disabled */ + (adapter->ring_feature[RING_F_RSS].limit <= 1) || + /* A sample rate of 0 indicates ATR disabled */ + (!adapter->atr_sample_rate)) ; /* do nothing not supported */ else /* otherwise supported and set the flag */ adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; @@ -10207,17 +10103,17 @@ static int ixgbe_set_features(struct net_device *netdev, else if (need_reset) ixgbe_do_reset(netdev); else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_FILTER)) + NETIF_F_HW_VLAN_CTAG_FILTER)) ixgbe_set_rx_mode(netdev); return 1; } static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], - struct net_device *dev, - const unsigned char *addr, u16 vid, - u16 flags, bool *notified, - struct netlink_ext_ack *extack) + struct net_device *dev, + const unsigned char *addr, u16 vid, + u16 flags, bool *notified, + struct netlink_ext_ack *extack) { /* guarantee we can provide a unique filter for the unicast address */ if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) { @@ -10239,7 +10135,7 @@ static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], * Configure some settings require for various bridge modes. **/ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, - __u16 mode) + __u16 mode) { struct ixgbe_hw *hw = &adapter->hw; unsigned int p, num_pools; @@ -10265,8 +10161,8 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, for (p = 0; p < num_pools; p++) { if (hw->mac.ops.set_source_address_pruning) hw->mac.ops.set_source_address_pruning(hw, - true, - p); + true, + p); } break; case BRIDGE_MODE_VEB: @@ -10289,8 +10185,8 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, for (p = 0; p < num_pools; p++) { if (hw->mac.ops.set_source_address_pruning) hw->mac.ops.set_source_address_pruning(hw, - false, - p); + false, + p); } break; default: @@ -10300,14 +10196,14 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, adapter->bridge_mode = mode; e_info(drv, "enabling bridge mode: %s\n", - mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); + mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); return 0; } static int ixgbe_ndo_bridge_setlink(struct net_device *dev, - struct nlmsghdr *nlh, u16 flags, - struct netlink_ext_ack *extack) + struct nlmsghdr *nlh, u16 flags, + struct netlink_ext_ack *extack) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct nlattr *attr, *br_spec; @@ -10334,8 +10230,8 @@ static int ixgbe_ndo_bridge_setlink(struct net_device *dev, } static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, - struct net_device *dev, - u32 filter_mask, int nlflags) + struct net_device *dev, + u32 filter_mask, int nlflags) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); @@ -10343,8 +10239,8 @@ static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, return 0; return ndo_dflt_bridge_getlink(skb, pid, seq, dev, - adapter->bridge_mode, 0, 0, nlflags, - filter_mask, NULL); + adapter->bridge_mode, 0, 0, nlflags, + filter_mask, NULL); } static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) @@ -10379,8 +10275,8 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) u16 reserved_pools; if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && - adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) || - adapter->num_rx_pools > IXGBE_MAX_MACVLANS) + adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) || + adapter->num_rx_pools > IXGBE_MAX_MACVLANS) return ERR_PTR(-EBUSY); /* Hardware has a limited number of available pools. Each VF, @@ -10400,12 +10296,12 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) */ if (used_pools < 32 && adapter->num_rx_pools < 16) reserved_pools = min_t(u16, - 32 - used_pools, - 16 - adapter->num_rx_pools); + 32 - used_pools, + 16 - adapter->num_rx_pools); else if (adapter->num_rx_pools < 32) reserved_pools = min_t(u16, - 64 - used_pools, - 32 - adapter->num_rx_pools); + 64 - used_pools, + 32 - adapter->num_rx_pools); else reserved_pools = 64 - used_pools; @@ -10452,7 +10348,7 @@ static void ixgbe_fwd_del(struct net_device *pdev, void *priv) /* delete unicast filter associated with offloaded interface */ ixgbe_del_mac_filter(adapter, accel->netdev->dev_addr, - VMDQ_P(accel->pool)); + VMDQ_P(accel->pool)); /* Allow remaining Rx packets to get flushed out of the * Rx FIFO before we drop the netdev for the ring. @@ -10484,7 +10380,7 @@ static void ixgbe_fwd_del(struct net_device *pdev, void *priv) static netdev_features_t ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, - netdev_features_t features) + netdev_features_t features) { unsigned int network_hdr_len, mac_hdr_len; @@ -10492,19 +10388,19 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, mac_hdr_len = skb_network_offset(skb); if (unlikely(mac_hdr_len > IXGBE_MAX_MAC_HDR_LEN)) return features & ~(NETIF_F_HW_CSUM | - NETIF_F_SCTP_CRC | - NETIF_F_GSO_UDP_L4 | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_TSO | - NETIF_F_TSO6); + NETIF_F_SCTP_CRC | + NETIF_F_GSO_UDP_L4 | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_TSO | + NETIF_F_TSO6); network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); if (unlikely(network_hdr_len > IXGBE_MAX_NETWORK_HDR_LEN)) return features & ~(NETIF_F_HW_CSUM | - NETIF_F_SCTP_CRC | - NETIF_F_GSO_UDP_L4 | - NETIF_F_TSO | - NETIF_F_TSO6); + NETIF_F_SCTP_CRC | + NETIF_F_GSO_UDP_L4 | + NETIF_F_TSO | + NETIF_F_TSO6); /* We can only support IPV4 TSO in tunnels if we can mangle the * inner IP ID field, so strip TSO if MANGLEID is not supported. @@ -10523,7 +10419,7 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) { - int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; + int i, frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct bpf_prog *old_prog; bool need_reset; @@ -10587,7 +10483,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) for (i = 0; i < num_queues; i++) if (adapter->xdp_ring[i]->xsk_pool) (void)ixgbe_xsk_wakeup(adapter->netdev, i, - XDP_WAKEUP_RX); + XDP_WAKEUP_RX); xdp_features_set_redirect_target(dev, true); } @@ -10603,7 +10499,7 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp) return ixgbe_xdp_setup(dev, xdp->prog); case XDP_SETUP_XSK_POOL: return ixgbe_xsk_pool_setup(adapter, xdp->xsk.pool, - xdp->xsk.queue_id); + xdp->xsk.queue_id); default: return -EINVAL; @@ -10640,7 +10536,7 @@ static int ixgbe_xdp_xmit(struct net_device *dev, int n, return -ENETDOWN; if (!netif_carrier_ok(adapter->netdev) || - !netif_running(adapter->netdev)) + !netif_running(adapter->netdev)) return -ENETDOWN; if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) @@ -10757,7 +10653,7 @@ static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter, } static void ixgbe_disable_txr(struct ixgbe_adapter *adapter, - struct ixgbe_ring *tx_ring) + struct ixgbe_ring *tx_ring) { set_bit(__IXGBE_TX_DISABLED, &tx_ring->state); ixgbe_disable_txr_hw(adapter, tx_ring); @@ -10781,7 +10677,7 @@ static void ixgbe_disable_rxr_hw(struct ixgbe_adapter *adapter, /* RXDCTL.EN may not change on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* delay mechanism from ixgbe_disable_rx */ @@ -10961,7 +10857,7 @@ static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter) * this case, simply return -1 to indicate this. */ if ((entry->vendor != pdev->vendor) || - (entry->device != pdev->device)) + (entry->device != pdev->device)) return -1; physfns++; @@ -10993,8 +10889,8 @@ bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, /* check eeprom to see if WOL is enabled for X540 and newer */ if (hw->mac.type >= ixgbe_mac_X540) { if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) || - ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && - (hw->bus.func == 0))) + ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && + (hw->bus.func == 0))) return true; } @@ -11181,7 +11077,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) */ if (pdev->is_virtfn) { WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n", - pci_name(pdev), pdev->vendor, pdev->device); + pci_name(pdev), pdev->vendor, pdev->device); return -EINVAL; } @@ -11241,7 +11137,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); hw->hw_addr = ioremap(pci_resource_start(pdev, 0), - pci_resource_len(pdev, 0)); + pci_resource_len(pdev, 0)); adapter->io_addr = hw->hw_addr; if (!hw->hw_addr) { err = -EIO; @@ -11384,15 +11280,15 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_CSUM; #define IXGBE_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ - NETIF_F_GSO_GRE_CSUM | \ - NETIF_F_GSO_IPXIP4 | \ - NETIF_F_GSO_IPXIP6 | \ - NETIF_F_GSO_UDP_TUNNEL | \ - NETIF_F_GSO_UDP_TUNNEL_CSUM) + NETIF_F_GSO_GRE_CSUM | \ + NETIF_F_GSO_IPXIP4 | \ + NETIF_F_GSO_IPXIP6 | \ + NETIF_F_GSO_UDP_TUNNEL | \ + NETIF_F_GSO_UDP_TUNNEL_CSUM) netdev->gso_partial_features = IXGBE_GSO_PARTIAL_FEATURES; netdev->features |= NETIF_F_GSO_PARTIAL | - IXGBE_GSO_PARTIAL_FEATURES; + IXGBE_GSO_PARTIAL_FEATURES; if (hw->mac.type >= ixgbe_mac_82599EB) netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4; @@ -11407,15 +11303,15 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) #endif /* copy netdev features into list of user selectable features */ netdev->hw_features |= netdev->features | - NETIF_F_HW_VLAN_CTAG_FILTER | - NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_RXALL | - NETIF_F_HW_L2FW_DOFFLOAD; + NETIF_F_HW_VLAN_CTAG_FILTER | + NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_RXALL | + NETIF_F_HW_L2FW_DOFFLOAD; if (hw->mac.type >= ixgbe_mac_82599EB) netdev->hw_features |= NETIF_F_NTUPLE | - NETIF_F_HW_TC; + NETIF_F_HW_TC; netdev->features |= NETIF_F_HIGHDMA; @@ -11429,14 +11325,14 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* set this bit last since it cannot be part of vlan_features */ netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | - NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_TX; + NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; netdev->priv_flags |= IFF_SUPP_NOFCS; netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_XSK_ZEROCOPY; + NETDEV_XDP_ACT_XSK_ZEROCOPY; /* MTU range: 68 - 9710 */ netdev->min_mtu = ETH_MIN_MTU; @@ -11462,7 +11358,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->ring_feature[RING_F_FCOE].limit = fcoe_l; netdev->features |= NETIF_F_FSO | - NETIF_F_FCOE_CRC; + NETIF_F_FCOE_CRC; netdev->vlan_features |= NETIF_F_FSO | NETIF_F_FCOE_CRC; @@ -11481,7 +11377,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } eth_platform_get_mac_address(&adapter->pdev->dev, - adapter->hw.mac.perm_addr); + adapter->hw.mac.perm_addr); eth_hw_addr_set(netdev, hw->mac.perm_addr); @@ -11799,7 +11695,7 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, u16 req_id, pf_func; if (adapter->hw.mac.type == ixgbe_mac_82598EB || - adapter->num_vfs == 0) + adapter->num_vfs == 0) goto skip_bad_vf_detection; bdev = pdev->bus->self; @@ -11864,7 +11760,7 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, if (vfdev->devfn == (req_id & 0xFF)) break; vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, - device_id, vfdev); + device_id, vfdev); } /* * There's a slim chance the VF could have been hot plugged, @@ -12051,7 +11947,7 @@ static void __exit ixgbe_exit_module(void) #ifdef CONFIG_IXGBE_DCA static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event, - void *p) + void *p) { int ret_val; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h index b1bfeb21537acc..825cf17fa1878d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h @@ -2452,7 +2452,7 @@ enum { #define IXGBE_PSRTYPE_L2HDR 0x00001000 /* SRRCTL bit definitions */ -#define IXGBE_SRRCTL_BSIZEPKT_SHIFT 10 /* so many KBs */ +#define IXGBE_SRRCTL_BSIZEPKT_STEP 1024 #define IXGBE_SRRCTL_RDMTS_SHIFT 22 #define IXGBE_SRRCTL_RDMTS_MASK 0x01C00000 #define IXGBE_SRRCTL_DROP_EN 0x10000000 diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 7b941505a9d024..15425d3c367872 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -151,7 +151,7 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; u16 i = rx_ring->next_to_use; dma_addr_t dma; bool ok = true; @@ -161,7 +161,7 @@ bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) return true; rx_desc = IXGBE_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; + bi = &rx_ring->rx_xsk_buffer_info[i]; i -= rx_ring->count; do { @@ -183,7 +183,7 @@ bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) i++; if (unlikely(!i)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; + bi = rx_ring->rx_xsk_buffer_info; i -= rx_ring->count; } @@ -257,7 +257,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -279,17 +279,17 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + bi = &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; if (unlikely(!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) { - struct ixgbe_rx_buffer *next_bi; + struct ixgbe_xsk_rx_buffer *next_bi; xsk_buff_free(bi->xdp); bi->xdp = NULL; ixgbe_inc_ntc(rx_ring); next_bi = - &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; next_bi->discard = true; continue; } @@ -345,6 +345,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, total_rx_bytes += skb->len; total_rx_packets++; + skb->protocol = eth_type_trans(skb, rx_ring->netdev); ixgbe_process_skb_fields(rx_ring, rx_desc, skb); ixgbe_rx_skb(q_vector, skb); } @@ -374,11 +375,11 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring) { - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; u16 i; for (i = 0; i < rx_ring->count; i++) { - bi = &rx_ring->rx_buffer_info[i]; + bi = &rx_ring->rx_xsk_buffer_info[i]; if (!bi->xdp) continue; From 7f9272b4a0053c2d5cc52a00d8ab9015849976d5 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Thu, 28 May 2026 15:03:24 +0200 Subject: [PATCH 19/20] ixgbe: branch prediction and cleanup * Added branch prediction hints to the ixgbe Rx path and clean up the packet accounting. * Moved total_rx_bytes and total_rx_packets accounting together. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 43 ++++++------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index db2b2a5800904e..f5f371c32c29cd 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -176,6 +176,7 @@ MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); MODULE_IMPORT_NS("LIBIE_FWLOG"); MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); MODULE_IMPORT_NS("LIBETH"); +MODULE_IMPORT_NS("LIBETH_XDP"); MODULE_LICENSE("GPL v2"); DEFINE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key); @@ -817,22 +818,12 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) le64_to_cpu((__force __le64)u0->b), ring_desc); } else { - pr_info("R [0x%03X] %016llX %016llX %016llX %s\n", + pr_info("R [0x%03X] %016llX %016llX %s\n", i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - (u64)rx_ring->rx_buffer_info[i].dma, ring_desc); } - - if (netif_msg_pktdata(adapter) && - rx_ring->rx_buffer_info[i].dma) { - print_hex_dump(KERN_INFO, "", - DUMP_PREFIX_ADDRESS, 16, 1, - page_address(rx_ring->rx_buffer_info[i].page) + - rx_ring->rx_buffer_info[i].page_offset, - IXGBE_RXBUFFER_3K, true); - } } } } @@ -1742,7 +1733,7 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) u16 ntu = rx_ring->next_to_use; /* nothing to do */ - if (!cleaned_count) + if (unlikely(!cleaned_count)) return; rx_desc = IXGBE_RX_DESC(rx_ring, ntu); @@ -1758,7 +1749,7 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) rx_desc++; ntu++; - if (unlikely(ntu == rx_ring->count)) { + if (unlikely(ntu == fq.count)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); ntu = 0; } @@ -2165,7 +2156,6 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* exit if we failed to retrieve a buffer */ if (unlikely(!xdp_res && !skb)) { - libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; cleaned_count++; break; @@ -2178,14 +2168,12 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, continue; /* verify the packet layout is correct */ - if (xdp_res || unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { + if (xdp_res || + unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; - /* populate checksum, timestamp, VLAN, and protocol */ ixgbe_process_skb_fields(rx_ring, rx_desc, skb); @@ -2215,11 +2203,12 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, } #endif /* IXGBE_FCOE */ + /* probably a little skewed due to removing CRC */ + total_rx_bytes += skb->len; + total_rx_packets++; + ixgbe_rx_skb(q_vector, skb); skb = NULL; - - /* update budget accounting */ - total_rx_packets++; } rx_ring->skb = skb; @@ -6642,13 +6631,6 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) return err; } -static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) -{ - struct ixgbe_q_vector *q_vector = rx_ring->q_vector; - - return q_vector ? q_vector->napi.napi_id : 0; -} - /** * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) * @adapter: pointer to ixgbe_adapter @@ -6670,7 +6652,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, }; struct device *dev = &adapter->pdev->dev; int orig_node = dev_to_node(dev); - void *napi_dev; + struct napi_struct *napi_dev; int ring_node = NUMA_NO_NODE; int ret; @@ -6679,8 +6661,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, ring_node = rx_ring->q_vector->numa_node; napi_dev = &rx_ring->q_vector->napi; } else { - fq.no_napi = true; - napi_dev = &adapter->pdev->dev; + napi_dev = NULL; } ret = libeth_rx_fq_create(&fq, napi_dev); From 27cae8355e09cf54b161aa91138114640f20af12 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Mon, 1 Jun 2026 09:51:36 +0200 Subject: [PATCH 20/20] ixgbe: support XDP multi-buffer on Rx path Implement XDP multi-buffer support for Rx fragmented packets using libeth_xdp. * Added MODULE_IMPORT_NS("LIBETH_XDP") and #include * Replaced struct sk_buff *skb in ixgbe_ring with struct libeth_xdp_buff_stash and ixgbe_clean_rx_ring skb cleanup with libeth_xdp_return_stash * Removed ixgbe_add_rx_frag() * Reorganized ixgbe_clean_rx_irq() and ixgbe_is_non_eop() * Updated ixgbe_xdp_setup() to allow multi-buffer XDP programs * Added NETDEV_XDP_ACT_RX_SG in xdp_features Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 3 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 119 +++++++----------- 2 files changed, 49 insertions(+), 73 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index e228baef36acea..899a2fa1914630 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -34,6 +34,7 @@ #include #include +#include /* common prefix used by pr_<> macros */ #undef pr_fmt @@ -327,12 +328,12 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; - struct sk_buff *skb; /* partial Rx skb across non-EOP */ struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ u32 rx_buf_len; + struct libeth_xdp_buff_stash xdp_stash; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index f5f371c32c29cd..c213687bca8d08 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1853,7 +1853,7 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, * ixgbe_is_non_eop - process handling of non-EOP buffers * @rx_ring: Rx ring being processed * @rx_desc: Rx descriptor for current buffer - * @skb: Current socket buffer containing buffer in progress + * @rsc_append_cnt: pointer to RSC append count accumulator * * This function updates next to clean. If the buffer is an EOP buffer * this function exits returning false, otherwise it will place the @@ -1861,8 +1861,8 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, * that this is in fact a non-EOP buffer. **/ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + u32 *rsc_append_cnt) { u32 ntc = rx_ring->next_to_clean + 1; @@ -1881,7 +1881,7 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, u32 rsc_cnt = le32_to_cpu(rsc_enabled); rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; - IXGBE_CB(skb)->append_cnt += rsc_cnt - 1; + *rsc_append_cnt += rsc_cnt - 1; /* update ntc based on RSC value */ ntc = le32_to_cpu(rx_desc->wb.upper.status_error); @@ -1994,33 +1994,6 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return false; } -/** - * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff - * @rx_ring: rx descriptor ring to transact packets on - * @rx_buffer: buffer containing page to add - * @skb: sk_buff to place the data into - * @size: size of data in rx_buffer - * - * This function will add the data contained in rx_buffer->page to the skb. - * This is done either through a direct copy if the data in the buffer is - * less than the skb header size, otherwise it will just attach the page as - * a frag to the skb. - * - * The function will then update the page offset if necessary and return - * true if the buffer can be reused by the adapter. - **/ -static void ixgbe_add_rx_frag(const struct libeth_fqe *rx_buffer, - struct sk_buff *skb, - unsigned int size) -{ - u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; - - skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, - rx_buffer->netmem, rx_buffer->offset + hr, - size, rx_buffer->truesize); -} - - static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, struct libeth_xdp_buff *xdp) @@ -2102,16 +2075,16 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); LIBETH_XDP_ONSTACK_BUFF(xdp); - struct sk_buff *skb = rx_ring->skb; unsigned int xdp_xmit = 0; + u32 rsc_append_cnt = 0; int xdp_res = 0; - xdp->data = NULL; - xdp->base.rxq = &rx_ring->xdp_rxq; + libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; struct libeth_fqe *rx_buffer; + struct sk_buff *skb; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -2132,44 +2105,42 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, dma_rmb(); rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; + libeth_xdp_process_buff(xdp, rx_buffer, size); - /* retrieve a buffer from the ring */ - if (!skb) { - libeth_xdp_process_buff(xdp, rx_buffer, size); - xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); - } else { - libeth_rx_sync_for_cpu(rx_buffer, size); - } - + cleaned_count++; + /* fetch next buffer in frame if non-eop */ + if (ixgbe_is_non_eop(rx_ring, rx_desc, &rsc_append_cnt)) + continue; + xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); if (xdp_res) { if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) xdp_xmit |= xdp_res; total_rx_packets++; - total_rx_bytes += size; - } else if (skb) { - ixgbe_add_rx_frag(rx_buffer, skb, size); - } else { - skb = xdp_build_skb_from_buff(&xdp->base); + total_rx_bytes += xdp_get_buff_len(&xdp->base); + rsc_append_cnt = 0; + xdp->data = NULL; + continue; } + skb = xdp_build_skb_from_buff(&xdp->base); + /* exit if we failed to retrieve a buffer */ - if (unlikely(!xdp_res && !skb)) { + if (unlikely(!skb)) { + rsc_append_cnt = 0; + libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; - cleaned_count++; break; } + xdp->data = NULL; - cleaned_count++; - - /* place incomplete frames back on ring for completion */ - if (ixgbe_is_non_eop(rx_ring, rx_desc, skb)) - continue; + /* apply RSC append count to skb if accumulated */ + IXGBE_CB(skb)->append_cnt = rsc_append_cnt; + rsc_append_cnt = 0; /* verify the packet layout is correct */ - if (xdp_res || - unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { + if (unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } @@ -2208,10 +2179,10 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_packets++; ixgbe_rx_skb(q_vector, skb); - skb = NULL; } - rx_ring->skb = skb; + /* place incomplete frames back on ring for completion */ + libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); if (xdp_xmit & IXGBE_XDP_REDIR) xdp_do_flush(); @@ -5261,11 +5232,8 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) goto skip_free; } - /* Free Rx ring sk_buff */ - if (rx_ring->skb) { - dev_kfree_skb(rx_ring->skb); - rx_ring->skb = NULL; - } + /* Free Rx ring xdp stash */ + libeth_xdp_return_stash(&rx_ring->xdp_stash); for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; @@ -10398,13 +10366,15 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) +static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) { - int i, frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; + u32 frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct bpf_prog *old_prog; bool need_reset; int num_queues; + bool requires_mbuf; if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) return -EINVAL; @@ -10413,15 +10383,19 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) return -EINVAL; /* verify ixgbe ring attributes are sufficient for XDP */ - for (i = 0; i < adapter->num_rx_queues; i++) { + for (int i = 0; i < adapter->num_rx_queues; i++) { struct ixgbe_ring *ring = adapter->rx_ring[i]; if (ring_is_rsc_enabled(ring)) return -EINVAL; } - if (frame_size > IXGBE_RXBUFFER_3K) - return -EINVAL; + requires_mbuf = frame_size > IXGBE_RX_PAGE_LEN(LIBETH_XDP_HEADROOM); + if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { + NL_SET_ERR_MSG_MOD(extack, + "Configured MTU requires non-linear frames and XDP prog does not support frags"); + return -EOPNOTSUPP; + } /* if the number of cpus is much larger than the maximum of queues, * we should stop it and then return with ENOMEM like before. @@ -10446,7 +10420,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (!prog) xdp_features_clear_redirect_target(dev); } else { - for (i = 0; i < adapter->num_rx_queues; i++) { + for (int i = 0; i < adapter->num_rx_queues; i++) { WRITE_ONCE(adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); } @@ -10461,7 +10435,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (need_reset && prog) { num_queues = min_t(int, adapter->num_rx_queues, adapter->num_xdp_queues); - for (i = 0; i < num_queues; i++) + for (int i = 0; i < num_queues; i++) if (adapter->xdp_ring[i]->xsk_pool) (void)ixgbe_xsk_wakeup(adapter->netdev, i, XDP_WAKEUP_RX); @@ -10477,7 +10451,7 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp) switch (xdp->command) { case XDP_SETUP_PROG: - return ixgbe_xdp_setup(dev, xdp->prog); + return ixgbe_xdp_setup(dev, xdp->prog, xdp->extack); case XDP_SETUP_XSK_POOL: return ixgbe_xsk_pool_setup(adapter, xdp->xsk.pool, xdp->xsk.queue_id); @@ -11313,7 +11287,8 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->priv_flags |= IFF_SUPP_NOFCS; netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_XSK_ZEROCOPY; + NETDEV_XDP_ACT_XSK_ZEROCOPY | + NETDEV_XDP_ACT_RX_SG; /* MTU range: 68 - 9710 */ netdev->min_mtu = ETH_MIN_MTU;