From 71dfda58aaaf4bf6b1bc59f9d8afa635fa1337d4 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Tue, 11 Nov 2014 09:26:34 -0800 Subject: [PATCH 1/5] net: Add device Rx page allocation function This patch implements __dev_alloc_pages and __dev_alloc_page. These are meant to replace the __skb_alloc_pages and __skb_alloc_page functions. The reason for doing this is that it occurred to me that __skb_alloc_page is supposed to be passed an sk_buff pointer, but it is NULL in all cases where it is used. Worse is that in the case of ixgbe it is passed NULL via the sk_buff pointer in the rx_buffer info structure which means the compiler is not correctly stripping it out. The naming for these functions is based on dev_alloc_skb and __dev_alloc_skb. There was originally a netdev_alloc_page, however that was passed a net_device pointer and this function is not so I thought it best to follow that naming scheme since that is the same difference between dev_alloc_skb and netdev_alloc_skb. In the case of anything greater than order 0 it is assumed that we want a compound page so __GFP_COMP is set for all allocations as we expect a compound page when assigning a page frag. The other change in this patch is to exploit the behaviors of the page allocator in how it handles flags. So for example we can always set __GFP_COMP and __GFP_MEMALLOC since they are ignored if they are not applicable or are overridden by another flag. Signed-off-by: Alexander Duyck Signed-off-by: David S. Miller --- include/linux/skbuff.h | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 103fbe8113f8..2e5221f1ec72 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2184,6 +2184,54 @@ static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev, return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC); } +/** + * __dev_alloc_pages - allocate page for network Rx + * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx + * @order: size of the allocation + * + * Allocate a new page. + * + * %NULL is returned if there is no free memory. +*/ +static inline struct page *__dev_alloc_pages(gfp_t gfp_mask, + unsigned int order) +{ + /* This piece of code contains several assumptions. + * 1. This is for device Rx, therefor a cold page is preferred. + * 2. The expectation is the user wants a compound page. + * 3. If requesting a order 0 page it will not be compound + * due to the check to see if order has a value in prep_new_page + * 4. __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to + * code in gfp_to_alloc_flags that should be enforcing this. + */ + gfp_mask |= __GFP_COLD | __GFP_COMP | __GFP_MEMALLOC; + + return alloc_pages_node(NUMA_NO_NODE, gfp_mask, order); +} + +static inline struct page *dev_alloc_pages(unsigned int order) +{ + return __dev_alloc_pages(GFP_ATOMIC, order); +} + +/** + * __dev_alloc_page - allocate a page for network Rx + * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx + * + * Allocate a new page. + * + * %NULL is returned if there is no free memory. + */ +static inline struct page *__dev_alloc_page(gfp_t gfp_mask) +{ + return __dev_alloc_pages(gfp_mask, 0); +} + +static inline struct page *dev_alloc_page(void) +{ + return __dev_alloc_page(GFP_ATOMIC); +} + /** * __skb_alloc_pages - allocate pages for ps-rx on a skb and preserve pfmemalloc data * @gfp_mask: alloc_pages_node mask. Set __GFP_NOMEMALLOC if not for network packet RX From aa9cd31c3f3ee36dc459b6070bacc581c1bfc334 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Tue, 11 Nov 2014 09:26:42 -0800 Subject: [PATCH 2/5] cxgb4/cxgb4vf: Replace __skb_alloc_page with __dev_alloc_page Drop the bloated use of __skb_alloc_page and replace it with __dev_alloc_page. In addition update the one other spot that is allocating a page so that it allocates with the correct flags. Cc: Hariprasad S Cc: Casey Leedom Signed-off-by: Alexander Duyck Signed-off-by: David S. Miller --- drivers/net/ethernet/chelsio/cxgb4/sge.c | 6 +++--- drivers/net/ethernet/chelsio/cxgb4vf/sge.c | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index 6399e008a7f7..dacd95008333 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -576,7 +576,7 @@ static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n, __be64 *d = &q->desc[q->pidx]; struct rx_sw_desc *sd = &q->sdesc[q->pidx]; - gfp |= __GFP_NOWARN | __GFP_COLD; + gfp |= __GFP_NOWARN; if (s->fl_pg_order == 0) goto alloc_small_pages; @@ -585,7 +585,7 @@ static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n, * Prefer large buffers */ while (n) { - pg = alloc_pages(gfp | __GFP_COMP, s->fl_pg_order); + pg = __dev_alloc_pages(gfp, s->fl_pg_order); if (unlikely(!pg)) { q->large_alloc_failed++; break; /* fall back to single pages */ @@ -615,7 +615,7 @@ static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n, alloc_small_pages: while (n--) { - pg = __skb_alloc_page(gfp, NULL); + pg = __dev_alloc_page(gfp); if (unlikely(!pg)) { q->alloc_failed++; break; diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c index 0e8d5b72c9cc..cd538afa40dd 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c @@ -602,6 +602,8 @@ static unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl, */ BUG_ON(fl->avail + n > fl->size - FL_PER_EQ_UNIT); + gfp |= __GFP_NOWARN; + /* * If we support large pages, prefer large buffers and fail over to * small pages if we can't allocate large pages to satisfy the refill. @@ -612,8 +614,7 @@ static unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl, goto alloc_small_pages; while (n) { - page = alloc_pages(gfp | __GFP_COMP | __GFP_NOWARN, - FL_PG_ORDER); + page = __dev_alloc_pages(gfp, FL_PG_ORDER); if (unlikely(!page)) { /* * We've failed inour attempt to allocate a "large @@ -657,7 +658,7 @@ static unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl, alloc_small_pages: while (n--) { - page = __skb_alloc_page(gfp | __GFP_NOWARN, NULL); + page = __dev_alloc_page(gfp); if (unlikely(!page)) { fl->alloc_failed++; break; From 5693d284dd718313ca2f63a189193d4189889183 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Tue, 11 Nov 2014 09:26:50 -0800 Subject: [PATCH 3/5] phonet: Replace calls to __skb_alloc_page with __dev_alloc_page Replace the calls to __skb_alloc_page that are passed NULL with calls to __dev_alloc_page. In addition remove __GFP_COLD flag from allocations as we only want it for the Rx buffer which is taken care of by __dev_alloc_skb, not for any secondary allocations such as the queue element transmit descriptors. Cc: Oliver Neukum Cc: Felipe Balbi Signed-off-by: Alexander Duyck Signed-off-by: David S. Miller --- drivers/net/usb/cdc-phonet.c | 6 +++--- drivers/usb/gadget/function/f_phonet.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/usb/cdc-phonet.c b/drivers/net/usb/cdc-phonet.c index 2ec1500d0077..415ce8b882c6 100644 --- a/drivers/net/usb/cdc-phonet.c +++ b/drivers/net/usb/cdc-phonet.c @@ -130,7 +130,7 @@ static int rx_submit(struct usbpn_dev *pnd, struct urb *req, gfp_t gfp_flags) struct page *page; int err; - page = __skb_alloc_page(gfp_flags | __GFP_NOMEMALLOC, NULL); + page = __dev_alloc_page(gfp_flags | __GFP_NOMEMALLOC); if (!page) return -ENOMEM; @@ -212,7 +212,7 @@ resubmit: if (page) put_page(page); if (req) - rx_submit(pnd, req, GFP_ATOMIC | __GFP_COLD); + rx_submit(pnd, req, GFP_ATOMIC); } static int usbpn_close(struct net_device *dev); @@ -231,7 +231,7 @@ static int usbpn_open(struct net_device *dev) for (i = 0; i < rxq_size; i++) { struct urb *req = usb_alloc_urb(0, GFP_KERNEL); - if (!req || rx_submit(pnd, req, GFP_KERNEL | __GFP_COLD)) { + if (!req || rx_submit(pnd, req, GFP_KERNEL)) { usb_free_urb(req); usbpn_close(dev); return -ENOMEM; diff --git a/drivers/usb/gadget/function/f_phonet.c b/drivers/usb/gadget/function/f_phonet.c index b9cfc1571d71..cde73974a52e 100644 --- a/drivers/usb/gadget/function/f_phonet.c +++ b/drivers/usb/gadget/function/f_phonet.c @@ -303,7 +303,7 @@ pn_rx_submit(struct f_phonet *fp, struct usb_request *req, gfp_t gfp_flags) struct page *page; int err; - page = __skb_alloc_page(gfp_flags | __GFP_NOMEMALLOC, NULL); + page = __dev_alloc_page(gfp_flags | __GFP_NOMEMALLOC); if (!page) return -ENOMEM; @@ -377,7 +377,7 @@ static void pn_rx_complete(struct usb_ep *ep, struct usb_request *req) if (page) put_page(page); if (req) - pn_rx_submit(fp, req, GFP_ATOMIC | __GFP_COLD); + pn_rx_submit(fp, req, GFP_ATOMIC); } /*-------------------------------------------------------------------------*/ @@ -437,7 +437,7 @@ static int pn_set_alt(struct usb_function *f, unsigned intf, unsigned alt) netif_carrier_on(dev); for (i = 0; i < phonet_rxq_size; i++) - pn_rx_submit(fp, fp->out_reqv[i], GFP_ATOMIC | __GFP_COLD); + pn_rx_submit(fp, fp->out_reqv[i], GFP_ATOMIC); } spin_unlock(&port->lock); return 0; From 42b17f09550170fd238d76368e8fa8dfcc893082 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Tue, 11 Nov 2014 09:26:57 -0800 Subject: [PATCH 4/5] fm10k/igb/ixgbe: Replace __skb_alloc_page with dev_alloc_page The Intel drivers were pretty much just using the plain vanilla GFP flags in their calls to __skb_alloc_page so this change makes it so that they use dev_alloc_page which just uses GFP_ATOMIC for the gfp_flags value. Cc: Jeff Kirsher Cc: Matthew Vick Cc: Don Skidmore Signed-off-by: Alexander Duyck Acked-by: Jeff Kirsher Signed-off-by: David S. Miller --- drivers/net/ethernet/intel/fm10k/fm10k_main.c | 2 +- drivers/net/ethernet/intel/igb/igb_main.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_main.c b/drivers/net/ethernet/intel/fm10k/fm10k_main.c index e645af412e76..73457ede53ec 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_main.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_main.c @@ -83,7 +83,7 @@ static bool fm10k_alloc_mapped_page(struct fm10k_ring *rx_ring, return true; /* alloc new page for storage */ - page = alloc_page(GFP_ATOMIC | __GFP_COLD); + page = dev_alloc_page(); if (unlikely(!page)) { rx_ring->rx_stats.alloc_failed++; return false; diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index a2d72a87cbde..1e35fae7a62b 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -6988,7 +6988,7 @@ static bool igb_alloc_mapped_page(struct igb_ring *rx_ring, return true; /* alloc new page for storage */ - page = __skb_alloc_page(GFP_ATOMIC | __GFP_COLD, NULL); + page = dev_alloc_page(); if (unlikely(!page)) { rx_ring->rx_stats.alloc_failed++; return false; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index a5ca877eac1a..f5fcba4f9d21 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1444,8 +1444,7 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, /* alloc new page for storage */ if (likely(!page)) { - page = __skb_alloc_pages(GFP_ATOMIC | __GFP_COLD | __GFP_COMP, - bi->skb, ixgbe_rx_pg_order(rx_ring)); + page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring)); if (unlikely(!page)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; From 160d2aba550b23c6a538158511d5adccc400f04c Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Tue, 11 Nov 2014 09:27:05 -0800 Subject: [PATCH 5/5] net: Remove __skb_alloc_page and __skb_alloc_pages Remove the two functions which are now dead code. Signed-off-by: Alexander Duyck Signed-off-by: David S. Miller --- include/linux/skbuff.h | 43 ------------------------------------------ 1 file changed, 43 deletions(-) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 2e5221f1ec72..73c370e615de 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2232,49 +2232,6 @@ static inline struct page *dev_alloc_page(void) return __dev_alloc_page(GFP_ATOMIC); } -/** - * __skb_alloc_pages - allocate pages for ps-rx on a skb and preserve pfmemalloc data - * @gfp_mask: alloc_pages_node mask. Set __GFP_NOMEMALLOC if not for network packet RX - * @skb: skb to set pfmemalloc on if __GFP_MEMALLOC is used - * @order: size of the allocation - * - * Allocate a new page. - * - * %NULL is returned if there is no free memory. -*/ -static inline struct page *__skb_alloc_pages(gfp_t gfp_mask, - struct sk_buff *skb, - unsigned int order) -{ - struct page *page; - - gfp_mask |= __GFP_COLD; - - if (!(gfp_mask & __GFP_NOMEMALLOC)) - gfp_mask |= __GFP_MEMALLOC; - - page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order); - if (skb && page && page->pfmemalloc) - skb->pfmemalloc = true; - - return page; -} - -/** - * __skb_alloc_page - allocate a page for ps-rx for a given skb and preserve pfmemalloc data - * @gfp_mask: alloc_pages_node mask. Set __GFP_NOMEMALLOC if not for network packet RX - * @skb: skb to set pfmemalloc on if __GFP_MEMALLOC is used - * - * Allocate a new page. - * - * %NULL is returned if there is no free memory. - */ -static inline struct page *__skb_alloc_page(gfp_t gfp_mask, - struct sk_buff *skb) -{ - return __skb_alloc_pages(gfp_mask, skb, 0); -} - /** * skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page * @page: The page that was allocated from skb_alloc_page