public-mirrors/linux
1
0
Fork 0
mirror of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-10-31 16:54:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

gro: Inline skb_gro_header and cache frag0 virtual address

Browse source 
The function skb_gro_header is called four times per packet which
quickly adds up at 10Gb/s.  This patch inlines it to allow better
optimisations.

Some architectures perform multiplication for page_address, which
is done by each skb_gro_header invocation.  This patch caches that
value in skb->cb to avoid the unnecessary multiplications.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Herbert Xu 2009-05-26 18:50:21 +00:00 • committed by David S. Miller
parent 42da6994ca
commit 78a478d0ef
2 changed files with 27 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
include/linux/netdevice.h
View file

@ -1008,6 +1008,9 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
void netif_napi_del(struct napi_struct *napi);
struct napi_gro_cb {
/* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
void *frag0;
/* This indicates where we are processing relative to skb->data. */
int data_offset;
@ -1107,9 +1110,9 @@ extern int dev_restart(struct net_device *dev);
#ifdef CONFIG_NETPOLL_TRAP
extern int netpoll_trap(void);
#endif
extern void *skb_gro_header(struct sk_buff *skb, unsigned int hlen);
extern int skb_gro_receive(struct sk_buff **head,
struct sk_buff *skb);
extern void skb_gro_reset_offset(struct sk_buff *skb);
static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
{
@ -1126,23 +1129,28 @@ static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
NAPI_GRO_CB(skb)->data_offset += len;
}
static inline void skb_gro_reset_offset(struct sk_buff *skb)
static inline void *skb_gro_header(struct sk_buff *skb, unsigned int hlen)
{
NAPI_GRO_CB(skb)->data_offset = 0;
unsigned int offset = skb_gro_offset(skb);
hlen += offset;
if (!NAPI_GRO_CB(skb)->frag0 ||
unlikely(skb_shinfo(skb)->frags[0].size + skb_headlen(skb) < hlen))
return pskb_may_pull(skb, hlen) ? skb->data + offset : NULL;
return NAPI_GRO_CB(skb)->frag0 + offset;
}
static inline void *skb_gro_mac_header(struct sk_buff *skb)
{
return skb_headlen(skb) ? skb_mac_header(skb) :
page_address(skb_shinfo(skb)->frags[0].page) +
skb_shinfo(skb)->frags[0].page_offset;
NAPI_GRO_CB(skb)->frag0;
}
static inline void *skb_gro_network_header(struct sk_buff *skb)
{
return skb_headlen(skb) ? skb_network_header(skb) :
page_address(skb_shinfo(skb)->frags[0].page) +
skb_shinfo(skb)->frags[0].page_offset + skb_network_offset(skb);
NAPI_GRO_CB(skb)->frag0 + skb_network_offset(skb);
}
static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,

27
net/core/dev.c
View file

@ -2390,21 +2390,6 @@ void napi_gro_flush(struct napi_struct *napi)
}
EXPORT_SYMBOL(napi_gro_flush);
void *skb_gro_header(struct sk_buff *skb, unsigned int hlen)
{
unsigned int offset = skb_gro_offset(skb);
hlen += offset;
if (unlikely(skb_headlen(skb) ||
skb_shinfo(skb)->frags[0].size < hlen ||
PageHighMem(skb_shinfo(skb)->frags[0].page)))
return pskb_may_pull(skb, hlen) ? skb->data + offset : NULL;
return page_address(skb_shinfo(skb)->frags[0].page) +
skb_shinfo(skb)->frags[0].page_offset + offset;
}
EXPORT_SYMBOL(skb_gro_header);
int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
{
struct sk_buff **pp = NULL;
@ -2520,6 +2505,18 @@ int napi_skb_finish(int ret, struct sk_buff *skb)
}
EXPORT_SYMBOL(napi_skb_finish);
void skb_gro_reset_offset(struct sk_buff *skb)
{
NAPI_GRO_CB(skb)->data_offset = 0;
NAPI_GRO_CB(skb)->frag0 = NULL;
if (!skb_headlen(skb) && !PageHighMem(skb_shinfo(skb)->frags[0].page))
NAPI_GRO_CB(skb)->frag0 =
page_address(skb_shinfo(skb)->frags[0].page) +
skb_shinfo(skb)->frags[0].page_offset;
}
EXPORT_SYMBOL(skb_gro_reset_offset);
int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
{
skb_gro_reset_offset(skb);