diff -upr linux-2.6.16.29/include/linux/mm.h linux-2.6.16.29/include/linux/mm.h --- linux-2.6.16.29/include/linux/mm.h 2006-09-12 22:02:10.000000000 +0400 +++ linux-2.6.16.29/include/linux/mm.h 2007-08-07 19:54:27.000000000 +0400 @@ -263,6 +263,15 @@ struct page { void *virtual; /* Kernel virtual address (NULL if not kmapped, ie. highmem) */ #endif /* WANT_PAGE_VIRTUAL */ + /* + * Used to implement support for notification on zero-copy TCP transfer + * completion. Not good to have this field here, it's better to have + * it in struct sk_buff, but it would make the code much more + * complicated and fragile, if maintained as a separate patch, since all + * skb then would have to contain only pages with the same value in this + * field. + */ + void *net_priv; }; #define page_private(page) ((page)->private) diff -upr linux-2.6.16.29/include/linux/net.h linux-2.6.16.29/include/linux/net.h --- linux-2.6.16.29/include/linux/net.h 2006-09-12 22:02:10.000000000 +0400 +++ linux-2.6.16.29/include/linux/net.h 2007-08-29 18:31:40.000000000 +0400 @@ -57,6 +57,7 @@ typedef enum { #define __SO_ACCEPTCON (1 << 16) /* performed a listen */ #ifdef __KERNEL__ +#include #define SOCK_ASYNC_NOSPACE 0 #define SOCK_ASYNC_WAITDATA 1 @@ -294,5 +295,30 @@ extern int net_msg_cost; extern int net_msg_burst; #endif +/* Support for notification on zero-copy TCP transfer completion */ +#define CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION +typedef void (*net_get_page_callback_t)(struct page *page); +typedef void (*net_put_page_callback_t)(struct page *page); + +extern net_get_page_callback_t net_get_page_callback; +extern net_put_page_callback_t net_put_page_callback; + +extern int net_set_get_put_page_callbacks( + net_get_page_callback_t get_callback, + net_put_page_callback_t put_callback); + +static inline void net_get_page(struct page *page) +{ + if (page->net_priv != 0) + net_get_page_callback(page); + get_page(page); +} +static inline void net_put_page(struct page *page) +{ + if (page->net_priv != 0) + net_put_page_callback(page); + put_page(page); +} + #endif /* __KERNEL__ */ #endif /* _LINUX_NET_H */ diff -upr linux-2.6.16.29/net/core/skbuff.c linux-2.6.16.29/net/core/skbuff.c --- linux-2.6.16.29/net/core/skbuff.c 2006-09-12 22:02:10.000000000 +0400 +++ linux-2.6.16.29/net/core/skbuff.c 2007-08-07 19:55:51.000000000 +0400 @@ -271,7 +271,7 @@ void skb_release_data(struct sk_buff *sk if (skb_shinfo(skb)->nr_frags) { int i; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); } if (skb_shinfo(skb)->frag_list) @@ -588,7 +588,7 @@ struct sk_buff *pskb_copy(struct sk_buff for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; - get_page(skb_shinfo(n)->frags[i].page); + net_get_page(skb_shinfo(n)->frags[i].page); } skb_shinfo(n)->nr_frags = i; } @@ -642,7 +642,7 @@ int pskb_expand_head(struct sk_buff *skb memcpy(data + size, skb->end, sizeof(struct skb_shared_info)); for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) - get_page(skb_shinfo(skb)->frags[i].page); + net_get_page(skb_shinfo(skb)->frags[i].page); if (skb_shinfo(skb)->frag_list) skb_clone_fraglist(skb); @@ -794,7 +794,7 @@ int ___pskb_trim(struct sk_buff *skb, un return -ENOMEM; } if (len <= offset) { - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); skb_shinfo(skb)->nr_frags--; } else { skb_shinfo(skb)->frags[i].size = len - offset; @@ -940,7 +940,7 @@ pull_pages: k = 0; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { if (skb_shinfo(skb)->frags[i].size <= eat) { - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); eat -= skb_shinfo(skb)->frags[i].size; } else { skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; @@ -1522,7 +1522,7 @@ static inline void skb_split_no_header(s * where splitting is expensive. * 2. Split is accurately. We make this. */ - get_page(skb_shinfo(skb)->frags[i].page); + net_get_page(skb_shinfo(skb)->frags[i].page); skb_shinfo(skb1)->frags[0].page_offset += len - pos; skb_shinfo(skb1)->frags[0].size -= len - pos; skb_shinfo(skb)->frags[i].size = len - pos; diff -upr linux-2.6.16.29/net/core/utils.c linux-2.6.16.29/net/core/utils.c --- linux-2.6.16.29/net/core/utils.c 2006-09-12 22:02:10.000000000 +0400 +++ linux-2.6.16.29/net/core/utils.c 2007-08-23 19:54:03.000000000 +0400 @@ -24,11 +24,15 @@ #include #include #include +#include #include #include #include +net_get_page_callback_t net_get_page_callback __read_mostly; +net_put_page_callback_t net_put_page_callback __read_mostly; + /* This is a maximally equidistributed combined Tausworthe generator based on code from GNU Scientific Library 1.5 (30 Jun 2004) @@ -190,3 +194,32 @@ __be32 in_aton(const char *str) } EXPORT_SYMBOL(in_aton); + +int net_set_get_put_page_callbacks( + net_get_page_callback_t get_callback, + net_put_page_callback_t put_callback) +{ + int res = 0; + + if ((net_get_page_callback != NULL) && (get_callback != NULL) && + (net_get_page_callback != get_callback)) { + res = -EBUSY; + goto out; + } + + if ((net_put_page_callback != NULL) && (put_callback != NULL) && + (net_put_page_callback != put_callback)) { + res = -EBUSY; + goto out; + } + + net_get_page_callback = get_callback; + net_put_page_callback = put_callback; + +out: + return res; +} +EXPORT_SYMBOL(net_set_get_put_page_callbacks); + +EXPORT_SYMBOL(net_get_page_callback); +EXPORT_SYMBOL(net_put_page_callback); diff -upr linux-2.6.16.29/net/ipv4/ip_output.c linux-2.6.16.29/net/ipv4/ip_output.c --- linux-2.6.16.29/net/ipv4/ip_output.c 2006-09-12 22:02:10.000000000 +0400 +++ linux-2.6.16.29/net/ipv4/ip_output.c 2007-08-07 19:54:27.000000000 +0400 @@ -997,7 +997,7 @@ alloc_new_skb: err = -EMSGSIZE; goto error; } - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0); frag = &skb_shinfo(skb)->frags[i]; } @@ -1155,7 +1155,7 @@ ssize_t ip_append_page(struct sock *sk, if (skb_can_coalesce(skb, i, page, offset)) { skb_shinfo(skb)->frags[i-1].size += len; } else if (i < MAX_SKB_FRAGS) { - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, offset, len); } else { err = -EMSGSIZE; diff -upr linux-2.6.16.29/net/ipv4/tcp.c linux-2.6.16.29/net/ipv4/tcp.c --- linux-2.6.16.29/net/ipv4/tcp.c 2006-09-12 22:02:10.000000000 +0400 +++ linux-2.6.16.29/net/ipv4/tcp.c 2007-08-07 19:54:27.000000000 +0400 @@ -558,7 +558,7 @@ new_segment: if (can_coalesce) { skb_shinfo(skb)->frags[i - 1].size += copy; } else { - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, offset, copy); } @@ -767,7 +767,7 @@ new_segment: goto new_segment; } else if (page) { if (off == PAGE_SIZE) { - put_page(page); + net_put_page(page); TCP_PAGE(sk) = page = NULL; off = 0; } @@ -808,9 +808,9 @@ new_segment: } else { skb_fill_page_desc(skb, i, page, off, copy); if (TCP_PAGE(sk)) { - get_page(page); + net_get_page(page); } else if (off + copy < PAGE_SIZE) { - get_page(page); + net_get_page(page); TCP_PAGE(sk) = page; } } diff -upr linux-2.6.16.29/net/ipv4/tcp_output.c linux-2.6.16.29/net/ipv4/tcp_output.c --- linux-2.6.16.29/net/ipv4/tcp_output.c 2006-09-12 22:02:10.000000000 +0400 +++ linux-2.6.16.29/net/ipv4/tcp_output.c 2007-08-07 19:54:27.000000000 +0400 @@ -633,7 +633,7 @@ static unsigned char *__pskb_trim_head(s k = 0; for (i=0; inr_frags; i++) { if (skb_shinfo(skb)->frags[i].size <= eat) { - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); eat -= skb_shinfo(skb)->frags[i].size; } else { skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; diff -upr linux-2.6.16.29/net/ipv6/ip6_output.c linux-2.6.16.29/net/ipv6/ip6_output.c --- linux-2.6.16.29/net/ipv6/ip6_output.c 2006-09-12 22:02:10.000000000 +0400 +++ linux-2.6.16.29/net/ipv6/ip6_output.c 2007-08-07 19:54:27.000000000 +0400 @@ -1100,7 +1100,7 @@ alloc_new_skb: err = -EMSGSIZE; goto error; } - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0); frag = &skb_shinfo(skb)->frags[i]; }