diff -upkr linux-2.6.25/include/linux/mm_types.h linux-2.6.25/include/linux/mm_types.h --- linux-2.6.25/include/linux/mm_types.h 2008-04-17 06:49:44.000000000 +0400 +++ linux-2.6.25/include/linux/mm_types.h 2008-04-18 13:38:38.000000000 +0400 @@ -88,6 +88,17 @@ struct page { void *virtual; /* Kernel virtual address (NULL if not kmapped, ie. highmem) */ #endif /* WANT_PAGE_VIRTUAL */ + + /* + * Used to implement support for notification on zero-copy TCP transfer + * completion. Not good to have this field here, it's better to have + * it in struct sk_buff, but it would make the code much more + * complicated and fragile, if maintained as a separate patch, since all + * skb then would have to contain only pages with the same value in this + * field. + */ + void *net_priv; + #ifdef CONFIG_CGROUP_MEM_RES_CTLR unsigned long page_cgroup; #endif diff -upkr linux-2.6.25/include/linux/net.h linux-2.6.25/include/linux/net.h --- linux-2.6.25/include/linux/net.h 2008-04-17 06:49:44.000000000 +0400 +++ linux-2.6.25/include/linux/net.h 2008-04-18 13:37:06.000000000 +0400 @@ -59,6 +59,7 @@ typedef enum { #ifdef __KERNEL__ #include #include +#include #define SOCK_ASYNC_NOSPACE 0 #define SOCK_ASYNC_WAITDATA 1 @@ -341,5 +342,30 @@ extern int net_msg_cost; extern int net_msg_burst; #endif +/* Support for notification on zero-copy TCP transfer completion */ +#define CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION +typedef void (*net_get_page_callback_t)(struct page *page); +typedef void (*net_put_page_callback_t)(struct page *page); + +extern net_get_page_callback_t net_get_page_callback; +extern net_put_page_callback_t net_put_page_callback; + +extern int net_set_get_put_page_callbacks( + net_get_page_callback_t get_callback, + net_put_page_callback_t put_callback); + +static inline void net_get_page(struct page *page) +{ + if (page->net_priv != 0) + net_get_page_callback(page); + get_page(page); +} +static inline void net_put_page(struct page *page) +{ + if (page->net_priv != 0) + net_put_page_callback(page); + put_page(page); +} + #endif /* __KERNEL__ */ #endif /* _LINUX_NET_H */ diff -upkr linux-2.6.25/net/core/skbuff.c linux-2.6.25/net/core/skbuff.c --- linux-2.6.25/net/core/skbuff.c 2008-04-17 06:49:44.000000000 +0400 +++ linux-2.6.25/net/core/skbuff.c 2008-04-18 13:37:06.000000000 +0400 @@ -297,7 +297,7 @@ static void skb_release_data(struct sk_b if (skb_shinfo(skb)->nr_frags) { int i; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); } if (skb_shinfo(skb)->frag_list) @@ -631,7 +631,7 @@ struct sk_buff *pskb_copy(struct sk_buff for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; - get_page(skb_shinfo(n)->frags[i].page); + net_get_page(skb_shinfo(n)->frags[i].page); } skb_shinfo(n)->nr_frags = i; } @@ -694,7 +694,7 @@ int pskb_expand_head(struct sk_buff *skb sizeof(struct skb_shared_info)); for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) - get_page(skb_shinfo(skb)->frags[i].page); + net_get_page(skb_shinfo(skb)->frags[i].page); if (skb_shinfo(skb)->frag_list) skb_clone_fraglist(skb); @@ -891,7 +891,7 @@ drop_pages: skb_shinfo(skb)->nr_frags = i; for (; i < nfrags; i++) - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); if (skb_shinfo(skb)->frag_list) skb_drop_fraglist(skb); @@ -1060,7 +1060,7 @@ pull_pages: k = 0; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { if (skb_shinfo(skb)->frags[i].size <= eat) { - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); eat -= skb_shinfo(skb)->frags[i].size; } else { skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; @@ -1840,7 +1840,7 @@ static inline void skb_split_no_header(s * where splitting is expensive. * 2. Split is accurately. We make this. */ - get_page(skb_shinfo(skb)->frags[i].page); + net_get_page(skb_shinfo(skb)->frags[i].page); skb_shinfo(skb1)->frags[0].page_offset += len - pos; skb_shinfo(skb1)->frags[0].size -= len - pos; skb_shinfo(skb)->frags[i].size = len - pos; @@ -2215,7 +2215,7 @@ struct sk_buff *skb_segment(struct sk_bu BUG_ON(i >= nfrags); *frag = skb_shinfo(skb)->frags[i]; - get_page(frag->page); + net_get_page(frag->page); size = frag->size; if (pos < offset) { diff -upkr linux-2.6.25/net/core/utils.c linux-2.6.25/net/core/utils.c --- linux-2.6.25/net/core/utils.c 2008-04-17 06:49:44.000000000 +0400 +++ linux-2.6.25/net/core/utils.c 2008-04-18 13:39:40.000000000 +0400 @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,12 @@ int net_msg_burst __read_mostly = 10; int net_msg_warn __read_mostly = 1; EXPORT_SYMBOL(net_msg_warn); +net_get_page_callback_t net_get_page_callback __read_mostly; +EXPORT_SYMBOL(net_get_page_callback); + +net_put_page_callback_t net_put_page_callback __read_mostly; +EXPORT_SYMBOL(net_put_page_callback); + /* * All net warning printk()s should be guarded by this function. */ @@ -283,6 +287,32 @@ out: EXPORT_SYMBOL(in6_pton); +int net_set_get_put_page_callbacks( + net_get_page_callback_t get_callback, + net_put_page_callback_t put_callback) +{ + int res = 0; + + if ((net_get_page_callback != NULL) && (get_callback != NULL) && + (net_get_page_callback != get_callback)) { + res = -EBUSY; + goto out; + } + + if ((net_put_page_callback != NULL) && (put_callback != NULL) && + (net_put_page_callback != put_callback)) { + res = -EBUSY; + goto out; + } + + net_get_page_callback = get_callback; + net_put_page_callback = put_callback; + +out: + return res; +} +EXPORT_SYMBOL(net_set_get_put_page_callbacks); + void inet_proto_csum_replace4(__sum16 *sum, struct sk_buff *skb, __be32 from, __be32 to, int pseudohdr) { diff -upkr linux-2.6.25/net/ipv4/ip_output.c linux-2.6.25/net/ipv4/ip_output.c --- linux-2.6.25/net/ipv4/ip_output.c 2008-04-17 06:49:44.000000000 +0400 +++ linux-2.6.25/net/ipv4/ip_output.c 2008-04-18 13:37:06.000000000 +0400 @@ -1017,7 +1017,7 @@ alloc_new_skb: err = -EMSGSIZE; goto error; } - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0); frag = &skb_shinfo(skb)->frags[i]; } @@ -1175,7 +1175,7 @@ ssize_t ip_append_page(struct sock *sk, if (skb_can_coalesce(skb, i, page, offset)) { skb_shinfo(skb)->frags[i-1].size += len; } else if (i < MAX_SKB_FRAGS) { - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, offset, len); } else { err = -EMSGSIZE; diff -upkr linux-2.6.25/net/ipv4/tcp.c linux-2.6.25/net/ipv4/tcp.c --- linux-2.6.25/net/ipv4/tcp.c 2008-04-17 06:49:44.000000000 +0400 +++ linux-2.6.25/net/ipv4/tcp.c 2008-04-18 13:37:06.000000000 +0400 @@ -713,7 +713,7 @@ new_segment: if (can_coalesce) { skb_shinfo(skb)->frags[i - 1].size += copy; } else { - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, offset, copy); } @@ -918,7 +918,7 @@ new_segment: goto new_segment; } else if (page) { if (off == PAGE_SIZE) { - put_page(page); + net_put_page(page); TCP_PAGE(sk) = page = NULL; off = 0; } @@ -959,9 +959,9 @@ new_segment: } else { skb_fill_page_desc(skb, i, page, off, copy); if (TCP_PAGE(sk)) { - get_page(page); + net_get_page(page); } else if (off + copy < PAGE_SIZE) { - get_page(page); + net_get_page(page); TCP_PAGE(sk) = page; } } diff -upkr linux-2.6.25/net/ipv4/tcp_output.c linux-2.6.25/net/ipv4/tcp_output.c --- linux-2.6.25/net/ipv4/tcp_output.c 2008-04-17 06:49:44.000000000 +0400 +++ linux-2.6.25/net/ipv4/tcp_output.c 2008-04-18 13:37:06.000000000 +0400 @@ -805,7 +805,7 @@ static void __pskb_trim_head(struct sk_b k = 0; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { if (skb_shinfo(skb)->frags[i].size <= eat) { - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); eat -= skb_shinfo(skb)->frags[i].size; } else { skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; diff -upkr linux-2.6.25/net/ipv6/ip6_output.c linux-2.6.25/net/ipv6/ip6_output.c --- linux-2.6.25/net/ipv6/ip6_output.c 2008-04-17 06:49:44.000000000 +0400 +++ linux-2.6.25/net/ipv6/ip6_output.c 2008-04-18 13:37:06.000000000 +0400 @@ -1332,7 +1332,7 @@ alloc_new_skb: err = -EMSGSIZE; goto error; } - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0); frag = &skb_shinfo(skb)->frags[i]; }