diff -upkr linux-2.6.23/include/linux/mm_types.h linux-2.6.23/include/linux/mm_types.h --- linux-2.6.23/include/linux/mm_types.h 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/include/linux/mm_types.h 2007-10-10 13:42:46.000000000 +0400 @@ -78,6 +78,15 @@ struct page { void *virtual; /* Kernel virtual address (NULL if not kmapped, ie. highmem) */ #endif /* WANT_PAGE_VIRTUAL */ + /* + * Used to implement support for notification on zero-copy TCP transfer + * completion. Not good to have this field here, it's better to have + * it in struct sk_buff, but it would make the code much more + * complicated and fragile, if maintained as a separate patch, since all + * skb then would have to contain only pages with the same value in this + * field. + */ + void *net_priv; }; #endif /* _LINUX_MM_TYPES_H */ diff -upkr linux-2.6.23/include/linux/net.h linux-2.6.23/include/linux/net.h --- linux-2.6.23/include/linux/net.h 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/include/linux/net.h 2007-10-10 13:42:46.000000000 +0400 @@ -57,6 +57,7 @@ typedef enum { #ifdef __KERNEL__ #include #include +#include #define SOCK_ASYNC_NOSPACE 0 #define SOCK_ASYNC_WAITDATA 1 @@ -319,5 +320,30 @@ extern int net_msg_cost; extern int net_msg_burst; #endif +/* Support for notification on zero-copy TCP transfer completion */ +#define NET_PAGE_CALLBACKS_DEFINED +typedef void (*net_get_page_callback_t)(struct page *page); +typedef void (*net_put_page_callback_t)(struct page *page); + +extern net_get_page_callback_t net_get_page_callback; +extern net_put_page_callback_t net_put_page_callback; + +extern int net_set_get_put_page_callbacks( + net_get_page_callback_t get_callback, + net_put_page_callback_t put_callback); + +static inline void net_get_page(struct page *page) +{ + if (page->net_priv != 0) + net_get_page_callback(page); + get_page(page); +} +static inline void net_put_page(struct page *page) +{ + if (page->net_priv != 0) + net_put_page_callback(page); + put_page(page); +} + #endif /* __KERNEL__ */ #endif /* _LINUX_NET_H */ diff -upkr linux-2.6.23/net/core/skbuff.c linux-2.6.23/net/core/skbuff.c --- linux-2.6.23/net/core/skbuff.c 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/net/core/skbuff.c 2007-10-10 13:42:46.000000000 +0400 @@ -262,7 +262,7 @@ static void skb_release_data(struct sk_b if (skb_shinfo(skb)->nr_frags) { int i; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); } if (skb_shinfo(skb)->frag_list) @@ -601,7 +601,7 @@ struct sk_buff *pskb_copy(struct sk_buff for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; - get_page(skb_shinfo(n)->frags[i].page); + net_get_page(skb_shinfo(n)->frags[i].page); } skb_shinfo(n)->nr_frags = i; } @@ -664,7 +664,7 @@ int pskb_expand_head(struct sk_buff *skb sizeof(struct skb_shared_info)); for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) - get_page(skb_shinfo(skb)->frags[i].page); + net_get_page(skb_shinfo(skb)->frags[i].page); if (skb_shinfo(skb)->frag_list) skb_clone_fraglist(skb); @@ -862,7 +862,7 @@ drop_pages: skb_shinfo(skb)->nr_frags = i; for (; i < nfrags; i++) - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); if (skb_shinfo(skb)->frag_list) skb_drop_fraglist(skb); @@ -1031,7 +1031,7 @@ pull_pages: k = 0; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { if (skb_shinfo(skb)->frags[i].size <= eat) { - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); eat -= skb_shinfo(skb)->frags[i].size; } else { skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; @@ -1600,7 +1600,7 @@ static inline void skb_split_no_header(s * where splitting is expensive. * 2. Split is accurately. We make this. */ - get_page(skb_shinfo(skb)->frags[i].page); + net_get_page(skb_shinfo(skb)->frags[i].page); skb_shinfo(skb1)->frags[0].page_offset += len - pos; skb_shinfo(skb1)->frags[0].size -= len - pos; skb_shinfo(skb)->frags[i].size = len - pos; @@ -1976,7 +1976,7 @@ struct sk_buff *skb_segment(struct sk_bu BUG_ON(i >= nfrags); *frag = skb_shinfo(skb)->frags[i]; - get_page(frag->page); + net_get_page(frag->page); size = frag->size; if (pos < offset) { diff -upkr linux-2.6.23/net/core/utils.c linux-2.6.23/net/core/utils.c --- linux-2.6.23/net/core/utils.c 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/net/core/utils.c 2007-10-10 13:43:13.000000000 +0400 @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,9 @@ int net_msg_burst __read_mostly = 10; int net_msg_warn __read_mostly = 1; EXPORT_SYMBOL(net_msg_warn); +net_get_page_callback_t net_get_page_callback __read_mostly; +net_put_page_callback_t net_put_page_callback __read_mostly; + /* * All net warning printk()s should be guarded by this function. */ @@ -293,3 +297,32 @@ out: } EXPORT_SYMBOL(in6_pton); + +int net_set_get_put_page_callbacks( + net_get_page_callback_t get_callback, + net_put_page_callback_t put_callback) +{ + int res = 0; + + if ((net_get_page_callback != NULL) && (get_callback != NULL) && + (net_get_page_callback != get_callback)) { + res = -EBUSY; + goto out; + } + + if ((net_put_page_callback != NULL) && (put_callback != NULL) && + (net_put_page_callback != put_callback)) { + res = -EBUSY; + goto out; + } + + net_get_page_callback = get_callback; + net_put_page_callback = put_callback; + +out: + return res; +} +EXPORT_SYMBOL(net_set_get_put_page_callbacks); + +EXPORT_SYMBOL(net_get_page_callback); +EXPORT_SYMBOL(net_put_page_callback); diff -upkr linux-2.6.23/net/ipv4/ip_output.c linux-2.6.23/net/ipv4/ip_output.c --- linux-2.6.23/net/ipv4/ip_output.c 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/net/ipv4/ip_output.c 2007-10-10 13:42:46.000000000 +0400 @@ -999,7 +999,7 @@ alloc_new_skb: err = -EMSGSIZE; goto error; } - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0); frag = &skb_shinfo(skb)->frags[i]; } @@ -1157,7 +1157,7 @@ ssize_t ip_append_page(struct sock *sk, if (skb_can_coalesce(skb, i, page, offset)) { skb_shinfo(skb)->frags[i-1].size += len; } else if (i < MAX_SKB_FRAGS) { - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, offset, len); } else { err = -EMSGSIZE; diff -upkr linux-2.6.23/net/ipv4/tcp.c linux-2.6.23/net/ipv4/tcp.c --- linux-2.6.23/net/ipv4/tcp.c 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/net/ipv4/tcp.c 2007-10-10 13:42:46.000000000 +0400 @@ -560,7 +560,7 @@ new_segment: if (can_coalesce) { skb_shinfo(skb)->frags[i - 1].size += copy; } else { - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, offset, copy); } @@ -765,7 +765,7 @@ new_segment: goto new_segment; } else if (page) { if (off == PAGE_SIZE) { - put_page(page); + net_put_page(page); TCP_PAGE(sk) = page = NULL; off = 0; } @@ -806,9 +806,9 @@ new_segment: } else { skb_fill_page_desc(skb, i, page, off, copy); if (TCP_PAGE(sk)) { - get_page(page); + net_get_page(page); } else if (off + copy < PAGE_SIZE) { - get_page(page); + net_get_page(page); TCP_PAGE(sk) = page; } } diff -upkr linux-2.6.23/net/ipv4/tcp_output.c linux-2.6.23/net/ipv4/tcp_output.c --- linux-2.6.23/net/ipv4/tcp_output.c 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/net/ipv4/tcp_output.c 2007-10-10 13:42:46.000000000 +0400 @@ -729,7 +729,7 @@ static void __pskb_trim_head(struct sk_b k = 0; for (i=0; inr_frags; i++) { if (skb_shinfo(skb)->frags[i].size <= eat) { - put_page(skb_shinfo(skb)->frags[i].page); + net_put_page(skb_shinfo(skb)->frags[i].page); eat -= skb_shinfo(skb)->frags[i].size; } else { skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; diff -upkr linux-2.6.23/net/ipv6/ip6_output.c linux-2.6.23/net/ipv6/ip6_output.c --- linux-2.6.23/net/ipv6/ip6_output.c 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/net/ipv6/ip6_output.c 2007-10-10 13:42:46.000000000 +0400 @@ -1295,7 +1295,7 @@ alloc_new_skb: err = -EMSGSIZE; goto error; } - get_page(page); + net_get_page(page); skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0); frag = &skb_shinfo(skb)->frags[i]; }