diff --git a/Makefile b/Makefile index 4c1a8d7b..c288af19 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,8 @@ endif LINUX_VERSION ?= $(shell uname -r) KDIR ?= /lib/modules/$(LINUX_VERSION)/build +#CC = gcc-8 +CC = gcc LINUX_SRC_DIR ?= ../net-next @@ -110,3 +112,6 @@ printClean-%: $(MAKE) -C $(KDIR) M=$(shell pwd) $@ endif + +# Prevents warnings related to the __init annotation for homa_load. +CFLAGS_homa_plumbing.o += -Wno-missing-attributes diff --git a/homa_devel.c b/homa_devel.c index 72299ff2..c4b913cc 100644 --- a/homa_devel.c +++ b/homa_devel.c @@ -1283,6 +1283,8 @@ void homa_validate_rbtree(struct rb_node *node, int depth, char *message) tt_printk(); BUG_ON(1); } +#else + return; #endif /* __UNIT_TEST__ */ } #endif /* See strip.py */ @@ -1303,11 +1305,9 @@ int homa_tcp_checksum(struct sk_buff *skb) data_csum = skb_checksum(skb, skb_transport_offset(skb), tcp_len, 0); if (skb_is_ipv6(skb)) { - const struct ipv6hdr *ip6h = ipv6_hdr(skb); - // Fold the manual sum with the IPv6 pseudo-header - return csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, tcp_len, - IPPROTO_TCP, data_csum); + return csum_ipv6_magic(&ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr, + tcp_len, IPPROTO_TCP, data_csum); } else { const struct iphdr *iph = ip_hdr(skb); diff --git a/homa_grant.c b/homa_grant.c index 039266c9..5efee563 100644 --- a/homa_grant.c +++ b/homa_grant.c @@ -118,6 +118,7 @@ static struct ctl_table grant_ctl_table[] = { .mode = 0644, .proc_handler = homa_grant_dointvec }, + {} }; #endif /* See strip.py */ @@ -1205,7 +1206,7 @@ void homa_grant_update_sysctl_deps(struct homa_grant *grant) * * Return: 0 for success, nonzero for error. */ -int homa_grant_dointvec(const struct ctl_table *table, int write, +int homa_grant_dointvec(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { struct ctl_table table_copy; diff --git a/homa_grant.h b/homa_grant.h index be93ca7d..df34b3ef 100644 --- a/homa_grant.h +++ b/homa_grant.h @@ -243,7 +243,7 @@ void homa_grant_adjust_peer(struct homa_grant *grant, void homa_grant_check_fifo(struct homa_grant *grant); void homa_grant_check_needy(struct homa_grant *grant); void homa_grant_check_rpc(struct homa_rpc *rpc); -int homa_grant_dointvec(const struct ctl_table *table, int write, +int homa_grant_dointvec(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); void homa_grant_find_oldest(struct homa_grant *grant); int homa_grant_find_victim(struct homa_grant *grant, struct homa_rpc *rpc); diff --git a/homa_impl.h b/homa_impl.h index bc2b3bbc..e4a9e95f 100644 --- a/homa_impl.h +++ b/homa_impl.h @@ -43,11 +43,10 @@ #include #include #include +#include #include #include #include -#include -#include #ifndef __UPSTREAM__ /* See strip.py */ #include "homa.h" @@ -81,6 +80,11 @@ struct homa_peer; struct homa_rpc; struct homa_sock; +/* Features not present in all kernels: */ +#ifndef __cond_acquires +#define __cond_acquires(x) +#endif + #ifndef __STRIP__ /* See strip.py */ #include "timetrace.h" #include "homa_metrics.h" @@ -723,7 +727,7 @@ int homa_net_start(struct net *net); __poll_t homa_poll(struct file *file, struct socket *sock, struct poll_table_struct *wait); int homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, - int flags, int *addr_len); + int flags, int noblock, int *addr_len); void homa_request_retrans(struct homa_rpc *rpc); void homa_resend_pkt(struct sk_buff *skb, struct homa_rpc *rpc, struct homa_sock *hsk); @@ -755,7 +759,7 @@ void homa_xmit_unknown(struct sk_buff *skb, struct homa_sock *hsk); #ifndef __STRIP__ /* See strip.py */ void homa_cutoffs_pkt(struct sk_buff *skb, struct homa_sock *hsk); -int homa_dointvec(const struct ctl_table *table, int write, +int homa_dointvec(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); void homa_incoming_sysctl_changed(struct homa *homa); int homa_ioc_abort(struct socket *sock, unsigned long arg); @@ -764,7 +768,7 @@ int homa_message_in_init(struct homa_rpc *rpc, int length, void homa_prios_changed(struct homa *homa); void homa_resend_data(struct homa_rpc *rpc, int start, int end, int priority); -int homa_sysctl_softirq_cores(const struct ctl_table *table, +int homa_sysctl_softirq_cores(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int homa_unsched_priority(struct homa *homa, struct homa_peer *peer, diff --git a/homa_incoming.c b/homa_incoming.c index 3393784c..e9cf9493 100644 --- a/homa_incoming.c +++ b/homa_incoming.c @@ -165,16 +165,14 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) { struct homa_data_hdr *h = (struct homa_data_hdr *)skb->data; struct homa_gap *gap, *dummy, *gap2; - u32 start = ntohl(h->seg.offset); - u32 length = homa_data_len(skb); - enum skb_drop_reason reason; - u32 end = start + length; + int start = ntohl(h->seg.offset); + int length = homa_data_len(skb); + int end = start + length; if (start >= rpc->msgin.length || length > (rpc->msgin.length - start)) { tt_record3("Packet extended past message end; id %d, offset %d, length %d", rpc->id, start, length); - reason = SKB_DROP_REASON_PKT_TOO_BIG; goto discard; } @@ -190,7 +188,6 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) rpc->msgin.recv_end, start)) { tt_record2("Couldn't allocate gap for id %d (start %d): no memory", rpc->id, start); - reason = SKB_DROP_REASON_NOMEM; goto discard; } tt_record3("Created new gap for id %d: start %d, end %d", @@ -210,13 +207,11 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) if (start < gap->start) { tt_record4("Packet overlaps gap start: id %d, start %d, end %d, gap_start %d", rpc->id, start, end, gap->start); - reason = SKB_DROP_REASON_DUP_FRAG; goto discard; } if (end > gap->end) { tt_record4("Packet overlaps gap end: id %d, start %d, end %d, gap_end %d", rpc->id, start, end, gap->start); - reason = SKB_DROP_REASON_DUP_FRAG; goto discard; } tt_record4("Increasing start for gap for id %d, old start %d, new %d, end %d", @@ -238,7 +233,6 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) if (end > gap->end) { tt_record4("Packet overlaps gap end: id %d, start %d, end %d, gap_end %d", rpc->id, start, end, gap->start); - reason = SKB_DROP_REASON_DUP_FRAG; goto discard; } tt_record4("Decreasing end for gap for id %d, old end %d, new %d, start %d", @@ -252,7 +246,6 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) if (!gap2) { tt_record2("Couldn't allocate gap for split for id %d (start %d): no memory", rpc->id, end); - reason = SKB_DROP_REASON_NOMEM; goto discard; } tt_record4("Splitting gap for id %d; old gap start %d, end %d, pkt_start %d", @@ -262,7 +255,6 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) goto keep; } /* Packet doesn't overlap any gap, so it is a duplicate. */ - reason = SKB_DROP_REASON_DUP_FRAG; discard: #ifndef __STRIP__ /* See strip.py */ @@ -273,7 +265,7 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) #endif /* See strip.py */ tt_record4("homa_add_packet discarding packet for id %d, offset %d, length %d, retransmit %d", rpc->id, start, length, h->retransmit); - kfree_skb_reason(skb, reason); + kfree_skb(skb); return; keep: @@ -369,6 +361,7 @@ int homa_copy_to_user(struct homa_rpc *rpc) int offset = ntohl(h->seg.offset); int buf_bytes, chunk_size; struct iov_iter iter; + struct iovec iov; int copied = 0; char __user *dst; @@ -398,13 +391,12 @@ int homa_copy_to_user(struct homa_rpc *rpc) } chunk_size = buf_bytes; } - error = import_ubuf(READ, dst, chunk_size, - &iter); - if (error) - goto free_skbs; + iov.iov_base = dst; + iov.iov_len = chunk_size; + iov_iter_init(&iter, READ, &iov, 1, chunk_size); error = skb_copy_datagram_iter(skbs[i], sizeof(*h) + - copied, &iter, + copied, &iter, chunk_size); if (error) goto free_skbs; @@ -466,8 +458,8 @@ void homa_dispatch_pkts(struct sk_buff *skb) hsk = homa_sock_find(hnet, dport); if (!hsk || (!homa_is_client(id) && !hsk->is_server)) { if (skb_is_ipv6(skb)) - icmp6_send(skb, ICMPV6_DEST_UNREACH, - ICMPV6_PORT_UNREACH, 0, NULL, IP6CB(skb)); + icmpv6_send(skb, ICMPV6_DEST_UNREACH, + ICMPV6_PORT_UNREACH, 0); else icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0); diff --git a/homa_metrics.c b/homa_metrics.c index be5a83d9..3bdd883b 100644 --- a/homa_metrics.c +++ b/homa_metrics.c @@ -10,10 +10,10 @@ DEFINE_PER_CPU(struct homa_metrics, homa_metrics); /* Describes file operations implemented for /proc/net/homa_metrics. */ static const struct proc_ops homa_metrics_ops = { - .proc_open = homa_metrics_open, - .proc_read = homa_metrics_read, - .proc_lseek = homa_metrics_lseek, - .proc_release = homa_metrics_release, + .proc_open = homa_metrics_open, + .proc_read = homa_metrics_read, + .proc_lseek = homa_metrics_lseek, + .proc_release = homa_metrics_release, }; /* Global information used to export metrics information through a file in diff --git a/homa_offload.c b/homa_offload.c index 2903b6e9..7fd86270 100644 --- a/homa_offload.c +++ b/homa_offload.c @@ -8,6 +8,9 @@ #include "homa_offload.h" #include "homa_pacer.h" #include "homa_qdisc.h" +#include "homa_wire.h" +#include +#include DEFINE_PER_CPU(struct homa_offload_core, homa_offload_core); @@ -81,11 +84,11 @@ void homa_set_softirq_cpu(struct sk_buff *skb, int cpu) int hash; rcu_read_lock(); - sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table); + sock_flow_table = rcu_dereference(rps_sock_flow_table); if (sock_flow_table) { - hash = cpu + net_hotdata.rps_cpu_mask + 1; + hash = cpu + rps_cpu_mask + 1; if (sock_flow_table->ents[hash] != hash) { - sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table); + sock_flow_table = rcu_dereference(rps_sock_flow_table); sock_flow_table->ents[hash] = hash; } __skb_set_sw_hash(skb, hash, false); diff --git a/homa_peer.c b/homa_peer.c index 8299db64..b38a6272 100644 --- a/homa_peer.c +++ b/homa_peer.c @@ -64,6 +64,7 @@ static struct ctl_table peer_ctl_table[] = { .mode = 0644, .proc_handler = homa_peer_dointvec }, + {} }; #endif /* See strip.py */ @@ -508,25 +509,24 @@ struct dst_entry *homa_get_dst(struct homa_peer *peer, struct homa_sock *hsk) */ int homa_peer_reset_dst(struct homa_peer *peer, struct homa_sock *hsk) { - struct dst_entry *dst; - struct flowi flow; + struct dst_entry *dst, *old; int result = 0; homa_peer_lock(peer); - memset(&flow, 0, sizeof(flow)); + memset(&peer->flow, 0, sizeof(peer->flow)); if (hsk->sock.sk_family == AF_INET) { struct rtable *rt; - flowi4_init_output(&flow.u.ip4, hsk->sock.sk_bound_dev_if, + flowi4_init_output(&peer->flow.u.ip4, hsk->sock.sk_bound_dev_if, hsk->sock.sk_mark, hsk->inet.tos, RT_SCOPE_UNIVERSE, hsk->sock.sk_protocol, 0, ipv6_to_ipv4(peer->addr), hsk->inet.inet_saddr, 0, 0, hsk->sock.sk_uid); security_sk_classify_flow(&hsk->sock, - &flow.u.__fl_common); + &peer->flow.u.__fl_common); rt = ip_route_output_flow(sock_net(&hsk->sock), - &flow.u.ip4, &hsk->sock); + &peer->flow.u.ip4, &hsk->sock); if (IS_ERR(rt)) { result = PTR_ERR(rt); INC_METRIC(peer_route_errors, 1); @@ -536,27 +536,29 @@ int homa_peer_reset_dst(struct homa_peer *peer, struct homa_sock *hsk) peer->dst_cookie = 0; } else { /* This code is derived from code in tcp_v6_connect. */ - flow.u.ip6.flowi6_proto = hsk->sock.sk_protocol; - flow.u.ip6.daddr = peer->addr; - flow.u.ip6.saddr = hsk->inet.pinet6->saddr; - flow.u.ip6.flowlabel = ip6_make_flowinfo(hsk->inet.tos, 0); - flow.u.ip6.flowi6_oif = hsk->sock.sk_bound_dev_if; - flow.u.ip6.flowi6_mark = hsk->sock.sk_mark; - flow.u.ip6.fl6_dport = 0; - flow.u.ip6.fl6_sport = 0; - flow.u.ip6.flowi6_uid = hsk->sock.sk_uid; + peer->flow.u.ip6.flowi6_proto = hsk->sock.sk_protocol; + peer->flow.u.ip6.daddr = peer->addr; + peer->flow.u.ip6.saddr = hsk->inet.pinet6->saddr; + peer->flow.u.ip6.flowlabel = ip6_make_flowinfo(hsk->inet.tos, + 0); + peer->flow.u.ip6.flowi6_oif = hsk->sock.sk_bound_dev_if; + peer->flow.u.ip6.flowi6_mark = hsk->sock.sk_mark; + peer->flow.u.ip6.fl6_dport = 0; + peer->flow.u.ip6.fl6_sport = 0; + peer->flow.u.ip6.flowi6_uid = hsk->sock.sk_uid; security_sk_classify_flow(&hsk->sock, - &flow.u.__fl_common); + &peer->flow.u.__fl_common); dst = ip6_dst_lookup_flow(sock_net(&hsk->sock), &hsk->sock, - &flow.u.ip6, NULL); + &peer->flow.u.ip6, + &peer->addr); + if (IS_ERR(dst)) { result = PTR_ERR(dst); INC_METRIC(peer_route_errors, 1); goto done; } - peer->dst_cookie = rt6_get_cookie(dst_rt6_info(dst)); + peer->dst_cookie = rt6_get_cookie((struct rt6_info *)dst); } - memcpy(&peer->flow, &flow, sizeof(flow)); /* From the standpoint of homa_get_dst, peer->dst is not updated * atomically with peer->dst_cookie, which means homa_get_dst could @@ -565,7 +567,9 @@ int homa_peer_reset_dst(struct homa_peer *peer, struct homa_sock *hsk) * a lost packet) or a valid dst to be replaced (resulting in * unnecessary work). */ - dst_release(rcu_replace_pointer(peer->dst, dst, true)); + old = rcu_dereference_protected(peer->dst, lockdep_is_held(&peer->lock)); + rcu_assign_pointer(peer->dst, dst); + dst_release(old); done: homa_peer_unlock(peer); @@ -722,7 +726,7 @@ void homa_peer_update_sysctl_deps(struct homa_peertab *peertab) * * Return: 0 for success, nonzero for error. */ -int homa_peer_dointvec(const struct ctl_table *table, int write, +int homa_peer_dointvec(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { struct homa_peertab *peertab; diff --git a/homa_peer.h b/homa_peer.h index c3faa7ac..287f30f0 100644 --- a/homa_peer.h +++ b/homa_peer.h @@ -289,7 +289,7 @@ struct homa_peer *homa_peer_alloc(struct homa_sock *hsk, const struct in6_addr *addr); struct homa_peertab *homa_peer_alloc_peertab(void); -int homa_peer_dointvec(const struct ctl_table *table, int write, +int homa_peer_dointvec(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); void homa_peer_free(struct rcu_head *head); void homa_peer_free_net(struct homa_net *hnet); diff --git a/homa_plumbing.c b/homa_plumbing.c index e90093a4..6ef2bca7 100644 --- a/homa_plumbing.c +++ b/homa_plumbing.c @@ -120,7 +120,6 @@ static struct proto homav6_prot = { .hash = homa_hash, .unhash = homa_unhash, .obj_size = sizeof(struct homa_v6_sock), - .ipv6_pinfo_offset = offsetof(struct homa_v6_sock, inet6), .no_autobind = 1, }; @@ -413,6 +412,7 @@ static struct ctl_table homa_ctl_table[] = { .mode = 0644, .proc_handler = homa_dointvec }, + {} }; #endif /* See strip.py */ @@ -1057,12 +1057,12 @@ int homa_setsockopt(struct sock *sk, int level, int optname, u64 start = homa_clock(); #endif /* See strip.py */ - if (optlen != sizeof(struct homa_rcvbuf_args)) { + if (optlen != sizeof(args)) { hsk->error_msg = "invalid optlen argument: must be sizeof(struct homa_rcvbuf_args)"; return -EINVAL; } - if (copy_from_sockptr(&args, optval, optlen)) { + if (unlikely(copy_from_sockptr(&args, optval, optlen))) { hsk->error_msg = "invalid address for homa_rcvbuf_args"; return -EFAULT; } @@ -1088,7 +1088,7 @@ int homa_setsockopt(struct sock *sk, int level, int optname, return -EINVAL; } - if (copy_from_sockptr(&arg, optval, optlen)) { + if (unlikely(copy_from_sockptr(&arg, optval, optlen))) { hsk->error_msg = "invalid address for SO_HOMA_SERVER value"; return -EFAULT; } @@ -1126,7 +1126,7 @@ int homa_getsockopt(struct sock *sk, int level, int optname, void *result; int len; - if (copy_from_sockptr(&len, USER_SOCKPTR(optlen), sizeof(int))) { + if (unlikely(copy_from_user(&len, optlen, sizeof(int)))) { hsk->error_msg = "invalid address for optlen argument to getsockopt"; return -EFAULT; } @@ -1160,12 +1160,12 @@ int homa_getsockopt(struct sock *sk, int level, int optname, return -ENOPROTOOPT; } - if (copy_to_sockptr(USER_SOCKPTR(optlen), &len, sizeof(int))) { + if (copy_to_user(optlen, &len, sizeof(int))) { hsk->error_msg = "couldn't update optlen argument to getsockopt: read-only?"; return -EFAULT; } - if (copy_to_sockptr(USER_SOCKPTR(optval), result, len)) { + if (copy_to_user(optval, result, len)) { hsk->error_msg = "couldn't update optval argument to getsockopt: read-only?"; return -EFAULT; } @@ -1205,12 +1205,6 @@ int homa_sendmsg(struct sock *sk, struct msghdr *msg, size_t length) goto error; } - if (unlikely(!msg->msg_control_is_user)) { - tt_record("homa_sendmsg error: !msg->msg_control_is_user"); - hsk->error_msg = "msg_control argument for sendmsg isn't in user space"; - result = -EINVAL; - goto error; - } if (unlikely(copy_from_user(&args, (void __user *)msg->msg_control, sizeof(args)))) { hsk->error_msg = "invalid address for msg_control argument to sendmsg"; @@ -1268,8 +1262,8 @@ int homa_sendmsg(struct sock *sk, struct msghdr *msg, size_t length) args.id = rpc->id; homa_rpc_unlock(rpc); /* Locked by homa_rpc_alloc_client. */ - if (unlikely(copy_to_user((void __user *)msg->msg_control, - &args, sizeof(args)))) { + if (unlikely(copy_to_user((void __user *)msg->msg_control, &args, + sizeof(args)))) { homa_rpc_lock(rpc); hsk->error_msg = "couldn't update homa_sendmsg_args argument to sendmsg: read-only?"; result = -EFAULT; @@ -1355,16 +1349,16 @@ int homa_sendmsg(struct sock *sk, struct msghdr *msg, size_t length) * @len: Total bytes of space available in msg->msg_iov; not used. * @flags: Flags from system call; only MSG_DONTWAIT is used. * @addr_len: Store the length of the sender address here + * @noblock: Non-zero means MSG_DONTWAIT was specified * Return: The length of the message on success, otherwise a negative * errno. Sets hsk->error_msg on errors. */ int homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, - int *addr_len) + int noblock, int *addr_len) { struct homa_sock *hsk = homa_sk(sk); struct homa_recvmsg_args control; struct homa_rpc *rpc = NULL; - int nonblocking; int result; IF_NO_STRIP(u64 start = homa_clock()); @@ -1417,7 +1411,6 @@ int homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, goto done; } - nonblocking = flags & MSG_DONTWAIT; if (control.id != 0) { rpc = homa_rpc_find_client(hsk, control.id); /* Locks RPC. */ if (!rpc) { @@ -1426,14 +1419,14 @@ int homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, goto done; } homa_rpc_hold(rpc); - result = homa_wait_private(rpc, nonblocking); + result = homa_wait_private(rpc, noblock); if (result != 0) { hsk->error_msg = "error while waiting for private RPC to complete"; control.id = 0; goto done; } } else { - rpc = homa_wait_shared(hsk, nonblocking); + rpc = homa_wait_shared(hsk, noblock); if (IS_ERR(rpc)) { /* If we get here, it means there was an error that * prevented us from finding an RPC to return. Errors @@ -1573,7 +1566,6 @@ int homa_softirq(struct sk_buff *skb) { struct sk_buff *packets, *other_pkts, *next; struct sk_buff **prev_link, **other_link; - enum skb_drop_reason reason; struct homa_common_hdr *h; int header_offset; @@ -1611,7 +1603,6 @@ int homa_softirq(struct sk_buff *skb) pr_notice("Homa can't handle fragmented packet (no space for header); discarding\n"); #endif /* See strip.py */ UNIT_LOG("", "pskb discard"); - reason = SKB_DROP_REASON_HDR_TRUNC; goto discard; } header_offset = skb_transport_header(skb) - skb->data; @@ -1633,7 +1624,6 @@ int homa_softirq(struct sk_buff *skb) skb->len - header_offset); #endif /* See strip.py */ INC_METRIC(short_packets, 1); - reason = SKB_DROP_REASON_PKT_TOO_SMALL; goto discard; } @@ -1652,7 +1642,6 @@ int homa_softirq(struct sk_buff *skb) homa_local_id(h->sender_id)); tt_freeze(); } - reason = SKB_CONSUMED; goto discard; } #endif /* See strip.py */ @@ -1674,7 +1663,7 @@ int homa_softirq(struct sk_buff *skb) discard: *prev_link = skb->next; - kfree_skb_reason(skb, reason); + kfree_skb(skb); } /* Now process the longer packets. Each iteration of this loop @@ -1768,6 +1757,7 @@ int homa_err_handler_v4(struct sk_buff *skb, u32 info) } if (error != 0) homa_abort_rpcs(homa, &daddr, port, error); + return 0; } @@ -1805,6 +1795,7 @@ int homa_err_handler_v6(struct sk_buff *skb, struct inet6_skb_parm *opt, } if (error != 0) homa_abort_rpcs(homa, &iph->daddr, port, error); + return 0; } @@ -1857,7 +1848,7 @@ __poll_t homa_poll(struct file *file, struct socket *sock, * * Return: 0 for success, nonzero for error. */ -int homa_dointvec(const struct ctl_table *table, int write, +int homa_dointvec(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { struct homa *homa = homa_net(current->nsproxy->net_ns)->homa; @@ -1946,7 +1937,7 @@ int homa_dointvec(const struct ctl_table *table, int write, * * Return: 0 for success, nonzero for error. */ -int homa_sysctl_softirq_cores(const struct ctl_table *table, int write, +int homa_sysctl_softirq_cores(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { struct homa_offload_core *offload_core; diff --git a/homa_qdisc.c b/homa_qdisc.c index 442548e2..496207d6 100755 --- a/homa_qdisc.c +++ b/homa_qdisc.c @@ -133,6 +133,7 @@ static struct ctl_table homa_qdisc_ctl_table[] = { .mode = 0644, .proc_handler = homa_qdisc_dointvec }, + {} }; static struct Qdisc_ops homa_qdisc_ops __read_mostly = { @@ -471,8 +472,7 @@ void homa_qdisc_destroy(struct Qdisc *qdisc) spin_lock_bh(&q->qdev->defer_lock); while (!skb_queue_empty(&q->deferred_tcp)) - kfree_skb_reason(__skb_dequeue(&q->deferred_tcp), - SKB_DROP_REASON_QDISC_DROP); + kfree_skb(__skb_dequeue(&q->deferred_tcp)); list_del_init(&q->defer_links); spin_unlock_bh(&q->qdev->defer_lock); homa_qdisc_qdev_put(q->qdev); @@ -842,7 +842,7 @@ int homa_qdisc_xmit_deferred_tcp(struct homa_qdisc_dev *qdev) "0x%x to 0x%x, data bytes %d, seq/ack %u", skb, ip_hdr(skb)->saddr, ip_hdr(skb)->daddr); homa_qdisc_add_queued(qdev, skb); - homa_qdisc_schedule_skb(skb, qdisc_from_priv(q)); + homa_qdisc_schedule_skb(skb, q->qdisc); return pkt_len; } @@ -977,7 +977,7 @@ int homa_qdisc_xmit_deferred_homa(struct homa_qdisc_dev *qdev) homa_qdisc_add_queued(qdev, skb); homa_qdisc_schedule_skb(skb, qdisc); } else { - kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_DROP); + kfree_skb(skb); } rcu_read_unlock_bh(); return pkt_len; @@ -996,7 +996,7 @@ void homa_qdisc_free_homa(struct homa_qdisc_dev *qdev) skb = homa_qdisc_get_deferred_homa(qdev); if (!skb) break; - kfree_skb_reason(skb, SKB_DROP_REASON_QUEUE_PURGE); + kfree_skb(skb); } } @@ -1297,7 +1297,7 @@ void homa_qdisc_refresh_from_dql(struct homa_qdisc_dev *qdev) * * Return: 0 for success, nonzero for error. */ -int homa_qdisc_dointvec(const struct ctl_table *table, int write, +int homa_qdisc_dointvec(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { struct homa_qdisc_shared *qshared; diff --git a/homa_qdisc.h b/homa_qdisc.h index fed55cdc..511e1290 100644 --- a/homa_qdisc.h +++ b/homa_qdisc.h @@ -353,7 +353,7 @@ void homa_qdisc_defer_homa(struct homa_qdisc_dev *qdev, void homa_qdisc_defer_tcp(struct homa_qdisc *q, struct sk_buff *skb); void homa_qdisc_destroy(struct Qdisc *sch); void homa_qdisc_dev_callback(struct rcu_head *head); -int homa_qdisc_dointvec(const struct ctl_table *table, int write, +int homa_qdisc_dointvec(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int homa_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free); diff --git a/homa_rpc.c b/homa_rpc.c index 61aba4bc..7ff60b84 100644 --- a/homa_rpc.c +++ b/homa_rpc.c @@ -584,10 +584,9 @@ int homa_rpc_reap(struct homa_sock *hsk, bool reap_all) * buffers left. */ if (rpc->msgin.length >= 0 && - !skb_queue_empty_lockless(&rpc->msgin.packets)) { + !skb_queue_empty(&rpc->msgin.packets)) { rx_frees += skb_queue_len(&rpc->msgin.packets); - __skb_queue_purge_reason(&rpc->msgin.packets, - SKB_CONSUMED); + __skb_queue_purge(&rpc->msgin.packets); } /* If we get here, it means all packets have been diff --git a/homa_skb.c b/homa_skb.c index f3545416..b9279430 100644 --- a/homa_skb.c +++ b/homa_skb.c @@ -17,9 +17,23 @@ extern int mock_max_skb_frags; #define HOMA_MAX_SKB_FRAGS MAX_SKB_FRAGS #endif +/* This function was added to later versions of the kernel, but isn't + * available in this version. + */ +static inline void skb_len_add(struct sk_buff *skb, int delta) +{ + skb->len += delta; + skb->data_len += delta; + skb->truesize += delta; +} + static void frag_page_set(skb_frag_t *frag, struct page *page) { +#ifdef CONFIG_NETMEM frag->netmem = page_to_netmem(page); +#else + __skb_frag_set_page(frag, page); +#endif } /** @@ -190,7 +204,7 @@ void *homa_skb_extend_frags(struct homa *homa, struct sk_buff *skb, int *length) frag = &shinfo->frags[shinfo->nr_frags - 1]; if (skb_frag_page(frag) == skb_core->skb_page && skb_core->page_inuse < skb_core->page_size && - (frag->offset + skb_frag_size(frag)) == + (frag->bv_offset + skb_frag_size(frag)) == skb_core->page_inuse) { if ((skb_core->page_size - skb_core->page_inuse) < actual_size) @@ -220,7 +234,7 @@ void *homa_skb_extend_frags(struct homa *homa, struct sk_buff *skb, int *length) shinfo->nr_frags++; frag_page_set(frag, skb_core->skb_page); get_page(skb_core->skb_page); - frag->offset = skb_core->page_inuse; + frag->bv_offset = skb_core->page_inuse; *length = actual_size; skb_frag_size_set(frag, actual_size); result = page_address(skb_frag_page(frag)) + skb_core->page_inuse; @@ -418,7 +432,7 @@ int homa_skb_append_from_skb(struct homa *homa, struct sk_buff *dst_skb, dst_shinfo->nr_frags++; frag_page_set(dst_frag, skb_frag_page(src_frag)); get_page(skb_frag_page(src_frag)); - dst_frag->offset = src_frag->offset + dst_frag->bv_offset = src_frag->bv_offset + (offset - src_frag_offset); skb_frag_size_set(dst_frag, chunk_size); offset += chunk_size; @@ -567,9 +581,9 @@ void homa_skb_get(struct sk_buff *skb, void *dest, int offset, int length) chunk_size = skb_frag_size(frag) - (offset - frag_offset); if (chunk_size > length) chunk_size = length; - memcpy(dst, page_address(skb_frag_page(frag)) + frag->offset - + (offset - frag_offset), - chunk_size); + memcpy(dst, page_address(skb_frag_page(frag)) + + frag->bv_offset + (offset - frag_offset), + chunk_size); offset += chunk_size; length -= chunk_size; dst += chunk_size; @@ -585,7 +599,7 @@ void homa_skb_get(struct sk_buff *skb, void *dest, int offset, int length) void homa_skb_release_pages(struct homa *homa) { int i, max_low_mark, min_pages, release, release_max; - struct homa_page_pool *max_pool; + struct homa_page_pool *max_pool = NULL; u64 now = homa_clock(); if (now < homa->skb_page_free_time) diff --git a/homa_wire.h b/homa_wire.h index a45a2cda..701cf479 100644 --- a/homa_wire.h +++ b/homa_wire.h @@ -148,10 +148,12 @@ struct homa_common_hdr { #endif /* See strip.py */ /** - * @window: Corresponds to the window field in TCP headers. Not used - * by HOMA. + * @gro_count: Corresponds to the window field in TCP headers, which + * isn't used by Homa. Value on the wire is undefined. Used only by + * homa_offload.c (it counts the total number of packets aggregated + * into this packet, including the top-level packet). */ - __be16 window; + __u16 gro_count; /** * @checksum: occupies the same bytes as the checksum in a TCP diff --git a/murmurhash3.h b/murmurhash3.h index 1ed1f0b6..e76a219f 100644 --- a/murmurhash3.h +++ b/murmurhash3.h @@ -18,9 +18,10 @@ static inline u32 murmurhash3(const void *data, u32 len, u32 seed) const u32 c2 = 0x1b873593; const u32 *key = data; u32 h = seed; + size_t i; len = len >> 2; - for (size_t i = 0; i < len; i++) { + for (i = 0; i < len; i++) { u32 k = key[i]; k *= c1; diff --git a/notes.txt b/notes.txt index d0c12ff1..d4d32773 100755 --- a/notes.txt +++ b/notes.txt @@ -1,5 +1,11 @@ Notes for Homa implementation in Linux: --------------------------------------- +* Backporting to 4.18.0: + * Things to port forward: + * Change to tt_unfreeze + * Changes in tthoma.py + * Changes in homa_qdisc.c + * Changes in test/Makefile * Linux doesn't seem to like packets getting passed to ip*xmit with reference counts > 1. Maybe change Homa to collect packet data in diff --git a/test/mock.c b/test/mock.c index ac4d4bc2..84af1efd 100644 --- a/test/mock.c +++ b/test/mock.c @@ -15,6 +15,7 @@ #include "utils.h" #include +#include /* It isn't safe to include some header files, such as stdlib, because * they conflict with kernel header files. The explicit declarations @@ -297,20 +298,17 @@ unsigned int nr_cpu_ids = 8; unsigned long page_offset_base; unsigned long phys_base; unsigned long vmemmap_base; -#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0) -kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES]; -#endif +struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; int __preempt_count; + +/* Value that will be returned by smp_processor_id. */ int cpu_number = 1; + char sock_flow_table[RPS_SOCK_FLOW_TABLE_SIZE(1024)]; -struct net_hotdata net_hotdata = { - .rps_cpu_mask = 0x1f, - .rps_sock_flow_table = (struct rps_sock_flow_table *) sock_flow_table -}; +struct rps_sock_flow_table *rps_sock_flow_table + = (struct rps_sock_flow_table *) sock_flow_table; +__u32 rps_cpu_mask = 0x1f; int debug_locks; -struct static_call_key __SCK__cond_resched; -struct static_call_key __SCK__might_resched; -struct static_call_key __SCK__preempt_schedule; struct paravirt_patch_template pv_ops; struct workqueue_struct *system_wq; struct static_key_true validate_usercopy_range; @@ -325,6 +323,12 @@ extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) {} +int alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *locks_mask, + size_t max_size, unsigned int cpu_mult, gfp_t gfp) +{ + return 0; +} + struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags, int node) { @@ -382,6 +386,11 @@ bool cancel_work_sync(struct work_struct *work) void __check_object_size(const void *ptr, unsigned long n, bool to_user) {} +int _cond_resched(void) +{ + return 0; +} + void consume_skb(struct sk_buff *skb) { kfree_skb(skb); } @@ -398,7 +407,7 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *iter) return 0; } while (bytes_left > 0) { - struct iovec *iov = (struct iovec *) iter_iov(iter); + struct iovec *iov = (struct iovec *)iter->iov; u64 int_base = (u64) iov->iov_base; size_t chunk_bytes = iov->iov_len; @@ -411,7 +420,7 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *iter) iov->iov_base = (void *) (int_base + chunk_bytes); iov->iov_len -= chunk_bytes; if (iov->iov_len == 0) - iter->__iov++; + iter->iov++; } return bytes; } @@ -474,7 +483,7 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, const struct in6_addr *daddr, __u32 len, __u8 proto, __wsum csum) { - return 0; + return ~666; } #ifdef CONFIG_DEBUG_LOCK_ALLOC @@ -484,7 +493,7 @@ int debug_lockdep_rcu_enabled(void) } #endif -int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *) +int do_wait_intr_irq(wait_queue_head_t *head, wait_queue_entry_t *entry) { UNIT_HOOK("do_wait_intr_irq"); if (mock_check_error(&mock_wait_intr_irq_errors)) @@ -496,8 +505,8 @@ void dst_release(struct dst_entry *dst) { if (!dst) return; - atomic_dec(&dst->__rcuref.refcnt); - if (atomic_read(&dst->__rcuref.refcnt) > 0) + atomic_dec(&dst->__refcnt); + if (atomic_read(&dst->__refcnt) > 0) return; if (!routes_in_use || unit_hash_get(routes_in_use, dst) == NULL) { FAIL(" %s on unknown route", __func__); @@ -511,7 +520,7 @@ void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) {} -void get_random_bytes(void *buf, size_t nbytes) +void get_random_bytes(void *buf, int nbytes) { memset(buf, 0, nbytes); } @@ -557,15 +566,12 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 range_ns, const enum hrtimer_mode mode) {} -void __icmp_send(struct sk_buff *skb, int type, int code, __be32 info, - const struct ip_options *opt) +void icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info) { unit_log_printf("; ", "icmp_send type %d, code %d", type, code); } -void icmp6_send(struct sk_buff *skb, u8 type, u8 code, u32 info, - const struct in6_addr *force_saddr, - const struct inet6_skb_parm *parm) +void icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info) { unit_log_printf("; ", "icmp6_send type %d, code %d", type, code); } @@ -575,8 +581,8 @@ int idle_cpu(int cpu) return mock_check_error(&mock_cpu_idle); } -ssize_t import_iovec(int type, const struct iovec __user *uvector, - unsigned int nr_segs, unsigned int fast_segs, +int import_iovec(int type, const struct iovec __user *uvector, + unsigned nr_segs, unsigned fast_segs, struct iovec **iov, struct iov_iter *iter) { ssize_t size; @@ -594,14 +600,6 @@ ssize_t import_iovec(int type, const struct iovec __user *uvector, return size; } -int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i) -{ - if (mock_check_error(&mock_import_ubuf_errors)) - return -EACCES; - iov_iter_ubuf(i, rw, buf, len); - return 0; -} - int inet6_add_offload(const struct net_offload *prot, unsigned char protocol) { return 0; @@ -712,16 +710,15 @@ void init_wait_entry(struct wait_queue_entry *wq_entry, int flags) {} void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, - struct lock_class_key *) + struct lock_class_key *key) {} -void iov_iter_init(struct iov_iter *i, unsigned int direction, - const struct iovec *iov, unsigned long nr_segs, - size_t count) +void iov_iter_init(struct iov_iter *i, int direction, const struct iovec *iov, + unsigned long nr_segs, size_t count) { direction &= READ | WRITE; - i->iter_type = ITER_IOVEC | direction; - i->__iov = iov; + i->type = ITER_IOVEC | direction; + i->iov = iov; i->nr_segs = nr_segs; i->iov_offset = 0; i->count = count; @@ -744,8 +741,8 @@ struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie) return dst; } -struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, - struct flowi6 *fl6, const struct in6_addr *final_dst) +struct dst_entry *ip6_dst_lookup_flow(const struct sock *sk, struct flowi6 *fl6, + const struct in6_addr *final_dst) { struct rtable *route; @@ -756,7 +753,7 @@ struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, FAIL(" malloc failed"); return ERR_PTR(-ENOMEM); } - atomic_set(&route->dst.__rcuref.refcnt, 1); + atomic_set(&route->dst.__refcnt, 1); route->dst.ops = &mock_dst_ops; route->dst.dev = &mock_devices[0]; route->dst.obsolete = 0; @@ -772,7 +769,7 @@ unsigned int ip6_mtu(const struct dst_entry *dst) } int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6, - u32 mark, struct ipv6_txoptions *opt, int tclass, u32 priority) + u32 mark, struct ipv6_txoptions *opt, int tclass) { char buffer[200]; const char *prefix = " "; @@ -865,7 +862,7 @@ struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4, FAIL(" malloc failed"); return ERR_PTR(-ENOMEM); } - atomic_set(&route->dst.__rcuref.refcnt, 1); + atomic_set(&route->dst.__refcnt, 1); route->dst.ops = &mock_dst_ops; route->dst.dev = &mock_devices[0]; route->dst.obsolete = 0; @@ -884,20 +881,22 @@ int ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, void device_set_wakeup_capable(struct device *dev, bool capable) {} -void device_wakeup_disable(struct device *dev) -{} +int device_wakeup_disable(struct device *dev) +{ + return 0; +} int device_wakeup_enable(struct device *dev) { return 0; } -int filp_close(struct file *, fl_owner_t id) +int filp_close(struct file *f, fl_owner_t id) { return 0; } -struct file *filp_open(const char *, int, umode_t) +struct file *filp_open(const char *f, int dummy, umode_t mode) { return NULL; } @@ -910,6 +909,11 @@ void __fortify_panic(const u8 reason, const size_t avail, const size_t size) while (1) ; } +void free_bucket_spinlocks(spinlock_t *locks) +{ + kvfree(locks); +} + ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos) { return 0; @@ -934,11 +938,7 @@ void kfree(const void *block) free((void *) block); } -#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 12, 0) -void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) -#else -void __kfree_skb(struct sk_buff *skb) -#endif +void kfree_skb(struct sk_buff *skb) { int i; struct skb_shared_info *shinfo = skb_shinfo(skb); @@ -964,19 +964,9 @@ void __kfree_skb(struct sk_buff *skb) free(skb); } -void kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason) -{ - while (segs) { - struct sk_buff *next = segs->next; - - __kfree_skb(segs); - segs = next; - } -} - void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) { - return mock_kmalloc(size, gfpflags); + return kmalloc(size, gfpflags); } #ifdef CONFIG_DEBUG_ATOMIC_SLEEP @@ -986,7 +976,7 @@ void __might_sleep(const char *file, int line) } #endif -void *mock_kmalloc(size_t size, gfp_t flags) +void *__kmalloc(size_t size, gfp_t flags) { void *block; @@ -1012,17 +1002,12 @@ void *mock_kmalloc(size_t size, gfp_t flags) void *__kmalloc_noprof(size_t size, gfp_t flags) { - return mock_kmalloc(size, flags); + return kmalloc(size, flags); } -void kvfree(const void *addr) +void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) { - kfree(addr); -} - -void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) -{ - return mock_kmalloc(size, flags); + return __kmalloc(size, flags); } struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), @@ -1046,6 +1031,16 @@ int kthread_stop(struct task_struct *k) return 0; } +void kvfree(const void *addr) +{ + kfree(addr); +} + +void *kvmalloc_node(size_t size, gfp_t flags, int node) +{ + return __kmalloc(size, flags); +} + #ifdef CONFIG_DEBUG_LIST bool __list_add_valid(struct list_head *new, struct list_head *prev, struct list_head *next) @@ -1099,9 +1094,8 @@ void lock_sock_nested(struct sock *sk, int subclass) mock_active_locks++; sk->sk_lock.owned = 1; } - -ssize_t __modver_version_show(const struct module_attribute *a, - struct module_kobject *b, char *c) +ssize_t __modver_version_show(struct module_attribute *mattr, + struct module_kobject *mk, char *buf) { return 0; } @@ -1143,22 +1137,6 @@ int netif_receive_skb(struct sk_buff *skb) void __netif_schedule(struct Qdisc *q) {} -void preempt_count_add(int val) -{ - int i; - - for (i = 0; i < val; i++) - preempt_disable(); -} - -void preempt_count_sub(int val) -{ - int i; - - for (i = 0; i < val; i++) - preempt_enable(); -} - long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state) { @@ -1168,7 +1146,7 @@ long prepare_to_wait_event(struct wait_queue_head *wq_head, return 0; } -int _printk(const char *format, ...) +int printk(const char *format, ...) { int len = strlen(mock_printk_output); int available; @@ -1201,7 +1179,7 @@ int _printk(const char *format, ...) struct proc_dir_entry *proc_create(const char *name, umode_t mode, struct proc_dir_entry *parent, - const struct proc_ops *proc_ops) + const struct file_operations *proc_fops) { struct proc_dir_entry *entry = malloc(40); @@ -1299,23 +1277,12 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) return 1; } -void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) -{ - UNIT_HOOK("unlock"); - mock_record_unlocked(lock); -} - void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) { UNIT_HOOK("unlock"); mock_record_unlocked(lock); } -void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) -{ - mock_record_unlocked(lock); -} - void _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) { @@ -1356,20 +1323,7 @@ int rcu_read_lock_bh_held(void) } #endif -void __rcu_read_lock(void) -{} - -void __rcu_read_unlock(void) -{} - -bool rcuref_get_slowpath(rcuref_t *ref) -{ - return true; -} - -void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t) {} - -int register_pernet_subsys(struct pernet_operations *) +int register_pernet_subsys(struct pernet_operations *ops) { return 0; } @@ -1405,7 +1359,7 @@ void rtnl_kfree_skbs(struct sk_buff *head, struct sk_buff *tail) while (true) { struct sk_buff *next = head->next; - __kfree_skb(head); + kfree_skb(head); if (head == tail) break; head = next; @@ -1438,8 +1392,7 @@ int __SCT__might_resched(void) void __SCT__preempt_schedule(void) {} -void security_sk_classify_flow(const struct sock *sk, - struct flowi_common *flic) +void security_sk_classify_flow(struct sock *sk, struct flowi *fl) {} void __show_free_areas(unsigned int filter, nodemask_t *nodemask, @@ -1454,15 +1407,6 @@ int sk_set_peek_off(struct sock *sk, int val) return 0; } -void sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb, - enum skb_drop_reason reason) -{ -#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 12, 0) - kfree_skb(skb); -#else - __kfree_skb(skb); -#endif -} __wsum skb_checksum(const struct sk_buff *skb, int offset, int len, __wsum csum) { return 0; @@ -1481,7 +1425,7 @@ int skb_copy_datagram_iter(const struct sk_buff *from, int offset, return 0; } while (bytes_left > 0) { - struct iovec *iov = (struct iovec *) iter_iov(iter); + struct iovec *iov = (struct iovec *)iter->iov; u64 int_base = (u64) iov->iov_base; size_t chunk_bytes = iov->iov_len; @@ -1497,7 +1441,7 @@ int skb_copy_datagram_iter(const struct sk_buff *from, int offset, iov->iov_base = (void *) (int_base + chunk_bytes); iov->iov_len -= chunk_bytes; if (iov->iov_len == 0) - iter->__iov++; + iter->iov++; } return 0; } @@ -1536,13 +1480,6 @@ void *skb_put(struct sk_buff *skb, unsigned int len) return result; } -void skb_queue_purge_reason(struct sk_buff_head *list, - enum skb_drop_reason reason) -{ - while (skb_queue_len(list) > 0) - kfree_skb(__skb_dequeue(list)); -} - struct sk_buff *skb_segment(struct sk_buff *head_skb, netdev_features_t features) { @@ -1572,13 +1509,13 @@ int sock_common_getsockopt(struct socket *sock, int level, int optname, } int sock_common_setsockopt(struct socket *sock, int level, int optname, - sockptr_t optval, unsigned int optlen) + char __user *optval, unsigned int optlen) { return 0; } int sock_no_accept(struct socket *sock, struct socket *newsock, - struct proto_accept_arg *arg) + int dummy1, bool dummy2) { return 0; } @@ -1610,7 +1547,7 @@ int sock_no_socketpair(struct socket *sock1, struct socket *sock2) return 0; } -void synchronize_rcu(void) +void synchronize_sched(void) {} void __tasklet_hi_schedule(struct tasklet_struct *t) @@ -1631,13 +1568,14 @@ void unregister_net_sysctl_table(struct ctl_table_header *header) UNIT_LOG("; ", "unregister_net_sysctl_table"); } -void unregister_pernet_subsys(struct pernet_operations *) +void unregister_pernet_subsys(struct pernet_operations *ops) {} -void unregister_qdisc(struct Qdisc_ops *qops) +int unregister_qdisc(struct Qdisc_ops *qops) { registered_qdiscs--; qdisc_ops = NULL; + return 0; } void vfree(const void *block) @@ -1663,16 +1601,14 @@ long wait_woken(struct wait_queue_entry *wq_entry, unsigned int mode, return 0; } -int __wake_up(struct wait_queue_head *wq_head, unsigned int mode, - int nr_exclusive, void *key) +void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key) { if (!mock_log_wakeups) - return 0; - if (nr_exclusive == 1) + return; + if (nr == 1) unit_log_printf("; ", "wake_up"); else unit_log_printf("; ", "wake_up_all"); - return 0; } void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, @@ -2396,7 +2332,7 @@ int mock_sock_init(struct homa_sock *hsk, struct homa_net *hnet, int port) mock_socket.sk = sk; sk->sk_net.net = mock_net_for_hnet(hnet); refcount_set(&sk->sk_wmem_alloc, 1); - init_waitqueue_head(&mock_socket.wq.wait); + init_waitqueue_head(&mock_socket.wq->wait); rcu_assign_pointer(sk->sk_wq, &mock_socket.wq); sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; sk->sk_protocol = IPPROTO_HOMA; diff --git a/test/mock.h b/test/mock.h index d9178fe2..ce16c7c4 100644 --- a/test/mock.h +++ b/test/mock.h @@ -10,10 +10,13 @@ #undef alloc_pages #define alloc_pages mock_alloc_pages +#undef atomic64_cmpxchg_relaxed #define atomic64_cmpxchg_relaxed mock_cmpxchg #undef alloc_percpu_gfp -#define alloc_percpu_gfp(type, flags) mock_kmalloc(10 * sizeof(type), flags) +#define alloc_percpu_gfp(type, flags) __kmalloc(10 * sizeof(type), flags) + +#define complete_and_exit(...) #define compound_order mock_compound_order @@ -55,13 +58,8 @@ #define homa_rpc_put mock_rpc_put -#undef kmalloc -#define kmalloc mock_kmalloc - #undef kmalloc_array -#define kmalloc_array(count, size, type) mock_kmalloc((count) * (size), type) - -#define kthread_complete_and_exit(...) +#define kmalloc_array(count, size, type) __kmalloc((count) * (size), type) #undef local_irq_save #define local_irq_save(flags) (flags) = 0 @@ -102,6 +100,8 @@ #undef register_net_sysctl #define register_net_sysctl mock_register_net_sysctl +#define rt6_get_cookie(...) 999 + #define signal_pending(...) mock_signal_pending #undef smp_processor_id @@ -226,7 +226,6 @@ unsigned int void mock_get_page(struct page *page); struct homa_net *mock_hnet(int index, struct homa *homa); -void *mock_kmalloc(size_t size, gfp_t flags); struct net *mock_net_for_hnet(struct homa_net *hnet); void *mock_net_generic(const struct net *net, unsigned int id); int mock_page_refs(struct page *page); diff --git a/test/rbtree.c b/test/rbtree.c index 9e730718..d3ff682f 100644 --- a/test/rbtree.c +++ b/test/rbtree.c @@ -1,10 +1,22 @@ -// SPDX-License-Identifier: GPL-2.0-or-later /* Red Black Trees (C) 1999 Andrea Arcangeli (C) 2002 David Woodhouse (C) 2012 Michel Lespinasse + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA linux/lib/rbtree.c */ @@ -13,7 +25,7 @@ #include /* - * red-black trees properties: https://en.wikipedia.org/wiki/Rbtree + * red-black trees properties: http://en.wikipedia.org/wiki/Rbtree * * 1) A node is either red or black * 2) The root is black @@ -58,7 +70,7 @@ static inline void rb_set_black(struct rb_node *rb) { - rb->__rb_parent_color += RB_BLACK; + rb->__rb_parent_color |= RB_BLACK; } static inline struct rb_node *rb_red_parent(struct rb_node *red) @@ -83,10 +95,14 @@ __rb_rotate_set_parents(struct rb_node *old, struct rb_node *new, static __always_inline void __rb_insert(struct rb_node *node, struct rb_root *root, + bool newleft, struct rb_node **leftmost, void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) { struct rb_node *parent = rb_red_parent(node), *gparent, *tmp; + if (newleft) + *leftmost = node; + while (true) { /* * Loop invariant: node is red. @@ -412,6 +428,7 @@ void __rb_erase_color(struct rb_node *parent, struct rb_root *root, { ____rb_erase_color(parent, root, augment_rotate); } +EXPORT_SYMBOL(__rb_erase_color); /* * Non-augmented rbtree manipulation functions. @@ -432,16 +449,37 @@ static const struct rb_augment_callbacks dummy_callbacks = { void rb_insert_color(struct rb_node *node, struct rb_root *root) { - __rb_insert(node, root, dummy_rotate); + __rb_insert(node, root, false, NULL, dummy_rotate); } +EXPORT_SYMBOL(rb_insert_color); void rb_erase(struct rb_node *node, struct rb_root *root) { struct rb_node *rebalance; - rebalance = __rb_erase_augmented(node, root, &dummy_callbacks); + rebalance = __rb_erase_augmented(node, root, + NULL, &dummy_callbacks); if (rebalance) ____rb_erase_color(rebalance, root, dummy_rotate); } +EXPORT_SYMBOL(rb_erase); + +void rb_insert_color_cached(struct rb_node *node, + struct rb_root_cached *root, bool leftmost) +{ + __rb_insert(node, &root->rb_root, leftmost, + &root->rb_leftmost, dummy_rotate); +} +EXPORT_SYMBOL(rb_insert_color_cached); + +void rb_erase_cached(struct rb_node *node, struct rb_root_cached *root) +{ + struct rb_node *rebalance; + rebalance = __rb_erase_augmented(node, &root->rb_root, + &root->rb_leftmost, &dummy_callbacks); + if (rebalance) + ____rb_erase_color(rebalance, &root->rb_root, dummy_rotate); +} +EXPORT_SYMBOL(rb_erase_cached); /* * Augmented rbtree manipulation functions. @@ -451,10 +489,12 @@ void rb_erase(struct rb_node *node, struct rb_root *root) */ void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, + bool newleft, struct rb_node **leftmost, void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) { - __rb_insert(node, root, augment_rotate); + __rb_insert(node, root, newleft, leftmost, augment_rotate); } +EXPORT_SYMBOL(__rb_insert_augmented); /* * This function returns the first node (in sort order) of the tree. @@ -470,6 +510,7 @@ struct rb_node *rb_first(const struct rb_root *root) n = n->rb_left; return n; } +EXPORT_SYMBOL(rb_first); struct rb_node *rb_last(const struct rb_root *root) { @@ -482,6 +523,7 @@ struct rb_node *rb_last(const struct rb_root *root) n = n->rb_right; return n; } +EXPORT_SYMBOL(rb_last); struct rb_node *rb_next(const struct rb_node *node) { @@ -497,7 +539,7 @@ struct rb_node *rb_next(const struct rb_node *node) if (node->rb_right) { node = node->rb_right; while (node->rb_left) - node = node->rb_left; + node=node->rb_left; return (struct rb_node *)node; } @@ -513,6 +555,7 @@ struct rb_node *rb_next(const struct rb_node *node) return parent; } +EXPORT_SYMBOL(rb_next); struct rb_node *rb_prev(const struct rb_node *node) { @@ -528,7 +571,7 @@ struct rb_node *rb_prev(const struct rb_node *node) if (node->rb_left) { node = node->rb_left; while (node->rb_right) - node = node->rb_right; + node=node->rb_right; return (struct rb_node *)node; } @@ -541,6 +584,7 @@ struct rb_node *rb_prev(const struct rb_node *node) return parent; } +EXPORT_SYMBOL(rb_prev); void rb_replace_node(struct rb_node *victim, struct rb_node *new, struct rb_root *root) @@ -557,6 +601,39 @@ void rb_replace_node(struct rb_node *victim, struct rb_node *new, rb_set_parent(victim->rb_right, new); __rb_change_child(victim, new, parent, root); } +EXPORT_SYMBOL(rb_replace_node); + +void rb_replace_node_cached(struct rb_node *victim, struct rb_node *new, + struct rb_root_cached *root) +{ + rb_replace_node(victim, new, &root->rb_root); + + if (root->rb_leftmost == victim) + root->rb_leftmost = new; +} +EXPORT_SYMBOL(rb_replace_node_cached); + +void rb_replace_node_rcu(struct rb_node *victim, struct rb_node *new, + struct rb_root *root) +{ + struct rb_node *parent = rb_parent(victim); + + /* Copy the pointers/colour from the victim to the replacement */ + *new = *victim; + + /* Set the surrounding nodes to point to the replacement */ + if (victim->rb_left) + rb_set_parent(victim->rb_left, new); + if (victim->rb_right) + rb_set_parent(victim->rb_right, new); + + /* Set the parent's pointer to the new node last after an RCU barrier + * so that the pointers onwards are seen to be set correctly when doing + * an RCU walk over the tree. + */ + __rb_change_child_rcu(victim, new, parent, root); +} +EXPORT_SYMBOL(rb_replace_node_rcu); static struct rb_node *rb_left_deepest_node(const struct rb_node *node) { @@ -587,6 +664,7 @@ struct rb_node *rb_next_postorder(const struct rb_node *node) * should be next */ return (struct rb_node *)parent; } +EXPORT_SYMBOL(rb_next_postorder); struct rb_node *rb_first_postorder(const struct rb_root *root) { @@ -595,3 +673,4 @@ struct rb_node *rb_first_postorder(const struct rb_root *root) return rb_left_deepest_node(root->rb_node); } +EXPORT_SYMBOL(rb_first_postorder); diff --git a/test/rhashtable.c b/test/rhashtable.c index 3e555d01..a9cbe019 100644 --- a/test/rhashtable.c +++ b/test/rhashtable.c @@ -1,4 +1,3 @@ -// SPDX-License-Identifier: GPL-2.0-only /* * Resizable, Scalable, Concurrent Hash Table * @@ -9,6 +8,10 @@ * Code partially derived from nft_hash * Rewritten with rehash code from br_multicast plus single list * pointer as suggested by Josh Triplett + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. */ #include @@ -26,12 +29,16 @@ #include #include +void mock_spin_unlock(spinlock_t *lock); +#define spin_unlock mock_spin_unlock + #define HASH_DEFAULT_SIZE 64UL #define HASH_MIN_SIZE 4U +#define BUCKET_LOCKS_PER_CPU 32UL union nested_table { union nested_table __rcu *table; - struct rhash_lock_head __rcu *bucket; + struct rhash_head __rcu *bucket; }; static u32 head_hashfn(struct rhashtable *ht, @@ -52,33 +59,22 @@ EXPORT_SYMBOL_GPL(lockdep_rht_mutex_is_held); int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash) { - if (!debug_locks) - return 1; - if (unlikely(tbl->nest)) - return 1; - return bit_spin_is_locked(0, (unsigned long *)&tbl->buckets[hash]); + spinlock_t *lock = rht_bucket_lock(tbl, hash); + + return (debug_locks) ? lockdep_is_held(lock) : 1; } EXPORT_SYMBOL_GPL(lockdep_rht_bucket_is_held); #else #define ASSERT_RHT_MUTEX(HT) #endif -static inline union nested_table *nested_table_top( - const struct bucket_table *tbl) -{ - /* The top-level bucket entry does not need RCU protection - * because it's set at the same time as tbl->nest. - */ - return (void *)rcu_dereference_protected(tbl->buckets[0], 1); -} - static void nested_table_free(union nested_table *ntbl, unsigned int size) { const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); const unsigned int len = 1 << shift; unsigned int i; - ntbl = rcu_dereference_protected(ntbl->table, 1); + ntbl = rcu_dereference_raw(ntbl->table); if (!ntbl) return; @@ -98,7 +94,7 @@ static void nested_bucket_table_free(const struct bucket_table *tbl) union nested_table *ntbl; unsigned int i; - ntbl = nested_table_top(tbl); + ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]); for (i = 0; i < len; i++) nested_table_free(ntbl + i, size); @@ -111,6 +107,7 @@ static void bucket_table_free(const struct bucket_table *tbl) if (tbl->nest) nested_bucket_table_free(tbl); + free_bucket_spinlocks(tbl->locks); kvfree(tbl); } @@ -121,7 +118,8 @@ static void bucket_table_free_rcu(struct rcu_head *head) static union nested_table *nested_table_alloc(struct rhashtable *ht, union nested_table __rcu **prev, - bool leaf) + unsigned int shifted, + unsigned int nhash) { union nested_table *ntbl; int i; @@ -130,19 +128,17 @@ static union nested_table *nested_table_alloc(struct rhashtable *ht, if (ntbl) return ntbl; - ntbl = alloc_hooks_tag(ht->alloc_tag, - kmalloc_noprof(PAGE_SIZE, GFP_ATOMIC|__GFP_ZERO)); + ntbl = kzalloc(PAGE_SIZE, GFP_ATOMIC); - if (ntbl && leaf) { - for (i = 0; i < PAGE_SIZE / sizeof(ntbl[0]); i++) - INIT_RHT_NULLS_HEAD(ntbl[i].bucket); + if (ntbl && shifted) { + for (i = 0; i < PAGE_SIZE / sizeof(ntbl[0].bucket); i++) + INIT_RHT_NULLS_HEAD(ntbl[i].bucket, ht, + (i << shifted) | nhash); } - if (cmpxchg((union nested_table **)prev, NULL, ntbl) == NULL) - return ntbl; - /* Raced with another thread. */ - kfree(ntbl); - return rcu_dereference(*prev); + rcu_assign_pointer(*prev, ntbl); + + return ntbl; } static struct bucket_table *nested_bucket_table_alloc(struct rhashtable *ht, @@ -158,13 +154,12 @@ static struct bucket_table *nested_bucket_table_alloc(struct rhashtable *ht, size = sizeof(*tbl) + sizeof(tbl->buckets[0]); - tbl = alloc_hooks_tag(ht->alloc_tag, - kmalloc_noprof(size, gfp|__GFP_ZERO)); + tbl = kzalloc(size, gfp); if (!tbl) return NULL; if (!nested_table_alloc(ht, (union nested_table __rcu **)tbl->buckets, - false)) { + 0, 0)) { kfree(tbl); return NULL; } @@ -179,35 +174,42 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, gfp_t gfp) { struct bucket_table *tbl = NULL; - size_t size; + size_t size, max_locks; int i; - static struct lock_class_key __key; - tbl = alloc_hooks_tag(ht->alloc_tag, - kvmalloc_node_noprof(struct_size(tbl, buckets, nbuckets), - gfp|__GFP_ZERO, NUMA_NO_NODE)); + size = sizeof(*tbl) + nbuckets * sizeof(tbl->buckets[0]); + if (gfp != GFP_KERNEL) + tbl = kzalloc(size, gfp | __GFP_NOWARN | __GFP_NORETRY); + else + tbl = kvzalloc(size, gfp); size = nbuckets; - if (tbl == NULL && !gfpflags_allow_blocking(gfp)) { + if (tbl == NULL && gfp != GFP_KERNEL) { tbl = nested_bucket_table_alloc(ht, nbuckets, gfp); nbuckets = 0; } - if (tbl == NULL) return NULL; - lockdep_init_map(&tbl->dep_map, "rhashtable_bucket", &__key, 0); - tbl->size = size; - rcu_head_init(&tbl->rcu); + max_locks = size >> 1; + if (tbl->nest) + max_locks = min_t(size_t, max_locks, 1U << tbl->nest); + + if (alloc_bucket_spinlocks(&tbl->locks, &tbl->locks_mask, max_locks, + ht->p.locks_mul, gfp) < 0) { + bucket_table_free(tbl); + return NULL; + } + INIT_LIST_HEAD(&tbl->walkers); tbl->hash_rnd = get_random_u32(); for (i = 0; i < nbuckets; i++) - INIT_RHT_NULLS_HEAD(tbl->buckets[i]); + INIT_RHT_NULLS_HEAD(tbl->buckets[i], ht, i); return tbl; } @@ -225,25 +227,23 @@ static struct bucket_table *rhashtable_last_table(struct rhashtable *ht, return new_tbl; } -static int rhashtable_rehash_one(struct rhashtable *ht, - struct rhash_lock_head __rcu **bkt, - unsigned int old_hash) +static int rhashtable_rehash_one(struct rhashtable *ht, unsigned int old_hash) { struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); - struct bucket_table *new_tbl = rhashtable_last_table(ht, old_tbl); + struct bucket_table *new_tbl = rhashtable_last_table(ht, + rht_dereference_rcu(old_tbl->future_tbl, ht)); + struct rhash_head __rcu **pprev = rht_bucket_var(old_tbl, old_hash); int err = -EAGAIN; struct rhash_head *head, *next, *entry; - struct rhash_head __rcu **pprev = NULL; + spinlock_t *new_bucket_lock; unsigned int new_hash; - unsigned long flags; if (new_tbl->nest) goto out; err = -ENOENT; - rht_for_each_from(entry, rht_ptr(bkt, old_tbl, old_hash), - old_tbl, old_hash) { + rht_for_each(entry, old_tbl, old_hash) { err = 0; next = rht_dereference_bucket(entry->next, old_tbl, old_hash); @@ -258,20 +258,18 @@ static int rhashtable_rehash_one(struct rhashtable *ht, new_hash = head_hashfn(ht, new_tbl, entry); - flags = rht_lock_nested(new_tbl, &new_tbl->buckets[new_hash], - SINGLE_DEPTH_NESTING); + new_bucket_lock = rht_bucket_lock(new_tbl, new_hash); - head = rht_ptr(new_tbl->buckets + new_hash, new_tbl, new_hash); + spin_lock_nested(new_bucket_lock, SINGLE_DEPTH_NESTING); + head = rht_dereference_bucket(new_tbl->buckets[new_hash], + new_tbl, new_hash); RCU_INIT_POINTER(entry->next, head); - rht_assign_unlock(new_tbl, &new_tbl->buckets[new_hash], entry, flags); + rcu_assign_pointer(new_tbl->buckets[new_hash], entry); + spin_unlock(new_bucket_lock); - if (pprev) - rcu_assign_pointer(*pprev, next); - else - /* Need to preserved the bit lock. */ - rht_assign_locked(bkt, next); + rcu_assign_pointer(*pprev, next); out: return err; @@ -281,20 +279,20 @@ static int rhashtable_rehash_chain(struct rhashtable *ht, unsigned int old_hash) { struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); - struct rhash_lock_head __rcu **bkt = rht_bucket_var(old_tbl, old_hash); - unsigned long flags; + spinlock_t *old_bucket_lock; int err; - if (!bkt) - return 0; - flags = rht_lock(old_tbl, bkt); + old_bucket_lock = rht_bucket_lock(old_tbl, old_hash); - while (!(err = rhashtable_rehash_one(ht, bkt, old_hash))) + spin_lock_bh(old_bucket_lock); + while (!(err = rhashtable_rehash_one(ht, old_hash))) ; - if (err == -ENOENT) + if (err == -ENOENT) { + old_tbl->rehash++; err = 0; - rht_unlock(old_tbl, bkt, flags); + } + spin_unlock_bh(old_bucket_lock); return err; } @@ -303,15 +301,21 @@ static int rhashtable_rehash_attach(struct rhashtable *ht, struct bucket_table *old_tbl, struct bucket_table *new_tbl) { + /* Protect future_tbl using the first bucket lock. */ + spin_lock_bh(old_tbl->locks); + + /* Did somebody beat us to it? */ + if (rcu_access_pointer(old_tbl->future_tbl)) { + spin_unlock_bh(old_tbl->locks); + return -EEXIST; + } + /* Make insertions go into the new, empty table right away. Deletions * and lookups will be attempted in both tables until we synchronize. - * As cmpxchg() provides strong barriers, we do not need - * rcu_assign_pointer(). */ + rcu_assign_pointer(old_tbl->future_tbl, new_tbl); - if (cmpxchg((struct bucket_table **)&old_tbl->future_tbl, NULL, - new_tbl) != NULL) - return -EEXIST; + spin_unlock_bh(old_tbl->locks); return 0; } @@ -341,16 +345,13 @@ static int rhashtable_rehash_table(struct rhashtable *ht) spin_lock(&ht->lock); list_for_each_entry(walker, &old_tbl->walkers, list) walker->tbl = NULL; + spin_unlock(&ht->lock); /* Wait for readers. All new readers will see the new * table, and thus no references to the old table will * remain. - * We do this inside the locked region so that - * rhashtable_walk_stop() can use rcu_head_after_call_rcu() - * to check if it should not re-link the table. */ call_rcu(&old_tbl->rcu, bucket_table_free_rcu); - spin_unlock(&ht->lock); return rht_dereference(new_tbl->future_tbl, ht) ? -EAGAIN : 0; } @@ -430,12 +431,8 @@ static void rht_deferred_worker(struct work_struct *work) else if (tbl->nest) err = rhashtable_rehash_alloc(ht, tbl, tbl->size); - if (!err || err == -EEXIST) { - int nerr; - - nerr = rhashtable_rehash_table(ht); - err = err ?: nerr; - } + if (!err) + err = rhashtable_rehash_table(ht); mutex_unlock(&ht->mutex); @@ -465,7 +462,7 @@ static int rhashtable_insert_rehash(struct rhashtable *ht, err = -ENOMEM; - new_tbl = bucket_table_alloc(ht, size, GFP_ATOMIC | __GFP_NOWARN); + new_tbl = bucket_table_alloc(ht, size, GFP_ATOMIC); if (new_tbl == NULL) goto fail; @@ -481,7 +478,7 @@ static int rhashtable_insert_rehash(struct rhashtable *ht, fail: /* Do not fail the insert if someone else did a rehash. */ - if (likely(rcu_access_pointer(tbl->future_tbl))) + if (likely(rcu_dereference_raw(tbl->future_tbl))) return 0; /* Schedule async rehash to retry allocation in process context. */ @@ -492,7 +489,6 @@ static int rhashtable_insert_rehash(struct rhashtable *ht, } static void *rhashtable_lookup_one(struct rhashtable *ht, - struct rhash_lock_head __rcu **bkt, struct bucket_table *tbl, unsigned int hash, const void *key, struct rhash_head *obj) { @@ -500,12 +496,13 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, .ht = ht, .key = key, }; - struct rhash_head __rcu **pprev = NULL; + struct rhash_head __rcu **pprev; struct rhash_head *head; int elasticity; elasticity = RHT_ELASTICITY; - rht_for_each_from(head, rht_ptr(bkt, tbl, hash), tbl, hash) { + pprev = rht_bucket_var(tbl, hash); + rht_for_each_continue(head, *pprev, tbl, hash) { struct rhlist_head *list; struct rhlist_head *plist; @@ -527,11 +524,7 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, RCU_INIT_POINTER(list->next, plist); head = rht_dereference_bucket(head->next, tbl, hash); RCU_INIT_POINTER(list->rhead.next, head); - if (pprev) - rcu_assign_pointer(*pprev, obj); - else - /* Need to preserve the bit lock */ - rht_assign_locked(bkt, obj); + rcu_assign_pointer(*pprev, obj); return NULL; } @@ -542,11 +535,13 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, return ERR_PTR(-ENOENT); } -static struct bucket_table *rhashtable_insert_one( - struct rhashtable *ht, struct rhash_lock_head __rcu **bkt, - struct bucket_table *tbl, unsigned int hash, struct rhash_head *obj, - void *data) +static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht, + struct bucket_table *tbl, + unsigned int hash, + struct rhash_head *obj, + void *data) { + struct rhash_head __rcu **pprev; struct bucket_table *new_tbl; struct rhash_head *head; @@ -556,7 +551,7 @@ static struct bucket_table *rhashtable_insert_one( if (PTR_ERR(data) != -EAGAIN && PTR_ERR(data) != -ENOENT) return ERR_CAST(data); - new_tbl = rht_dereference_rcu(tbl->future_tbl, ht); + new_tbl = rcu_dereference(tbl->future_tbl); if (new_tbl) return new_tbl; @@ -569,7 +564,11 @@ static struct bucket_table *rhashtable_insert_one( if (unlikely(rht_grow_above_100(ht, tbl))) return ERR_PTR(-EAGAIN); - head = rht_ptr(bkt, tbl, hash); + pprev = rht_bucket_insert(ht, tbl, hash); + if (!pprev) + return ERR_PTR(-ENOMEM); + + head = rht_dereference_bucket(*pprev, tbl, hash); RCU_INIT_POINTER(obj->next, head); if (ht->rhlist) { @@ -579,10 +578,11 @@ static struct bucket_table *rhashtable_insert_one( RCU_INIT_POINTER(list->next, NULL); } - /* bkt is always the head of the list, so it holds - * the lock, which we need to preserve - */ - rht_assign_locked(bkt, obj); + rcu_assign_pointer(*pprev, obj); + + atomic_inc(&ht->nelems); + if (rht_grow_above_75(ht, tbl)) + schedule_work(&ht->run_work); return NULL; } @@ -592,44 +592,47 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, { struct bucket_table *new_tbl; struct bucket_table *tbl; - struct rhash_lock_head __rcu **bkt; - unsigned long flags; unsigned int hash; + spinlock_t *lock; void *data; - new_tbl = rcu_dereference(ht->tbl); + tbl = rcu_dereference(ht->tbl); - do { + /* All insertions must grab the oldest table containing + * the hashed bucket that is yet to be rehashed. + */ + for (;;) { + hash = rht_head_hashfn(ht, tbl, obj, ht->p); + lock = rht_bucket_lock(tbl, hash); + spin_lock_bh(lock); + + if (tbl->rehash <= hash) + break; + + spin_unlock_bh(lock); + tbl = rcu_dereference(tbl->future_tbl); + } + + data = rhashtable_lookup_one(ht, tbl, hash, key, obj); + new_tbl = rhashtable_insert_one(ht, tbl, hash, obj, data); + if (PTR_ERR(new_tbl) != -EEXIST) + data = ERR_CAST(new_tbl); + + while (!IS_ERR_OR_NULL(new_tbl)) { tbl = new_tbl; hash = rht_head_hashfn(ht, tbl, obj, ht->p); - if (rcu_access_pointer(tbl->future_tbl)) - /* Failure is OK */ - bkt = rht_bucket_var(tbl, hash); - else - bkt = rht_bucket_insert(ht, tbl, hash); - if (bkt == NULL) { - new_tbl = rht_dereference_rcu(tbl->future_tbl, ht); - data = ERR_PTR(-EAGAIN); - } else { - bool inserted; - - flags = rht_lock(tbl, bkt); - data = rhashtable_lookup_one(ht, bkt, tbl, - hash, key, obj); - new_tbl = rhashtable_insert_one(ht, bkt, tbl, - hash, obj, data); - inserted = data && !new_tbl; - if (inserted) - atomic_inc(&ht->nelems); - if (PTR_ERR(new_tbl) != -EEXIST) - data = ERR_CAST(new_tbl); - - rht_unlock(tbl, bkt, flags); - - if (inserted && rht_grow_above_75(ht, tbl)) - schedule_work(&ht->run_work); - } - } while (!IS_ERR_OR_NULL(new_tbl)); + spin_lock_nested(rht_bucket_lock(tbl, hash), + SINGLE_DEPTH_NESTING); + + data = rhashtable_lookup_one(ht, tbl, hash, key, obj); + new_tbl = rhashtable_insert_one(ht, tbl, hash, obj, data); + if (PTR_ERR(new_tbl) != -EEXIST) + data = ERR_CAST(new_tbl); + + spin_unlock(rht_bucket_lock(tbl, hash)); + } + + spin_unlock_bh(lock); if (PTR_ERR(data) == -EAGAIN) data = ERR_PTR(rhashtable_insert_rehash(ht, tbl) ?: @@ -669,7 +672,7 @@ EXPORT_SYMBOL_GPL(rhashtable_insert_slow); * structure outside the hash table. * * This function may be called from any process context, including - * non-preemptible context, but cannot be called from softirq or + * non-preemptable context, but cannot be called from softirq or * hardirq context. * * You must call rhashtable_walk_exit after this function returns. @@ -694,7 +697,7 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_enter); * rhashtable_walk_exit - Free an iterator * @iter: Hash table Iterator * - * This function frees resources allocated by rhashtable_walk_enter. + * This function frees resources allocated by rhashtable_walk_init. */ void rhashtable_walk_exit(struct rhashtable_iter *iter) { @@ -715,7 +718,7 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_exit); * * Returns zero if successful. * - * Returns -EAGAIN if resize event occurred. Note that the iterator + * Returns -EAGAIN if resize event occured. Note that the iterator * will rewind back to the beginning and you may use it immediately * by calling rhashtable_walk_next. * @@ -951,11 +954,10 @@ void rhashtable_walk_stop(struct rhashtable_iter *iter) ht = iter->ht; spin_lock(&ht->lock); - if (rcu_head_after_call_rcu(&tbl->rcu, bucket_table_free_rcu)) - /* This bucket table is being freed, don't re-link it. */ - iter->walker.tbl = NULL; - else + if (tbl->rehash < tbl->size) list_add(&iter->walker.list, &tbl->walkers); + else + iter->walker.tbl = NULL; spin_unlock(&ht->lock); out: @@ -1003,6 +1005,7 @@ static u32 rhashtable_jhash2(const void *key, u32 length, u32 seed) * .key_offset = offsetof(struct test_obj, key), * .key_len = sizeof(int), * .hashfn = jhash, + * .nulls_base = (1U << RHT_BASE_SHIFT), * }; * * Configuration Example 2: Variable length keys @@ -1024,7 +1027,7 @@ static u32 rhashtable_jhash2(const void *key, u32 length, u32 seed) * .obj_hashfn = my_hash_fn, * }; */ -int rhashtable_init_noprof(struct rhashtable *ht, +int rhashtable_init(struct rhashtable *ht, const struct rhashtable_params *params) { struct bucket_table *tbl; @@ -1034,13 +1037,14 @@ int rhashtable_init_noprof(struct rhashtable *ht, (params->obj_hashfn && !params->obj_cmpfn)) return -EINVAL; + if (params->nulls_base && params->nulls_base < (1U << RHT_BASE_SHIFT)) + return -EINVAL; + memset(ht, 0, sizeof(*ht)); mutex_init(&ht->mutex); spin_lock_init(&ht->lock); memcpy(&ht->p, params, sizeof(*params)); - alloc_tag_record(ht->alloc_tag); - if (params->min_size) ht->p.min_size = roundup_pow_of_two(params->min_size); @@ -1057,6 +1061,11 @@ int rhashtable_init_noprof(struct rhashtable *ht, size = rounded_hashtable_size(&ht->p); + if (params->locks_mul) + ht->p.locks_mul = roundup_pow_of_two(params->locks_mul); + else + ht->p.locks_mul = BUCKET_LOCKS_PER_CPU; + ht->key_len = ht->p.key_len; if (!params->hashfn) { ht->p.hashfn = jhash; @@ -1067,16 +1076,9 @@ int rhashtable_init_noprof(struct rhashtable *ht, } } - /* - * This is api initialization and thus we need to guarantee the - * initial rhashtable allocation. Upon failure, retry with the - * smallest possible size with __GFP_NOFAIL semantics. - */ tbl = bucket_table_alloc(ht, size, GFP_KERNEL); - if (unlikely(tbl == NULL)) { - size = max_t(u16, ht->p.min_size, HASH_MIN_SIZE); - tbl = bucket_table_alloc(ht, size, GFP_KERNEL | __GFP_NOFAIL); - } + if (tbl == NULL) + return -ENOMEM; atomic_set(&ht->nelems, 0); @@ -1086,7 +1088,7 @@ int rhashtable_init_noprof(struct rhashtable *ht, return 0; } -EXPORT_SYMBOL_GPL(rhashtable_init_noprof); +EXPORT_SYMBOL_GPL(rhashtable_init); /** * rhltable_init - initialize a new hash list table @@ -1097,15 +1099,19 @@ EXPORT_SYMBOL_GPL(rhashtable_init_noprof); * * See documentation for rhashtable_init. */ -int rhltable_init_noprof(struct rhltable *hlt, const struct rhashtable_params *params) +int rhltable_init(struct rhltable *hlt, const struct rhashtable_params *params) { int err; - err = rhashtable_init_noprof(&hlt->ht, params); + /* No rhlist NULLs marking for now. */ + if (params->nulls_base) + return -EINVAL; + + err = rhashtable_init(&hlt->ht, params); hlt->ht.rhlist = true; return err; } -EXPORT_SYMBOL_GPL(rhltable_init_noprof); +EXPORT_SYMBOL_GPL(rhltable_init); static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj, void (*free_fn)(void *ptr, void *arg), @@ -1158,7 +1164,7 @@ void rhashtable_free_and_destroy(struct rhashtable *ht, struct rhash_head *pos, *next; cond_resched(); - for (pos = rht_ptr_exclusive(rht_bucket(tbl, i)), + for (pos = rht_dereference(*rht_bucket(tbl, i), ht), next = !rht_is_a_nulls(pos) ? rht_dereference(pos->next, ht) : NULL; !rht_is_a_nulls(pos); @@ -1185,16 +1191,18 @@ void rhashtable_destroy(struct rhashtable *ht) } EXPORT_SYMBOL_GPL(rhashtable_destroy); -struct rhash_lock_head __rcu **__rht_bucket_nested( - const struct bucket_table *tbl, unsigned int hash) +struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, + unsigned int hash) { const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); + static struct rhash_head __rcu *rhnull = + (struct rhash_head __rcu *)NULLS_MARKER(0); unsigned int index = hash & ((1 << tbl->nest) - 1); unsigned int size = tbl->size >> tbl->nest; unsigned int subhash = hash; union nested_table *ntbl; - ntbl = nested_table_top(tbl); + ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]); ntbl = rht_dereference_bucket_rcu(ntbl[index].table, tbl, hash); subhash >>= tbl->nest; @@ -1207,43 +1215,40 @@ struct rhash_lock_head __rcu **__rht_bucket_nested( } if (!ntbl) - return NULL; + return &rhnull; return &ntbl[subhash].bucket; } -EXPORT_SYMBOL_GPL(__rht_bucket_nested); - -struct rhash_lock_head __rcu **rht_bucket_nested( - const struct bucket_table *tbl, unsigned int hash) -{ - static struct rhash_lock_head __rcu *rhnull; - - if (!rhnull) - INIT_RHT_NULLS_HEAD(rhnull); - return __rht_bucket_nested(tbl, hash) ?: &rhnull; -} EXPORT_SYMBOL_GPL(rht_bucket_nested); -struct rhash_lock_head __rcu **rht_bucket_nested_insert( - struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash) +struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, + struct bucket_table *tbl, + unsigned int hash) { const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); unsigned int index = hash & ((1 << tbl->nest) - 1); unsigned int size = tbl->size >> tbl->nest; union nested_table *ntbl; + unsigned int shifted; + unsigned int nhash; - ntbl = nested_table_top(tbl); + ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]); hash >>= tbl->nest; + nhash = index; + shifted = tbl->nest; ntbl = nested_table_alloc(ht, &ntbl[index].table, - size <= (1 << shift)); + size <= (1 << shift) ? shifted : 0, nhash); while (ntbl && size > (1 << shift)) { index = hash & ((1 << shift) - 1); size >>= shift; hash >>= shift; + nhash |= index << shifted; + shifted += shift; ntbl = nested_table_alloc(ht, &ntbl[index].table, - size <= (1 << shift)); + size <= (1 << shift) ? shifted : 0, + nhash); } if (!ntbl) diff --git a/test/unit_homa_incoming.c b/test/unit_homa_incoming.c index cc4eef96..c26188ea 100644 --- a/test/unit_homa_incoming.c +++ b/test/unit_homa_incoming.c @@ -929,23 +929,6 @@ TEST_F(homa_incoming, homa_copy_to_user__skb_data_extends_past_message_end) homa_rpc_unlock(crpc); EXPECT_STREQ("", unit_log_get()); } -TEST_F(homa_incoming, homa_copy_to_user__error_in_import_ubuf) -{ - struct homa_rpc *crpc; - - crpc = unit_client_rpc(&self->hsk, UNIT_RCVD_ONE_PKT, self->client_ip, - self->server_ip, self->server_port, self->client_id, - 1000, 4000); - ASSERT_NE(NULL, crpc); - - unit_log_clear(); - mock_import_ubuf_errors = 1; - homa_rpc_lock(crpc); - EXPECT_EQ(13, -homa_copy_to_user(crpc)); - homa_rpc_unlock(crpc); - EXPECT_STREQ("", unit_log_get()); - EXPECT_EQ(0, skb_queue_len(&crpc->msgin.packets)); -} TEST_F(homa_incoming, homa_copy_to_user__error_in_skb_copy_datagram_iter) { struct homa_rpc *crpc; diff --git a/test/unit_homa_offload.c b/test/unit_homa_offload.c index 7922b359..e6fbffa8 100644 --- a/test/unit_homa_offload.c +++ b/test/unit_homa_offload.c @@ -11,13 +11,13 @@ #define cur_offload_core (&per_cpu(homa_offload_core, smp_processor_id())) -static struct sk_buff *test_tcp_gro_receive(struct list_head *held_list, +static struct sk_buff **test_tcp_gro_receive(struct sk_buff **gro_list, struct sk_buff *skb) { UNIT_LOG("; ", "test_tcp_gro_receive"); return NULL; } -static struct sk_buff *unit_tcp6_gro_receive(struct list_head *held_list, +static struct sk_buff **unit_tcp6_gro_receive(struct sk_buff **gro_list, struct sk_buff *skb) { UNIT_LOG("; ", "unit_tcp6_gro_receive"); @@ -31,16 +31,13 @@ FIXTURE(homa_offload) struct homa_sock hsk; struct in6_addr ip; struct homa_data_hdr header; - struct napi_struct napi; + struct sk_buff *gro_list; struct sk_buff *skb, *skb2; - struct list_head empty_list; struct net_offload tcp_offloads; struct net_offload tcp6_offloads; }; FIXTURE_SETUP(homa_offload) { - int i; - homa_init(&self->homa); self->hnet = mock_hnet(0, &self->homa); self->homa.flags |= HOMA_FLAG_DONT_THROTTLE; @@ -56,28 +53,22 @@ FIXTURE_SETUP(homa_offload) self->header.message_length = htonl(10000); self->header.incoming = htonl(10000); self->header.seg.offset = htonl(2000); - for (i = 0; i < GRO_HASH_BUCKETS; i++) { - INIT_LIST_HEAD(&self->napi.gro.hash[i].list); - self->napi.gro.hash[i].count = 0; - } - self->napi.gro.bitmask = 0; self->skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 2000); NAPI_GRO_CB(self->skb)->same_flow = 0; + ((struct iphdr *) skb_network_header(self->skb))->protocol = IPPROTO_HOMA+1; + NAPI_GRO_CB(self->skb)->data_offset = sizeof(struct homa_data_hdr); NAPI_GRO_CB(self->skb)->last = self->skb; - NAPI_GRO_CB(self->skb)->count = 1; self->header.seg.offset = htonl(4000); self->header.common.dport = htons(88); self->header.common.sender_id = cpu_to_be64(1002); self->skb2 = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); NAPI_GRO_CB(self->skb2)->same_flow = 0; + NAPI_GRO_CB(self->skb2)->data_offset = sizeof(struct homa_data_hdr); NAPI_GRO_CB(self->skb2)->last = self->skb2; - NAPI_GRO_CB(self->skb2)->count = 1; - self->napi.gro.bitmask = 6; - self->napi.gro.hash[2].count = 2; - list_add_tail(&self->skb->list, &self->napi.gro.hash[2].list); - list_add_tail(&self->skb2->list, &self->napi.gro.hash[2].list); - INIT_LIST_HEAD(&self->empty_list); + self->gro_list = self->skb; + self->skb->next = self->skb2; + self->skb2->next = NULL; self->tcp_offloads.callbacks.gro_receive = test_tcp_gro_receive; inet_offloads[IPPROTO_TCP] = &self->tcp_offloads; self->tcp6_offloads.callbacks.gro_receive = unit_tcp6_gro_receive; @@ -93,11 +84,12 @@ FIXTURE_SETUP(homa_offload) } FIXTURE_TEARDOWN(homa_offload) { - struct sk_buff *skb, *tmp; - homa_offload_end(); - list_for_each_entry_safe(skb, tmp, &self->napi.gro.hash[2].list, list) - kfree_skb(skb); + while (self->gro_list) { + struct sk_buff *next = self->gro_list->next; + kfree_skb(self->gro_list); + self->gro_list = next; + } homa_destroy(&self->homa); unit_teardown(); } @@ -132,9 +124,7 @@ TEST_F(homa_offload, homa_gro_receive__update_offset_from_sequence) self->header.seg.offset = -1; skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); NAPI_GRO_CB(skb)->same_flow = 0; - cur_offload_core->held_skb = NULL; - cur_offload_core->held_bucket = 99; - EXPECT_EQ(NULL, homa_gro_receive(&self->empty_list, skb)); + EXPECT_EQ(NULL, homa_gro_receive(&self->gro_list, skb)); h = (struct homa_data_hdr *) skb_transport_header(skb); EXPECT_EQ(6000, htonl(h->seg.offset)); @@ -143,18 +133,16 @@ TEST_F(homa_offload, homa_gro_receive__update_offset_from_sequence) self->header.seg.offset = ntohl(5000); skb2 = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); NAPI_GRO_CB(skb2)->same_flow = 0; - EXPECT_EQ(NULL, homa_gro_receive(&self->empty_list, skb2)); + EXPECT_EQ(NULL, homa_gro_receive(&self->gro_list, skb2)); h = (struct homa_data_hdr *)skb_transport_header(skb2); EXPECT_EQ(5000, htonl(h->seg.offset)); - - kfree_skb(skb); - kfree_skb(skb2); } TEST_F(homa_offload, homa_gro_receive__HOMA_GRO_SHORT_BYPASS) { struct in6_addr client_ip = unit_get_in_addr("196.168.0.1"); struct in6_addr server_ip = unit_get_in_addr("1.2.3.4"); - struct sk_buff *skb, *skb2, *skb3, *skb4, *result; + struct sk_buff *skb, *skb2, *skb3, *skb4; + struct sk_buff **result; int client_port = 40000; u64 client_id = 1234; u64 server_id = 1235; @@ -179,7 +167,7 @@ TEST_F(homa_offload, homa_gro_receive__HOMA_GRO_SHORT_BYPASS) /* First attempt: HOMA_GRO_SHORT_BYPASS not enabled. */ skb = mock_skb_alloc(&self->ip, &h.common, 1400, 2000); - result = homa_gro_receive(&self->empty_list, skb); + result = homa_gro_receive(&self->gro_list, skb); EXPECT_EQ(0, -PTR_ERR(result)); EXPECT_EQ(0, homa_metrics_per_cpu()->gro_data_bypasses); @@ -189,7 +177,7 @@ TEST_F(homa_offload, homa_gro_receive__HOMA_GRO_SHORT_BYPASS) self->homa.gro_policy |= HOMA_GRO_SHORT_BYPASS; cur_offload_core->last_gro = 400; skb2 = mock_skb_alloc(&self->ip, &h.common, 1400, 2000); - result = homa_gro_receive(&self->empty_list, skb2); + result = homa_gro_receive(&self->gro_list, skb2); EXPECT_EQ(0, -PTR_ERR(result)); EXPECT_EQ(0, homa_metrics_per_cpu()->gro_data_bypasses); @@ -198,26 +186,23 @@ TEST_F(homa_offload, homa_gro_receive__HOMA_GRO_SHORT_BYPASS) h.incoming = htonl(1400); cur_offload_core->last_gro = 400; skb3 = mock_skb_alloc(&self->ip, &h.common, 1400, 4000); - result = homa_gro_receive(&self->empty_list, skb3); + result = homa_gro_receive(&self->gro_list, skb3); EXPECT_EQ(EINPROGRESS, -PTR_ERR(result)); EXPECT_EQ(1, homa_metrics_per_cpu()->gro_data_bypasses); - /* Third attempt: no bypass because core busy. */ + /* Fourth attempt: no bypass because core busy. */ cur_offload_core->last_gro = 600; skb4 = mock_skb_alloc(&self->ip, &h.common, 1400, 4000); - result = homa_gro_receive(&self->empty_list, skb3); + result = homa_gro_receive(&self->gro_list, skb4); EXPECT_EQ(0, -PTR_ERR(result)); EXPECT_EQ(1, homa_metrics_per_cpu()->gro_data_bypasses); - - kfree_skb(skb); - kfree_skb(skb2); - kfree_skb(skb4); } TEST_F(homa_offload, homa_gro_receive__fast_grant_optimization) { struct in6_addr client_ip = unit_get_in_addr("196.168.0.1"); struct in6_addr server_ip = unit_get_in_addr("1.2.3.4"); - struct sk_buff *skb, *skb2, *skb3, *result; + struct sk_buff *skb, *skb2, *skb3; + struct sk_buff **result; struct homa_grant_hdr h; int client_port = 40000; u64 client_id = 1234; @@ -243,7 +228,7 @@ TEST_F(homa_offload, homa_gro_receive__fast_grant_optimization) /* First attempt: HOMA_GRO_FAST_GRANTS not enabled. */ self->homa.gro_policy = 0; skb = mock_skb_alloc(&client_ip, &h.common, 0, 0); - result = homa_gro_receive(&self->empty_list, skb); + result = homa_gro_receive(&self->gro_list, skb); EXPECT_EQ(0, -PTR_ERR(result)); EXPECT_EQ(0, homa_metrics_per_cpu()->gro_grant_bypasses); EXPECT_STREQ("", unit_log_get()); @@ -252,7 +237,7 @@ TEST_F(homa_offload, homa_gro_receive__fast_grant_optimization) self->homa.gro_policy = HOMA_GRO_FAST_GRANTS; cur_offload_core->last_gro = 400; skb2 = mock_skb_alloc(&client_ip, &h.common, 0, 0); - result = homa_gro_receive(&self->empty_list, skb2); + result = homa_gro_receive(&self->gro_list, skb2); EXPECT_EQ(EINPROGRESS, -PTR_ERR(result)); EXPECT_EQ(1, homa_metrics_per_cpu()->gro_grant_bypasses); EXPECT_SUBSTR("xmit DATA 1400@10000", unit_log_get()); @@ -260,161 +245,120 @@ TEST_F(homa_offload, homa_gro_receive__fast_grant_optimization) /* Third attempt: core is too busy for fast grants. */ cur_offload_core->last_gro = 600; skb3 = mock_skb_alloc(&client_ip, &h.common, 0, 0); - result = homa_gro_receive(&self->empty_list, skb3); + result = homa_gro_receive(&self->gro_list, skb3); EXPECT_EQ(0, -PTR_ERR(result)); EXPECT_EQ(1, homa_metrics_per_cpu()->gro_grant_bypasses); - kfree_skb(skb); - kfree_skb(skb3); } -TEST_F(homa_offload, homa_gro_receive__no_held_skb) +TEST_F(homa_offload, homa_gro_receive__no_held_skbs) { + struct sk_buff *held_list = NULL; struct sk_buff *skb; int same_flow; self->header.seg.offset = htonl(6000); skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); - skb->hash = 2; NAPI_GRO_CB(skb)->same_flow = 0; - cur_offload_core->held_skb = NULL; - cur_offload_core->held_bucket = 2; - EXPECT_EQ(NULL, homa_gro_receive(&self->napi.gro.hash[2].list, skb)); + EXPECT_EQ(NULL, homa_gro_receive(&held_list, skb)); same_flow = NAPI_GRO_CB(skb)->same_flow; EXPECT_EQ(0, same_flow); - EXPECT_EQ(skb, cur_offload_core->held_skb); - EXPECT_EQ(2, cur_offload_core->held_bucket); kfree_skb(skb); } -TEST_F(homa_offload, homa_gro_receive__empty_merge_list) +TEST_F(homa_offload, homa_gro_receive__skip_held_skbs_that_arent_homa_packets) { struct sk_buff *skb; int same_flow; + if (skb_is_ipv6(self->gro_list)) + ipv6_hdr(self->gro_list)->nexthdr = IPPROTO_TCP; + else + ip_hdr(self->gro_list)->protocol = IPPROTO_TCP; + self->header.seg.offset = htonl(6000); skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); - skb->hash = 2; NAPI_GRO_CB(skb)->same_flow = 0; - cur_offload_core->held_skb = self->skb; - cur_offload_core->held_bucket = 3; - EXPECT_EQ(NULL, homa_gro_receive(&self->napi.gro.hash[2].list, skb)); + EXPECT_EQ(NULL, homa_gro_receive(&self->gro_list, skb)); same_flow = NAPI_GRO_CB(skb)->same_flow; - EXPECT_EQ(0, same_flow); - EXPECT_EQ(skb, cur_offload_core->held_skb); - EXPECT_EQ(2, cur_offload_core->held_bucket); - kfree_skb(skb); + EXPECT_EQ(1, same_flow); + unit_log_clear(); + unit_log_frag_list(self->gro_list, 0); + EXPECT_STREQ("", + unit_log_get()); + unit_log_clear(); + unit_log_frag_list(self->gro_list->next, 0); + EXPECT_STREQ("DATA 1400@6000", + unit_log_get()); } -TEST_F(homa_offload, homa_gro_receive__held_skb_not_in_merge_list) +TEST_F(homa_offload, homa_gro_receive__add_to_frag_list) { struct sk_buff *skb; int same_flow; self->header.seg.offset = htonl(6000); skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); - skb->hash = 3; NAPI_GRO_CB(skb)->same_flow = 0; - cur_offload_core->held_skb = skb; - cur_offload_core->held_bucket = 2; - EXPECT_EQ(NULL, homa_gro_receive(&self->napi.gro.hash[3].list, skb)); + EXPECT_EQ(NULL, homa_gro_receive(&self->gro_list, skb)); same_flow = NAPI_GRO_CB(skb)->same_flow; - EXPECT_EQ(0, same_flow); - EXPECT_EQ(skb, cur_offload_core->held_skb); - EXPECT_EQ(3, cur_offload_core->held_bucket); - kfree_skb(skb); -} -TEST_F(homa_offload, homa_gro_receive__held_skb__in_merge_list_but_wrong_proto) -{ - struct sk_buff *skb; - int same_flow; + EXPECT_EQ(1, same_flow); - self->header.seg.offset = htonl(6000); + self->header.seg.offset = htonl(7400); skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); - skb->hash = 3; NAPI_GRO_CB(skb)->same_flow = 0; - cur_offload_core->held_skb = self->skb; - if (skb_is_ipv6(self->skb)) - ipv6_hdr(self->skb)->nexthdr = IPPROTO_TCP; - else - ip_hdr(self->skb)->protocol = IPPROTO_TCP; - cur_offload_core->held_bucket = 2; - EXPECT_EQ(NULL, homa_gro_receive(&self->napi.gro.hash[3].list, skb)); + EXPECT_EQ(NULL, homa_gro_receive(&self->gro_list, skb)); same_flow = NAPI_GRO_CB(skb)->same_flow; - EXPECT_EQ(0, same_flow); - EXPECT_EQ(skb, cur_offload_core->held_skb); - EXPECT_EQ(3, cur_offload_core->held_bucket); - kfree_skb(skb); + EXPECT_EQ(1, same_flow); + unit_log_clear(); + unit_log_frag_list(self->gro_list, 0); + EXPECT_STREQ("DATA 1400@6000; DATA 1400@7400", unit_log_get()); } -TEST_F(homa_offload, homa_gro_receive__merge) +TEST_F(homa_offload, homa_gro_receive__max_gro_skbs) { - struct sk_buff *skb, *skb2; - int same_flow; + struct homa_common_hdr *h; + struct sk_buff *skb; - cur_offload_core->held_skb = self->skb2; - cur_offload_core->held_bucket = 2; + h = (struct homa_common_hdr *)skb_transport_header(self->gro_list); + // First packet fits below the limit. + self->homa.max_gro_skbs = 3; self->header.seg.offset = htonl(6000); - self->header.common.sender_id = cpu_to_be64(1002); skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); - NAPI_GRO_CB(skb)->same_flow = 0; - EXPECT_EQ(NULL, homa_gro_receive(&self->napi.gro.hash[3].list, skb)); - same_flow = NAPI_GRO_CB(skb)->same_flow; - EXPECT_EQ(1, same_flow); - EXPECT_EQ(2, NAPI_GRO_CB(self->skb2)->count); + EXPECT_EQ(NULL, homa_gro_receive(&self->gro_list, skb)); + EXPECT_EQ(1, NAPI_GRO_CB(self->gro_list)->count); + EXPECT_EQ(1, h->gro_count); - self->header.seg.offset = htonl(7000); - self->header.common.sender_id = cpu_to_be64(1004); - skb2 = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); - NAPI_GRO_CB(skb2)->same_flow = 0; - EXPECT_EQ(NULL, homa_gro_receive(&self->napi.gro.hash[3].list, skb2)); - same_flow = NAPI_GRO_CB(skb)->same_flow; - EXPECT_EQ(1, same_flow); - EXPECT_EQ(3, NAPI_GRO_CB(self->skb2)->count); + // Second packet also fits below the limit. + self->header.seg.offset = htonl(8000); + skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); + EXPECT_EQ(NULL, homa_gro_receive(&self->gro_list, skb)); + EXPECT_EQ(2, NAPI_GRO_CB(self->gro_list)->count); + EXPECT_EQ(2, h->gro_count); - unit_log_frag_list(self->skb2, 1); - EXPECT_STREQ("DATA from 196.168.0.1:40000, dport 88, id 1002, message_length 10000, offset 6000, data_length 1400, incoming 10000; " - "DATA from 196.168.0.1:40000, dport 88, id 1004, message_length 10000, offset 7000, data_length 1400, incoming 10000", - unit_log_get()); + // Third packet hits the limit. + self->header.seg.offset = htonl(10000); + skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); + EXPECT_EQ(&self->gro_list, homa_gro_receive(&self->gro_list, skb)); + EXPECT_EQ(3, NAPI_GRO_CB(self->gro_list)->count); + EXPECT_EQ(3, h->gro_count); } -TEST_F(homa_offload, homa_gro_receive__max_gro_skbs) +TEST_F(homa_offload, homa_gro_receive__set_softirq_cpu) { + struct sk_buff *held_list = NULL; struct sk_buff *skb; - // First packet: fits below the limit. - self->homa.max_gro_skbs = 3; - cur_offload_core->held_skb = self->skb2; - cur_offload_core->held_bucket = 2; - self->header.seg.offset = htonl(6000); - skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); - homa_gro_receive(&self->napi.gro.hash[3].list, skb); - EXPECT_EQ(2, NAPI_GRO_CB(self->skb2)->count); - EXPECT_EQ(2, self->napi.gro.hash[2].count); + cpu_number = 5; - // Second packet hits the limit. - self->header.common.sport = htons(40001); - skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); - unit_log_clear(); - EXPECT_EQ(EINPROGRESS, -PTR_ERR(homa_gro_receive( - &self->napi.gro.hash[3].list, skb))); - EXPECT_EQ(3, NAPI_GRO_CB(self->skb2)->count); - EXPECT_EQ(1, self->napi.gro.hash[2].count); - EXPECT_STREQ("netif_receive_skb, id 1002, offset 4000", - unit_log_get()); - kfree_skb(self->skb2); - EXPECT_EQ(1, self->napi.gro.hash[2].count); - EXPECT_EQ(6, self->napi.gro.bitmask); - - // Third packet also hits the limit for skb, causing the bucket - // to become empty. - self->homa.max_gro_skbs = 2; - cur_offload_core->held_skb = self->skb; + /* First call: HOMA_GRO_SAME_CORE not set. */ skb = mock_skb_alloc(&self->ip, &self->header.common, 1400, 0); - unit_log_clear(); - EXPECT_EQ(EINPROGRESS, -PTR_ERR(homa_gro_receive( - &self->napi.gro.hash[3].list, skb))); - EXPECT_EQ(2, NAPI_GRO_CB(self->skb)->count); - EXPECT_EQ(0, self->napi.gro.hash[2].count); - EXPECT_EQ(2, self->napi.gro.bitmask); - EXPECT_STREQ("netif_receive_skb, id 1000, offset 2000", - unit_log_get()); - kfree_skb(self->skb); + NAPI_GRO_CB(skb)->same_flow = 0; + skb->hash = 0; + self->homa.gro_policy &= ~HOMA_GRO_SAME_CORE; + EXPECT_EQ(NULL, homa_gro_receive(&held_list, skb)); + EXPECT_EQ(0, skb->hash); + + /* Second call: HOMA_GRO_SAME_CORE set. */ + self->homa.gro_policy |= HOMA_GRO_SAME_CORE; + EXPECT_EQ(NULL, homa_gro_receive(&held_list, skb)); + EXPECT_EQ(rps_cpu_mask + 6, skb->hash); + kfree_skb(skb); } TEST_F(homa_offload, homa_gro_gen2) @@ -512,7 +456,6 @@ TEST_F(homa_offload, homa_gro_gen3__all_cores_busy_so_pick_first) EXPECT_EQ(5000, per_cpu(homa_offload_core, 3).last_active); } - TEST_F(homa_offload, homa_gro_complete__clear_held_skb) { struct homa_offload_core *offload_core = &per_cpu(homa_offload_core, diff --git a/test/unit_homa_outgoing.c b/test/unit_homa_outgoing.c index 2e39c41b..425c2c21 100644 --- a/test/unit_homa_outgoing.c +++ b/test/unit_homa_outgoing.c @@ -1068,7 +1068,7 @@ TEST_F(homa_outgoing, __homa_xmit_data__fill_dst) #endif /* See strip.py */ unit_log_clear(); dst = crpc->peer->dst; - old_refcount = atomic_read(&dst->__rcuref.refcnt); + old_refcount = atomic_read(&dst->__refcnt); skb_get(crpc->msgout.packets); #ifndef __STRIP__ /* See strip.py */ @@ -1078,7 +1078,7 @@ TEST_F(homa_outgoing, __homa_xmit_data__fill_dst) #endif /* See strip.py */ EXPECT_STREQ("xmit DATA 1000@0", unit_log_get()); EXPECT_EQ(dst, skb_dst(crpc->msgout.packets)); - EXPECT_EQ(old_refcount+1, atomic_read(&dst->__rcuref.refcnt)); + EXPECT_EQ(old_refcount+1, atomic_read(&dst->__refcnt)); } #ifndef __STRIP__ /* See strip.py */ TEST_F(homa_outgoing, __homa_xmit_data__ipv6_call_homa_hijack_set_hdr) diff --git a/test/unit_homa_peer.c b/test/unit_homa_peer.c index 3bc583a4..226d673d 100644 --- a/test/unit_homa_peer.c +++ b/test/unit_homa_peer.c @@ -149,11 +149,10 @@ TEST_F(homa_peer, homa_peer_release_fn) peer = homa_peer_alloc(&self->hsk, ip3333); dst = peer->dst; dst_hold(dst); - EXPECT_EQ(2, atomic_read(&dst->__rcuref.refcnt)); - homa_peer_release(peer); + EXPECT_EQ(2, atomic_read(&dst->__refcnt)); homa_peer_release_fn(peer, NULL); - EXPECT_EQ(1, atomic_read(&dst->__rcuref.refcnt)); + EXPECT_EQ(1, atomic_read(&dst->__refcnt)); dst_release(dst); } @@ -440,7 +439,7 @@ TEST_F(homa_peer, homa_peer_alloc__success) EXPECT_EQ(0, peer->cutoff_version); EXPECT_EQ(1, homa_metrics_per_cpu()->peer_allocs); #endif /* See strip.py */ - EXPECT_EQ(1, atomic_read(&peer->dst->__rcuref.refcnt)); + EXPECT_EQ(1, atomic_read(&peer->dst->__refcnt)); homa_peer_release(peer); } TEST_F(homa_peer, homa_peer_alloc__kmalloc_error) @@ -478,10 +477,10 @@ TEST_F(homa_peer, homa_peer_free) ASSERT_FALSE(IS_ERR(peer)); dst = peer->dst; dst_hold(dst); - ASSERT_EQ(2, atomic_read(&dst->__rcuref.refcnt)); + ASSERT_EQ(2, atomic_read(&dst->__refcnt)); homa_peer_release(peer); - ASSERT_EQ(1, atomic_read(&dst->__rcuref.refcnt)); + ASSERT_EQ(1, atomic_read(&dst->__refcnt)); dst_release(dst); } @@ -614,7 +613,7 @@ TEST_F(homa_peer, homa_get_dst__normal) struct dst_entry *dst; dst = homa_get_dst(peer, &self->hsk); - EXPECT_EQ(2, atomic_read(&dst->__rcuref.refcnt)); + EXPECT_EQ(2, atomic_read(&dst->__refcnt)); IF_NO_STRIP(EXPECT_EQ(0, homa_metrics_per_cpu()->peer_dst_refreshes)); dst_release(dst); homa_peer_release(peer); @@ -628,7 +627,7 @@ TEST_F(homa_peer, homa_get_dst__must_refresh_obsolete) peer->dst->obsolete = 1; mock_dst_check_errors = 1; dst = homa_get_dst(peer, &self->hsk); - EXPECT_EQ(2, atomic_read(&dst->__rcuref.refcnt)); + EXPECT_EQ(2, atomic_read(&dst->__refcnt)); IF_NO_STRIP(EXPECT_EQ(1, homa_metrics_per_cpu()->peer_dst_refreshes)); EXPECT_NE(old, dst); dst_release(dst); @@ -644,7 +643,7 @@ TEST_F(homa_peer, homa_get_dst__multiple_refresh_failures) mock_dst_check_errors = 0xf; mock_route_errors = 0xf; dst = homa_get_dst(peer, &self->hsk); - EXPECT_EQ(2, atomic_read(&dst->__rcuref.refcnt)); + EXPECT_EQ(2, atomic_read(&dst->__refcnt)); IF_NO_STRIP(EXPECT_EQ(1, homa_metrics_per_cpu()->peer_dst_refreshes)); EXPECT_EQ(old, dst); EXPECT_EQ(3, mock_dst_check_errors); diff --git a/test/unit_homa_plumbing.c b/test/unit_homa_plumbing.c index 70ea301f..a9c142ca 100644 --- a/test/unit_homa_plumbing.c +++ b/test/unit_homa_plumbing.c @@ -29,7 +29,6 @@ FIXTURE(homa_plumbing) { struct msghdr sendmsg_hdr; struct homa_sendmsg_args sendmsg_args; char buffer[2000]; - sockptr_t optval; union sockaddr_in_union addr; }; FIXTURE_SETUP(homa_plumbing) @@ -86,11 +85,8 @@ FIXTURE_SETUP(homa_plumbing) 2, 200); self->sendmsg_hdr.msg_control = &self->sendmsg_args; self->sendmsg_hdr.msg_controllen = sizeof(self->sendmsg_args); - self->sendmsg_hdr.msg_control_is_user = 1; self->sendmsg_args.id = 0; self->sendmsg_args.completion_cookie = 0; - self->optval.user = (void *) 0x100000; - self->optval.is_kernel = 0; unit_log_clear(); if (self->homa.wmem_max == 0) printf("homa_plumbing fixture set wmem_max 0\n"); @@ -578,14 +574,14 @@ TEST_F(homa_plumbing, homa_socket__homa_sock_init_failure) TEST_F(homa_plumbing, homa_setsockopt__bad_level) { EXPECT_EQ(ENOPROTOOPT, -homa_setsockopt(&self->hsk.sock, 0, 0, - self->optval, sizeof(struct homa_rcvbuf_args))); + (void *)0x100000, sizeof(struct homa_rcvbuf_args))); EXPECT_STREQ("homa_setsockopt invoked with level not IPPROTO_HOMA", self->hsk.error_msg); } TEST_F(homa_plumbing, homa_setsockopt__recvbuf_bad_optlen) { EXPECT_EQ(EINVAL, -homa_setsockopt(&self->hsk.sock, IPPROTO_HOMA, - SO_HOMA_RCVBUF, self->optval, + SO_HOMA_RCVBUF, (void *)0x100000, sizeof(struct homa_rcvbuf_args) - 1)); EXPECT_STREQ("invalid optlen argument: must be sizeof(struct homa_rcvbuf_args)", self->hsk.error_msg); @@ -594,7 +590,7 @@ TEST_F(homa_plumbing, homa_setsockopt__recvbuf_copy_from_sockptr_fails) { mock_copy_data_errors = 1; EXPECT_EQ(EFAULT, -homa_setsockopt(&self->hsk.sock, IPPROTO_HOMA, - SO_HOMA_RCVBUF, self->optval, + SO_HOMA_RCVBUF, (void *)0x100000, sizeof(struct homa_rcvbuf_args))); EXPECT_STREQ("invalid address for homa_rcvbuf_args", self->hsk.error_msg); @@ -603,10 +599,9 @@ TEST_F(homa_plumbing, homa_setsockopt__recvbuf_region_not_writable) { struct homa_rcvbuf_args args = {0x100000, 5*HOMA_BPAGE_SIZE}; - self->optval.user = &args; mock_copy_to_user_errors = 1; EXPECT_EQ(EFAULT, -homa_setsockopt(&self->hsk.sock, IPPROTO_HOMA, - SO_HOMA_RCVBUF, self->optval, + SO_HOMA_RCVBUF, (void *)&args, sizeof(struct homa_rcvbuf_args))); EXPECT_STREQ("receive buffer region is not writable", self->hsk.error_msg); @@ -619,11 +614,10 @@ TEST_F(homa_plumbing, homa_setsockopt__recvbuf_success) args.start = (((uintptr_t)(buffer + PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1)); args.length = 64*HOMA_BPAGE_SIZE; - self->optval.user = &args; homa_pool_free(self->hsk.buffer_pool); self->hsk.buffer_pool = homa_pool_alloc(&self->hsk); EXPECT_EQ(0, -homa_setsockopt(&self->hsk.sock, IPPROTO_HOMA, - SO_HOMA_RCVBUF, self->optval, + SO_HOMA_RCVBUF, (void *)&args, sizeof(struct homa_rcvbuf_args))); EXPECT_EQ(args.start, (uintptr_t)self->hsk.buffer_pool->region); EXPECT_EQ(64, self->hsk.buffer_pool->num_bpages); @@ -634,7 +628,7 @@ TEST_F(homa_plumbing, homa_setsockopt__recvbuf_success) TEST_F(homa_plumbing, homa_setsockopt__server_bad_optlen) { EXPECT_EQ(EINVAL, -homa_setsockopt(&self->hsk.sock, IPPROTO_HOMA, - SO_HOMA_SERVER, self->optval, sizeof(int) - 1)); + SO_HOMA_SERVER, (void *)0x100000, sizeof(int) - 1)); EXPECT_STREQ("invalid optlen argument: must be sizeof(int)", self->hsk.error_msg); } @@ -642,7 +636,7 @@ TEST_F(homa_plumbing, homa_setsockopt__server_copy_from_sockptr_fails) { mock_copy_data_errors = 1; EXPECT_EQ(EFAULT, -homa_setsockopt(&self->hsk.sock, IPPROTO_HOMA, - SO_HOMA_SERVER, self->optval, sizeof(int))); + SO_HOMA_SERVER, (void *)0x100000, sizeof(int))); EXPECT_STREQ("invalid address for SO_HOMA_SERVER value", self->hsk.error_msg); } @@ -650,20 +644,19 @@ TEST_F(homa_plumbing, homa_setsockopt__server_success) { int arg = 7; - self->optval.user = &arg; EXPECT_EQ(0, -homa_setsockopt(&self->hsk.sock, IPPROTO_HOMA, - SO_HOMA_SERVER, self->optval, sizeof(int))); + SO_HOMA_SERVER, (void *)&arg, sizeof(int))); EXPECT_EQ(1, self->hsk.is_server); arg = 0; EXPECT_EQ(0, -homa_setsockopt(&self->hsk.sock, IPPROTO_HOMA, - SO_HOMA_SERVER, self->optval, sizeof(int))); + SO_HOMA_SERVER, (void *)&arg, sizeof(int))); EXPECT_EQ(0, self->hsk.is_server); } TEST_F(homa_plumbing, homa_setsockopt__bad_optname) { EXPECT_EQ(ENOPROTOOPT, -homa_setsockopt(&self->hsk.sock, IPPROTO_HOMA, 0, - self->optval, sizeof(struct homa_rcvbuf_args))); + (void *)0x100000, sizeof(struct homa_rcvbuf_args))); EXPECT_STREQ("setsockopt option not supported by Homa", self->hsk.error_msg); } @@ -790,15 +783,6 @@ TEST_F(homa_plumbing, homa_sendmsg__msg_name_null) self->hsk.error_msg); EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); } -TEST_F(homa_plumbing, homa_sendmsg__msg_control_not_in_user_space) -{ - self->sendmsg_hdr.msg_control_is_user = 0; - EXPECT_EQ(EINVAL, -homa_sendmsg(&self->hsk.inet.sk, - &self->sendmsg_hdr, self->sendmsg_hdr.msg_iter.count)); - EXPECT_STREQ("msg_control argument for sendmsg isn't in user space", - self->hsk.error_msg); - EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); -} TEST_F(homa_plumbing, homa_sendmsg__cant_read_msg_control) { mock_copy_data_errors = 1; @@ -1023,7 +1007,7 @@ TEST_F(homa_plumbing, homa_recvmsg__wrong_args_length) { self->recvmsg_hdr.msg_controllen -= 1; EXPECT_EQ(EINVAL, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("invalid msg_controllen in recvmsg", self->hsk.error_msg); } @@ -1031,7 +1015,7 @@ TEST_F(homa_plumbing, homa_recvmsg__cant_read_args) { mock_copy_data_errors = 1; EXPECT_EQ(EFAULT, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("invalid address for msg_control argument to recvmsg", self->hsk.error_msg); } @@ -1042,7 +1026,7 @@ TEST_F(homa_plumbing, homa_recvmsg__clear_cookie) self->recvmsg_args.completion_cookie = 12345; self->recvmsg_args.num_bpages = 1000000; EXPECT_EQ(EINVAL, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_EQ(0, self->recvmsg_args.completion_cookie); self->recvmsg_args.num_bpages = 0; } @@ -1050,7 +1034,7 @@ TEST_F(homa_plumbing, homa_recvmsg__num_bpages_too_large) { self->recvmsg_args.num_bpages = HOMA_MAX_BPAGES + 1; EXPECT_EQ(EINVAL, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("num_pages exceeds HOMA_MAX_BPAGES", self->hsk.error_msg); EXPECT_EQ(HOMA_MAX_BPAGES + 1, self->recvmsg_args.num_bpages); self->recvmsg_args.num_bpages = 0; @@ -1060,7 +1044,7 @@ TEST_F(homa_plumbing, homa_recvmsg__reserved_not_zero) self->recvmsg_args.reserved = 1; self->recvmsg_args.num_bpages = 10; EXPECT_EQ(EINVAL, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("reserved fields in homa_recvmsg_args must be zero", self->hsk.error_msg); EXPECT_EQ(10, self->recvmsg_args.num_bpages); @@ -1073,7 +1057,7 @@ TEST_F(homa_plumbing, homa_recvmsg__no_buffer_pool) self->hsk.buffer_pool = NULL; self->recvmsg_args.num_bpages = 10; EXPECT_EQ(EINVAL, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("SO_HOMA_RECVBUF socket option has not been set", self->hsk.error_msg); self->hsk.buffer_pool = saved_pool; @@ -1091,7 +1075,7 @@ TEST_F(homa_plumbing, homa_recvmsg__release_buffers) self->recvmsg_args.bpage_offsets[1] = HOMA_BPAGE_SIZE; EXPECT_EQ(EAGAIN, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, MSG_DONTWAIT, &self->recvmsg_hdr.msg_namelen)); + 0, MSG_DONTWAIT, 1, &self->recvmsg_hdr.msg_namelen)); EXPECT_EQ(0, atomic_read(&self->hsk.buffer_pool->descriptors[0].refs)); EXPECT_EQ(0, atomic_read(&self->hsk.buffer_pool->descriptors[1].refs)); EXPECT_EQ(0, self->recvmsg_args.num_bpages); @@ -1103,7 +1087,7 @@ TEST_F(homa_plumbing, homa_recvmsg__error_in_release_buffers) self->hsk.buffer_pool->num_bpages << HOMA_BPAGE_SHIFT; EXPECT_EQ(EINVAL, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, MSG_DONTWAIT, &self->recvmsg_hdr.msg_namelen)); + 0, MSG_DONTWAIT, 1, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("error while releasing buffer pages", self->hsk.error_msg); EXPECT_EQ(0, self->recvmsg_args.num_bpages); @@ -1113,7 +1097,7 @@ TEST_F(homa_plumbing, homa_recvmsg__private_rpc_doesnt_exist) self->recvmsg_args.id = 99; EXPECT_EQ(EINVAL, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("invalid RPC id passed to recvmsg", self->hsk.error_msg); } @@ -1129,7 +1113,7 @@ TEST_F(homa_plumbing, homa_recvmsg__error_from_homa_wait_private) self->recvmsg_args.id = crpc->id; EXPECT_EQ(EAGAIN, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, MSG_DONTWAIT, &self->recvmsg_hdr.msg_namelen)); + 0, MSG_DONTWAIT, 1, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("error while waiting for private RPC to complete", self->hsk.error_msg); EXPECT_EQ(0, self->recvmsg_args.id); @@ -1148,7 +1132,7 @@ TEST_F(homa_plumbing, homa_recvmsg__private_rpc_has_error) self->recvmsg_args.id = crpc->id; EXPECT_EQ(ETIMEDOUT, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, MSG_DONTWAIT, &self->recvmsg_hdr.msg_namelen)); + 0, MSG_DONTWAIT, 1, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("RPC failed", self->hsk.error_msg); EXPECT_EQ(self->client_id, self->recvmsg_args.id); EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); @@ -1156,7 +1140,7 @@ TEST_F(homa_plumbing, homa_recvmsg__private_rpc_has_error) TEST_F(homa_plumbing, homa_recvmsg__error_from_homa_wait_shared) { EXPECT_EQ(EAGAIN, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, MSG_DONTWAIT, &self->recvmsg_hdr.msg_namelen)); + 0, MSG_DONTWAIT, 1, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("error while waiting for shared RPC to complete", self->hsk.error_msg); } @@ -1169,7 +1153,7 @@ TEST_F(homa_plumbing, homa_recvmsg__MSG_DONT_WAIT) EXPECT_NE(NULL, crpc); EXPECT_EQ(EAGAIN, -homa_recvmsg(&self->hsk.inet.sk, - &self->recvmsg_hdr, 0, MSG_DONTWAIT, + &self->recvmsg_hdr, 0, MSG_DONTWAIT, 1, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("error while waiting for shared RPC to complete", self->hsk.error_msg); @@ -1193,7 +1177,7 @@ TEST_F(homa_plumbing, homa_recvmsg__normal_completion_ipv4) crpc->completion_cookie = 44444; EXPECT_EQ(2000, homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_EQ(self->client_id, self->recvmsg_args.id); EXPECT_EQ(44444, self->recvmsg_args.completion_cookie); EXPECT_EQ(AF_INET, self->addr.in4.sin_family); @@ -1228,7 +1212,7 @@ TEST_F(homa_plumbing, homa_recvmsg__normal_completion_ipv6) crpc->completion_cookie = 44444; EXPECT_EQ(2000, homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_EQ(self->client_id, self->recvmsg_args.id); EXPECT_EQ(44444, self->recvmsg_args.completion_cookie); EXPECT_EQ(AF_INET6, self->addr.in6.sin6_family); @@ -1251,7 +1235,7 @@ TEST_F(homa_plumbing, homa_recvmsg__rpc_has_error) homa_rpc_abort(crpc, -ETIMEDOUT); EXPECT_EQ(ETIMEDOUT, -homa_recvmsg(&self->hsk.inet.sk, - &self->recvmsg_hdr, 0, 0, + &self->recvmsg_hdr, 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("RPC failed", self->hsk.error_msg); EXPECT_EQ(self->client_id, self->recvmsg_args.id); @@ -1275,7 +1259,7 @@ TEST_F(homa_plumbing, homa_recvmsg__add_ack) peer = crpc->peer; EXPECT_EQ(2000, homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_EQ(1, peer->num_acks); } TEST_F(homa_plumbing, homa_recvmsg__server_normal_completion) @@ -1286,7 +1270,7 @@ TEST_F(homa_plumbing, homa_recvmsg__server_normal_completion) EXPECT_NE(NULL, srpc); EXPECT_EQ(100, homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_EQ(self->server_id, self->recvmsg_args.id); EXPECT_EQ(RPC_IN_SERVICE, srpc->state); EXPECT_EQ(0, srpc->peer->num_acks); @@ -1302,7 +1286,7 @@ TEST_F(homa_plumbing, homa_recvmsg__delete_server_rpc_after_error) srpc->error = -ENOMEM; EXPECT_EQ(ENOMEM, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_EQ(self->server_id, self->recvmsg_args.id); EXPECT_EQ(RPC_DEAD, srpc->state); EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); @@ -1324,7 +1308,7 @@ TEST_F(homa_plumbing, homa_recvmsg__reap_because_of_SOCK_NOSPACE) set_bit(HOMA_SOCK_NOSPACE, &self->hsk.flags); EXPECT_EQ(2000, homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_EQ(1, refcount_read(&self->hsk.sock.sk_wmem_alloc)); EXPECT_EQ(0, self->hsk.dead_skbs); IF_NO_STRIP(EXPECT_EQ(1, homa_metrics_per_cpu()->reaper_calls)); @@ -1341,7 +1325,7 @@ TEST_F(homa_plumbing, homa_recvmsg__error_copying_out_args) mock_copy_to_user_errors = 1; EXPECT_EQ(EFAULT, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, 0, &self->recvmsg_hdr.msg_namelen)); + 0, 0, 0, &self->recvmsg_hdr.msg_namelen)); EXPECT_STREQ("couldn't update homa_recvmsg_args argument to recvmsg: read-only?", self->hsk.error_msg); EXPECT_EQ(0, self->recvmsg_args.id); @@ -1358,7 +1342,7 @@ TEST_F(homa_plumbing, homa_recvmsg__copy_back_args_even_after_error) self->recvmsg_args.bpage_offsets[1] = HOMA_BPAGE_SIZE; EXPECT_EQ(EAGAIN, -homa_recvmsg(&self->hsk.inet.sk, &self->recvmsg_hdr, - 0, MSG_DONTWAIT, &self->recvmsg_hdr.msg_namelen)); + 0, MSG_DONTWAIT, 1, &self->recvmsg_hdr.msg_namelen)); EXPECT_EQ(0, self->recvmsg_args.num_bpages); } @@ -1564,7 +1548,7 @@ TEST_F(homa_plumbing, homa_err_handler_v4__port_unreachable) icmp->data = skb_tail_pointer(icmp); memcpy(skb_put(icmp, failed->len), failed->head, failed->len); - EXPECT_EQ(0, homa_err_handler_v4(icmp, 111)); + homa_err_handler_v4(icmp, 111); EXPECT_EQ(ENOTCONN, -crpc->error); kfree_skb(icmp); @@ -1592,7 +1576,7 @@ TEST_F(homa_plumbing, homa_err_handler_v4__host_unreachable) icmp->data = skb_tail_pointer(icmp); memcpy(skb_put(icmp, failed->len), failed->head, failed->len); - EXPECT_EQ(0, homa_err_handler_v4(icmp, 111)); + homa_err_handler_v4(icmp, 111); EXPECT_EQ(EHOSTUNREACH, -crpc->error); kfree_skb(icmp); diff --git a/test/unit_homa_skb.c b/test/unit_homa_skb.c index f4a9dfc2..394f0964 100644 --- a/test/unit_homa_skb.c +++ b/test/unit_homa_skb.c @@ -237,7 +237,7 @@ TEST_F(homa_skb, homa_skb_extend_frags__cant_merge_allocate_new_page) EXPECT_NE(NULL, p3); EXPECT_EQ(1000, length); EXPECT_EQ(2, skb_shinfo(self->skb)->nr_frags); - EXPECT_EQ(0, skb_shinfo(self->skb)->frags[1].offset); + EXPECT_EQ(0, skb_shinfo(self->skb)->frags[1].page_offset); EXPECT_EQ(2000, self->skb->len); EXPECT_EQ(1000, skb_core->page_inuse); @@ -267,7 +267,7 @@ TEST_F(homa_skb, homa_skb_extend_frags__cant_merge_use_same_page_reduce_length) EXPECT_EQ(p2 + 512, p3); EXPECT_EQ(512, length); EXPECT_EQ(2, skb_shinfo(self->skb)->nr_frags); - EXPECT_EQ(1536, skb_shinfo(self->skb)->frags[1].offset); + EXPECT_EQ(1536, skb_shinfo(self->skb)->frags[1].page_offset); EXPECT_EQ(2048, skb_core->page_inuse); kfree_skb(skb2); @@ -397,13 +397,13 @@ TEST_F(homa_skb, homa_skb_append_to_frag__basics) EXPECT_EQ(2, shinfo->nr_frags); EXPECT_EQ(10, skb_frag_size(&shinfo->frags[0])); p = ((char *) page_address(skb_frag_page(&shinfo->frags[0]))) - + shinfo->frags[0].offset; + + shinfo->frags[0].page_offset; p[skb_frag_size(&shinfo->frags[0])] = 0; EXPECT_STREQ("abcd012345", p); EXPECT_EQ(15, skb_frag_size(&shinfo->frags[1])); p = ((char *) page_address(skb_frag_page(&shinfo->frags[1]))) - + shinfo->frags[1].offset; + + shinfo->frags[1].page_offset; EXPECT_STREQ("6789ABCDEFGHIJ", p); } TEST_F(homa_skb, homa_skb_append_to_frag__no_memory) diff --git a/test/unit_homa_sock.c b/test/unit_homa_sock.c index 4f872247..e3d8e973 100644 --- a/test/unit_homa_sock.c +++ b/test/unit_homa_sock.c @@ -198,13 +198,16 @@ TEST_F(homa_sock, homa_sock_init__ip_header_length) TEST_F(homa_sock, homa_sock_init__hijack_tcp) { struct homa_sock hijack, no_hijack; + int protocol; self->homa.hijack_tcp = 0; mock_sock_init(&no_hijack, self->hnet, 0); self->homa.hijack_tcp = 1; mock_sock_init(&hijack, self->hnet, 0); - EXPECT_EQ(IPPROTO_HOMA, no_hijack.sock.sk_protocol); - EXPECT_EQ(IPPROTO_TCP, hijack.sock.sk_protocol); + protocol = no_hijack.sock.sk_protocol; + EXPECT_EQ(IPPROTO_HOMA, protocol); + protocol = hijack.sock.sk_protocol; + EXPECT_EQ(IPPROTO_TCP, protocol); unit_sock_destroy(&hijack); unit_sock_destroy(&no_hijack); } diff --git a/timetrace.c b/timetrace.c index 0a04e232..51087a8a 100644 --- a/timetrace.c +++ b/timetrace.c @@ -51,10 +51,10 @@ struct tt_buffer *tt_buffers[NR_CPUS]; * from /proc. */ static const struct proc_ops tt_pops = { - .proc_open = tt_proc_open, - .proc_read = tt_proc_read, - .proc_lseek = tt_proc_lseek, - .proc_release = tt_proc_release + .proc_open = tt_proc_open, + .proc_read = tt_proc_read, + .proc_lseek = tt_proc_lseek, + .proc_release = tt_proc_release }; /* Used to remove the /proc file during tt_destroy. */