udp传输丢包问题

本文主要介绍关于kubernetes,conntrack的知识点，对【k8s之udp丢包问题】和【udp传输丢包问题】有兴趣的朋友可以看下由【分享放大价值】投稿的技术文章，希望该技术和经验能帮到你解决你所遇的kubernetes,疑难杂症相关技术问题。

udp传输丢包问题

最近在k8s环境上，碰到udp丢包的问题，测试拓扑图如下，

#include #include #include #include #include #include #include #include #include #include #include #include #define __USE_GNU #include #include #include #include #define THREAD_COUNT 4 struct sockaddr_in sin; unsigned char sendBuf[sizeof(struct ethhdr) + 60]; int info[THREAD_COUNT]; int sockfd[THREAD_COUNT]; static int source_port; static int source; static int last_use[THREAD_COUNT]; unsigned short checksum(unsigned short *buf,int nword) { unsigned long sum; for(sum=0;nword>0;nword--) { sum += *buf++; sum = (sum>>16) + (sum&0xffff); } return ~sum; } #define PORT_RANGE_START (40000) #define PORT_RANGE_END (65535) #define PORT_RANGE_MASK (PORT_RANGE_END - PORT_RANGE_START) uint16_t get_port(uint32_t seed) { return (seed & PORT_RANGE_MASK) + PORT_RANGE_START; } int create_socket(int index) { //套接字 sockfd[index] = socket(AF_INET, SOCK_RAW, IPPROTO_UDP); if (sockfd[index] == -1) { perror("create socket fail"); return 0; } int daddr = inet_addr("2.2.2.4"); sin.sin_family = AF_INET; sin.sin_port = htons(80); sin.sin_addr.s_addr = daddr; int val = 1; if (setsockopt(sockfd[index], IPPROTO_IP, IP_HDRINCL, &val, sizeof(val)) <0) { printf("IP_HDRINCL\n"); return 0; } //填充IP头部 struct iphdr ipheader; memset(&ipheader, 0, sizeof(struct iphdr)); ipheader.version = 0x4; ipheader.ihl = 0x5; ipheader.tos = 0x00; ipheader.tot_len = htons(60); //20 + 8 + 32 ipheader.id = 0x1000; ipheader.frag_off = 0x0000; ipheader.ttl = 128; ipheader.protocol = 17; ipheader.check = 0; ipheader.saddr = inet_addr("2.2.2.2"); ipheader.daddr = daddr; ipheader.check = checksum((void *)&ipheader, sizeof(struct iphdr)); struct udphdr udpheader; udpheader.source = htons(1000); udpheader.dest = htons(80); udpheader.len = htons(40); udpheader.check = 0; memset(sendBuf, 0, sizeof(sendBuf)); memcpy(sendBuf, &ipheader, sizeof(struct iphdr)); memcpy(sendBuf + sizeof(struct iphdr), &udpheader, 8); } void *worker(void *arg) { int *a = (int *)arg; int index = *a; printf("the index is: %d\n", index); //显示是第几个线程 cpu_set_t mask; //CPU核的集合 CPU_ZERO(&mask); //置空 CPU_SET(index, &mask); //设置亲和力值 if (sched_setaffinity(0, sizeof(mask), &mask) == -1)//设置线程CPU亲和力 { printf("warning: could not set CPU affinity, continuing...\n"); } create_socket(index); while (1) { if (source == last_use[index]) continue; struct udphdr* udpheader = NULL; udpheader = (struct udphdr*)(sendBuf + sizeof(struct iphdr)); last_use[index] = source; udpheader->source = htons(source); udpheader->check = 0; udpheader->check = checksum((void *)udpheader, 40); printf("source port is %d on thread %d socket: %d\n", udpheader->source, index, sockfd[index]); int len= sendto(sockfd[index], sendBuf, sizeof(sendBuf), 0, (const struct sockaddr*)&sin, sizeof(struct sockaddr)); if (len > 0) printf("send success.\n"); else perror("send fail"); } } int main(void) { int i = 0; //创建多个线程 for (i = 0; i

下面编译程序，开始复现

//配置ens8 ip为2.2.2.2/24
root@node2:~#ifconfig ens8 2.2.2.2/24
//2.2.2.4是不存在的，所以需要配置静态arp
root@node2:~#arp -s 2.2.2.4 00:00:00:00:00:01

//编译
root@node2:~#gcc -o udp udp_client.c -lpthread

//程序执行前，查看计数
root@node2:~# conntrack -S
cpu=0           found=0 invalid=57309 ignore=1171 insert=0 insert_failed=2636 drop=1730 early_drop=0 error=0 search_restart=3215
cpu=1           found=0 invalid=53922 ignore=1368 insert=0 insert_failed=3344 drop=2464 early_drop=0 error=0 search_restart=3048
cpu=2           found=0 invalid=61931 ignore=1591 insert=0 insert_failed=2573 drop=2035 early_drop=0 error=0 search_restart=2669
cpu=3           found=0 invalid=59014 ignore=383 insert=0 insert_failed=1124 drop=571 early_drop=0 error=0 search_restart=1790

//执行程序一段时间，比如一分钟
root@node2:~#./udp

//再次查看计数，可看到insert_failed已经在涨了
root@node2:~# conntrack -S
cpu=0           found=0 invalid=57309 ignore=1171 insert=0 insert_failed=2637 drop=1730 early_drop=0 error=0 search_restart=3215
cpu=1           found=0 invalid=53922 ignore=1368 insert=0 insert_failed=3345 drop=2464 early_drop=0 error=0 search_restart=3048
cpu=2           found=0 invalid=61931 ignore=1591 insert=0 insert_failed=2573 drop=2035 early_drop=0 error=0 search_restart=2669
cpu=3           found=0 invalid=59014 ignore=383 insert=0 insert_failed=1126 drop=571 early_drop=0 error=0 search_restart=1790

参考

此问题的复现方法和fix时的log
https://github.com/brb/conntrack-race

腾讯云-k8s网络疑难杂症排查分享
https://segmentfault.com/a/1190000020069368?utm_source=tag-newest

也可参考：k8s之udp丢包问题 - 简书 (jianshu.com) 

本文《k8s之udp丢包问题》版权归分享放大价值所有，引用k8s之udp丢包问题需遵循CC 4.0 BY-SA版权协议。