FD.io VPP  v19.08-27-gf4dcae4
Vector Packet Processing
tcp_output.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2019 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vnet/tcp/tcp.h>
17 #include <math.h>
18 
19 typedef enum _tcp_output_next
20 {
27 
28 #define foreach_tcp4_output_next \
29  _ (DROP, "error-drop") \
30  _ (IP_LOOKUP, "ip4-lookup") \
31  _ (IP_REWRITE, "ip4-rewrite") \
32  _ (IP_ARP, "ip4-arp")
33 
34 #define foreach_tcp6_output_next \
35  _ (DROP, "error-drop") \
36  _ (IP_LOOKUP, "ip6-lookup") \
37  _ (IP_REWRITE, "ip6-rewrite") \
38  _ (IP_ARP, "ip6-discover-neighbor")
39 
40 static char *tcp_error_strings[] = {
41 #define tcp_error(n,s) s,
42 #include <vnet/tcp/tcp_error.def>
43 #undef tcp_error
44 };
45 
46 typedef struct
47 {
51 
52 static u8 *
53 format_tcp_tx_trace (u8 * s, va_list * args)
54 {
55  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
56  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
57  tcp_tx_trace_t *t = va_arg (*args, tcp_tx_trace_t *);
58  u32 indent = format_get_indent (s);
59 
60  s = format (s, "%U\n%U%U",
61  format_tcp_header, &t->tcp_header, 128,
62  format_white_space, indent,
64 
65  return s;
66 }
67 
68 #ifndef CLIB_MARCH_VARIANT
69 static u8
71 {
72  u8 wnd_scale = 0;
73  while (wnd_scale < TCP_MAX_WND_SCALE && (window >> wnd_scale) > TCP_WND_MAX)
74  wnd_scale++;
75  return wnd_scale;
76 }
77 
78 /**
79  * Update max segment size we're able to process.
80  *
81  * The value is constrained by our interface's MTU and IP options. It is
82  * also what we advertise to our peer.
83  */
84 void
86 {
87  /* TODO find our iface MTU */
88  tc->mss = tcp_cfg.default_mtu - sizeof (tcp_header_t);
89 }
90 
91 /**
92  * TCP's initial window
93  */
96 {
97  /* RFC 6928 recommends the value lower. However at the time our connections
98  * are initialized, fifos may not be allocated. Therefore, advertise the
99  * smallest possible unscaled window size and update once fifos are
100  * assigned to the session.
101  */
102  /*
103  tcp_update_rcv_mss (tc);
104  TCP_IW_N_SEGMENTS * tc->mss;
105  */
106  return tcp_cfg.min_rx_fifo;
107 }
108 
109 /**
110  * Compute initial window and scale factor. As per RFC1323, window field in
111  * SYN and SYN-ACK segments is never scaled.
112  */
113 u32
115 {
116  /* Compute rcv wscale only if peer advertised support for it */
117  if (tc->state != TCP_STATE_SYN_RCVD || tcp_opts_wscale (&tc->rcv_opts))
118  tc->rcv_wscale = tcp_window_compute_scale (tcp_cfg.max_rx_fifo);
119 
120  tc->rcv_wnd = tcp_initial_wnd_unscaled (tc);
121 
122  return clib_min (tc->rcv_wnd, TCP_WND_MAX);
123 }
124 
125 static inline void
127 {
128  u32 available_space, wnd;
129  i32 observed_wnd;
130 
131  ASSERT (tc->rcv_opts.mss < transport_rx_fifo_size (&tc->connection));
132 
133  /*
134  * Figure out how much space we have available
135  */
136  available_space = transport_max_rx_enqueue (&tc->connection);
137  if (PREDICT_FALSE (available_space < tc->rcv_opts.mss))
138  available_space = 0;
139 
140  /*
141  * Use the above and what we know about what we've previously advertised
142  * to compute the new window
143  */
144  observed_wnd = (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las);
145 
146  /* Bad. Thou shalt not shrink */
147  if (PREDICT_FALSE ((i32) available_space < observed_wnd))
148  {
149  wnd = clib_max (observed_wnd, 0);
150  TCP_EVT_DBG (TCP_EVT_RCV_WND_SHRUNK, tc, observed_wnd, available_space);
151  }
152  else
153  {
154  wnd = available_space;
155  }
156 
157  /* Make sure we have a multiple of rcv_wscale */
158  if (wnd && tc->rcv_wscale)
159  {
160  wnd &= ~((1 << tc->rcv_wscale) - 1);
161  if (wnd == 0)
162  wnd = 1 << tc->rcv_wscale;
163  }
164 
165  tc->rcv_wnd = clib_min (wnd, TCP_WND_MAX << tc->rcv_wscale);
166 }
167 
168 /**
169  * Compute and return window to advertise, scaled as per RFC1323
170  */
171 static inline u32
173 {
174  if (state < TCP_STATE_ESTABLISHED)
176 
177  tcp_update_rcv_wnd (tc);
178  return tc->rcv_wnd >> tc->rcv_wscale;
179 }
180 
181 /**
182  * Write TCP options to segment.
183  */
184 static u32
186 {
187  u32 opts_len = 0;
188  u32 buf, seq_len = 4;
189 
190  if (tcp_opts_mss (opts))
191  {
192  *data++ = TCP_OPTION_MSS;
193  *data++ = TCP_OPTION_LEN_MSS;
194  buf = clib_host_to_net_u16 (opts->mss);
195  clib_memcpy_fast (data, &buf, sizeof (opts->mss));
196  data += sizeof (opts->mss);
197  opts_len += TCP_OPTION_LEN_MSS;
198  }
199 
200  if (tcp_opts_wscale (opts))
201  {
202  *data++ = TCP_OPTION_WINDOW_SCALE;
203  *data++ = TCP_OPTION_LEN_WINDOW_SCALE;
204  *data++ = opts->wscale;
205  opts_len += TCP_OPTION_LEN_WINDOW_SCALE;
206  }
207 
208  if (tcp_opts_sack_permitted (opts))
209  {
210  *data++ = TCP_OPTION_SACK_PERMITTED;
212  opts_len += TCP_OPTION_LEN_SACK_PERMITTED;
213  }
214 
215  if (tcp_opts_tstamp (opts))
216  {
217  *data++ = TCP_OPTION_TIMESTAMP;
218  *data++ = TCP_OPTION_LEN_TIMESTAMP;
219  buf = clib_host_to_net_u32 (opts->tsval);
220  clib_memcpy_fast (data, &buf, sizeof (opts->tsval));
221  data += sizeof (opts->tsval);
222  buf = clib_host_to_net_u32 (opts->tsecr);
223  clib_memcpy_fast (data, &buf, sizeof (opts->tsecr));
224  data += sizeof (opts->tsecr);
225  opts_len += TCP_OPTION_LEN_TIMESTAMP;
226  }
227 
228  if (tcp_opts_sack (opts))
229  {
230  int i;
231 
232  if (opts->n_sack_blocks != 0)
233  {
234  *data++ = TCP_OPTION_SACK_BLOCK;
235  *data++ = 2 + opts->n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
236  for (i = 0; i < opts->n_sack_blocks; i++)
237  {
238  buf = clib_host_to_net_u32 (opts->sacks[i].start);
239  clib_memcpy_fast (data, &buf, seq_len);
240  data += seq_len;
241  buf = clib_host_to_net_u32 (opts->sacks[i].end);
242  clib_memcpy_fast (data, &buf, seq_len);
243  data += seq_len;
244  }
245  opts_len += 2 + opts->n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
246  }
247  }
248 
249  /* Terminate TCP options */
250  if (opts_len % 4)
251  {
252  *data++ = TCP_OPTION_EOL;
253  opts_len += TCP_OPTION_LEN_EOL;
254  }
255 
256  /* Pad with zeroes to a u32 boundary */
257  while (opts_len % 4)
258  {
259  *data++ = TCP_OPTION_NOOP;
260  opts_len += TCP_OPTION_LEN_NOOP;
261  }
262  return opts_len;
263 }
264 
265 static int
267 {
268  u8 len = 0;
269 
270  opts->flags |= TCP_OPTS_FLAG_MSS;
271  opts->mss = tcp_cfg.default_mtu; /*XXX discover that */
272  len += TCP_OPTION_LEN_MSS;
273 
274  opts->flags |= TCP_OPTS_FLAG_WSCALE;
275  opts->wscale = wnd_scale;
277 
278  opts->flags |= TCP_OPTS_FLAG_TSTAMP;
279  opts->tsval = tcp_time_now ();
280  opts->tsecr = 0;
282 
283  if (TCP_USE_SACKS)
284  {
285  opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
287  }
288 
289  /* Align to needed boundary */
290  len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
291  return len;
292 }
293 
294 static int
296 {
297  u8 len = 0;
298 
299  opts->flags |= TCP_OPTS_FLAG_MSS;
300  opts->mss = tc->mss;
301  len += TCP_OPTION_LEN_MSS;
302 
303  if (tcp_opts_wscale (&tc->rcv_opts))
304  {
305  opts->flags |= TCP_OPTS_FLAG_WSCALE;
306  opts->wscale = tc->rcv_wscale;
308  }
309 
310  if (tcp_opts_tstamp (&tc->rcv_opts))
311  {
312  opts->flags |= TCP_OPTS_FLAG_TSTAMP;
313  opts->tsval = tcp_time_now ();
314  opts->tsecr = tc->tsval_recent;
316  }
317 
318  if (tcp_opts_sack_permitted (&tc->rcv_opts))
319  {
320  opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
322  }
323 
324  /* Align to needed boundary */
325  len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
326  return len;
327 }
328 
329 static int
331 {
332  u8 len = 0;
333 
334  opts->flags = 0;
335 
336  if (tcp_opts_tstamp (&tc->rcv_opts))
337  {
338  opts->flags |= TCP_OPTS_FLAG_TSTAMP;
339  opts->tsval = tcp_tstamp (tc);
340  opts->tsecr = tc->tsval_recent;
342  }
343  if (tcp_opts_sack_permitted (&tc->rcv_opts))
344  {
345  if (vec_len (tc->snd_sacks))
346  {
347  opts->flags |= TCP_OPTS_FLAG_SACK;
348  if (tc->snd_sack_pos >= vec_len (tc->snd_sacks))
349  tc->snd_sack_pos = 0;
350  opts->sacks = &tc->snd_sacks[tc->snd_sack_pos];
351  opts->n_sack_blocks = vec_len (tc->snd_sacks) - tc->snd_sack_pos;
352  opts->n_sack_blocks = clib_min (opts->n_sack_blocks,
354  tc->snd_sack_pos += opts->n_sack_blocks;
355  len += 2 + TCP_OPTION_LEN_SACK_BLOCK * opts->n_sack_blocks;
356  }
357  }
358 
359  /* Align to needed boundary */
360  len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
361  return len;
362 }
363 
364 always_inline int
367 {
368  switch (state)
369  {
370  case TCP_STATE_ESTABLISHED:
371  case TCP_STATE_CLOSE_WAIT:
372  case TCP_STATE_FIN_WAIT_1:
373  case TCP_STATE_LAST_ACK:
374  case TCP_STATE_CLOSING:
375  case TCP_STATE_FIN_WAIT_2:
376  case TCP_STATE_TIME_WAIT:
377  case TCP_STATE_CLOSED:
378  return tcp_make_established_options (tc, opts);
379  case TCP_STATE_SYN_RCVD:
380  return tcp_make_synack_options (tc, opts);
381  case TCP_STATE_SYN_SENT:
382  return tcp_make_syn_options (opts, tc->rcv_wscale);
383  default:
384  clib_warning ("State not handled! %d", state);
385  return 0;
386  }
387 }
388 
389 /**
390  * Update burst send vars
391  *
392  * - Updates snd_mss to reflect the effective segment size that we can send
393  * by taking into account all TCP options, including SACKs.
394  * - Cache 'on the wire' options for reuse
395  * - Updates receive window which can be reused for a burst.
396  *
397  * This should *only* be called when doing bursts
398  */
399 void
401 {
402  tcp_main_t *tm = &tcp_main;
403 
404  /* Compute options to be used for connection. These may be reused when
405  * sending data or to compute the effective mss (snd_mss) */
406  tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts,
407  TCP_STATE_ESTABLISHED);
408 
409  /* XXX check if MTU has been updated */
410  tc->snd_mss = clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
411  ASSERT (tc->snd_mss > 0);
412 
413  tcp_options_write (tm->wrk_ctx[tc->c_thread_index].cached_opts,
414  &tc->snd_opts);
415 
416  tcp_update_rcv_wnd (tc);
417 
418  if (tc->flags & TCP_CONN_RATE_SAMPLE)
419  tc->flags |= TCP_CONN_TRACK_BURST;
420 
421  if (tc->snd_una == tc->snd_nxt)
423 }
424 
425 void
427 {
428  u16 default_min_mss = 536;
429  tcp_update_rcv_mss (tc);
430 
431  /* TODO cache mss and consider PMTU discovery */
432  tc->snd_mss = clib_min (tc->rcv_opts.mss, tc->mss);
433 
434  if (tc->snd_mss < 45)
435  {
436  /* Assume that at least the min default mss works */
437  tc->snd_mss = default_min_mss;
438  tc->rcv_opts.mss = default_min_mss;
439  }
440 
441  /* We should have enough space for 40 bytes of options */
442  ASSERT (tc->snd_mss > 45);
443 
444  /* If we use timestamp option, account for it */
445  if (tcp_opts_tstamp (&tc->rcv_opts))
446  tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP;
447 }
448 #endif /* CLIB_MARCH_VARIANT */
449 
450 static void *
452 {
453  if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
455  /* Zero all flags but free list index and trace flag */
456  b->flags &= VLIB_BUFFER_NEXT_PRESENT - 1;
457  b->current_data = 0;
458  b->current_length = 0;
460  vnet_buffer (b)->tcp.flags = 0;
461 
462  /* Leave enough space for headers */
464 }
465 
466 #ifndef CLIB_MARCH_VARIANT
467 static void *
469 {
470  ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
471  b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
473  b->current_data = 0;
474  vnet_buffer (b)->tcp.flags = 0;
476  /* Leave enough space for headers */
478 }
479 
480 /**
481  * Prepare ACK
482  */
483 static inline void
485  u8 flags)
486 {
487  tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
488  u8 tcp_opts_len, tcp_hdr_opts_len;
489  tcp_header_t *th;
490  u16 wnd;
491 
492  wnd = tcp_window_to_advertise (tc, state);
493 
494  /* Make and write options */
495  tcp_opts_len = tcp_make_established_options (tc, snd_opts);
496  tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
497 
498  th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
499  tc->rcv_nxt, tcp_hdr_opts_len, flags, wnd);
500 
501  tcp_options_write ((u8 *) (th + 1), snd_opts);
502  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
503 
504  if (wnd == 0)
506  else
508 }
509 
510 /**
511  * Convert buffer to ACK
512  */
513 static inline void
515 {
516  tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_ACK);
517  TCP_EVT_DBG (TCP_EVT_ACK_SENT, tc);
518  tc->rcv_las = tc->rcv_nxt;
519 }
520 
521 /**
522  * Convert buffer to FIN-ACK
523  */
524 void
526 {
527  tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_ACK);
528 }
529 
530 /**
531  * Convert buffer to SYN
532  */
533 void
535 {
536  u8 tcp_hdr_opts_len, tcp_opts_len;
537  tcp_header_t *th;
538  u16 initial_wnd;
539  tcp_options_t snd_opts;
540 
541  initial_wnd = tcp_initial_window_to_advertise (tc);
542 
543  /* Make and write options */
544  clib_memset (&snd_opts, 0, sizeof (snd_opts));
545  tcp_opts_len = tcp_make_syn_options (&snd_opts, tc->rcv_wscale);
546  tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
547 
548  th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
549  tc->rcv_nxt, tcp_hdr_opts_len, TCP_FLAG_SYN,
550  initial_wnd);
551  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
552  tcp_options_write ((u8 *) (th + 1), &snd_opts);
553 }
554 
555 /**
556  * Convert buffer to SYN-ACK
557  */
558 void
560 {
561  tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
562  u8 tcp_opts_len, tcp_hdr_opts_len;
563  tcp_header_t *th;
564  u16 initial_wnd;
565 
566  clib_memset (snd_opts, 0, sizeof (*snd_opts));
567  initial_wnd = tcp_initial_window_to_advertise (tc);
568  tcp_opts_len = tcp_make_synack_options (tc, snd_opts);
569  tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
570 
571  th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
572  tc->rcv_nxt, tcp_hdr_opts_len,
573  TCP_FLAG_SYN | TCP_FLAG_ACK, initial_wnd);
574  tcp_options_write ((u8 *) (th + 1), snd_opts);
575 
576  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
577 }
578 
579 always_inline void
581  u8 is_ip4, u32 fib_index, u8 flush)
582 {
583  vlib_main_t *vm = wrk->vm;
584  u32 *to_next, next_index;
585  vlib_frame_t *f;
586 
587  b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
588  b->error = 0;
589 
590  vnet_buffer (b)->sw_if_index[VLIB_TX] = fib_index;
591  vnet_buffer (b)->sw_if_index[VLIB_RX] = 0;
592 
593  /* Send to IP lookup */
594  next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
596 
597  f = wrk->ip_lookup_tx_frames[!is_ip4];
598  if (!f)
599  {
600  f = vlib_get_frame_to_node (vm, next_index);
601  ASSERT (f);
602  wrk->ip_lookup_tx_frames[!is_ip4] = f;
603  }
604 
605  to_next = vlib_frame_vector_args (f);
606  to_next[f->n_vectors] = bi;
607  f->n_vectors += 1;
608  if (flush || f->n_vectors == VLIB_FRAME_SIZE)
609  {
610  vlib_put_frame_to_node (vm, next_index, f);
611  wrk->ip_lookup_tx_frames[!is_ip4] = 0;
612  }
613 }
614 
615 static void
617  u32 bi, u8 is_ip4, u32 fib_index)
618 {
619  tcp_enqueue_to_ip_lookup_i (wrk, b, bi, is_ip4, fib_index, 1);
620 }
621 
622 static void
624  u8 is_ip4, u32 fib_index)
625 {
626  tcp_enqueue_to_ip_lookup_i (wrk, b, bi, is_ip4, fib_index, 0);
627  if (wrk->vm->thread_index == 0 && vlib_num_workers ())
629 }
630 
631 always_inline void
633  u8 is_ip4, u8 flush)
634 {
635  u32 *to_next, next_index;
636  vlib_frame_t *f;
637 
638  b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
639  b->error = 0;
640 
641  /* Decide where to send the packet */
642  next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
644 
645  /* Get frame to v4/6 output node */
646  f = wrk->tx_frames[!is_ip4];
647  if (!f)
648  {
649  f = vlib_get_frame_to_node (wrk->vm, next_index);
650  ASSERT (f);
651  wrk->tx_frames[!is_ip4] = f;
652  }
653  to_next = vlib_frame_vector_args (f);
654  to_next[f->n_vectors] = bi;
655  f->n_vectors += 1;
656  if (flush || f->n_vectors == VLIB_FRAME_SIZE)
657  {
658  vlib_put_frame_to_node (wrk->vm, next_index, f);
659  wrk->tx_frames[!is_ip4] = 0;
660  }
661 }
662 
663 static void
665  u8 is_ip4)
666 {
667  tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 0);
668 }
669 
670 static void
672  u8 is_ip4)
673 {
674  tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 1);
675 }
676 #endif /* CLIB_MARCH_VARIANT */
677 
678 static int
680  tcp_state_t state, u8 thread_index, u8 is_ip4)
681 {
682  ip4_header_t *ih4;
683  ip6_header_t *ih6;
684  tcp_header_t *th0;
685  ip4_address_t src_ip40, dst_ip40;
686  ip6_address_t src_ip60, dst_ip60;
688  u32 tmp;
689  u32 seq, ack;
690  u8 flags;
691 
692  /* Find IP and TCP headers */
693  th0 = tcp_buffer_hdr (b0);
694 
695  /* Save src and dst ip */
696  if (is_ip4)
697  {
698  ih4 = vlib_buffer_get_current (b0);
699  ASSERT ((ih4->ip_version_and_header_length & 0xF0) == 0x40);
700  src_ip40.as_u32 = ih4->src_address.as_u32;
701  dst_ip40.as_u32 = ih4->dst_address.as_u32;
702  }
703  else
704  {
705  ih6 = vlib_buffer_get_current (b0);
706  ASSERT ((ih6->ip_version_traffic_class_and_flow_label & 0xF0) == 0x60);
707  clib_memcpy_fast (&src_ip60, &ih6->src_address, sizeof (ip6_address_t));
708  clib_memcpy_fast (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t));
709  }
710 
711  src_port = th0->src_port;
712  dst_port = th0->dst_port;
713 
714  /* Try to determine what/why we're actually resetting */
715  if (state == TCP_STATE_CLOSED)
716  {
717  if (!tcp_syn (th0))
718  return -1;
719 
720  tmp = clib_net_to_host_u32 (th0->seq_number);
721 
722  /* Got a SYN for no listener. */
723  flags = TCP_FLAG_RST | TCP_FLAG_ACK;
724  ack = clib_host_to_net_u32 (tmp + 1);
725  seq = 0;
726  }
727  else
728  {
729  flags = TCP_FLAG_RST;
730  seq = th0->ack_number;
731  ack = 0;
732  }
733 
734  tcp_reuse_buffer (vm, b0);
735  tcp_trajectory_add_start (b0, 4);
736  th0 = vlib_buffer_push_tcp_net_order (b0, dst_port, src_port, seq, ack,
737  sizeof (tcp_header_t), flags, 0);
738 
739  if (is_ip4)
740  {
741  ih4 = vlib_buffer_push_ip4 (vm, b0, &dst_ip40, &src_ip40,
742  IP_PROTOCOL_TCP, 1);
743  th0->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ih4);
744  }
745  else
746  {
747  int bogus = ~0;
748  ih6 = vlib_buffer_push_ip6 (vm, b0, &dst_ip60, &src_ip60,
749  IP_PROTOCOL_TCP);
750  th0->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ih6, &bogus);
751  ASSERT (!bogus);
752  }
753 
754  return 0;
755 }
756 
757 #ifndef CLIB_MARCH_VARIANT
758 /**
759  * Send reset without reusing existing buffer
760  *
761  * It extracts connection info out of original packet
762  */
763 void
765  u32 thread_index, u8 is_ip4)
766 {
767  tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
768  vlib_main_t *vm = wrk->vm;
769  vlib_buffer_t *b;
770  u32 bi, sw_if_index, fib_index;
771  u8 tcp_hdr_len, flags = 0;
772  tcp_header_t *th, *pkt_th;
773  u32 seq, ack;
774  ip4_header_t *ih4, *pkt_ih4;
775  ip6_header_t *ih6, *pkt_ih6;
776  fib_protocol_t fib_proto;
777 
778  if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
779  return;
780 
781  b = vlib_get_buffer (vm, bi);
782  sw_if_index = vnet_buffer (pkt)->sw_if_index[VLIB_RX];
783  fib_proto = is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6;
784  fib_index = fib_table_get_index_for_sw_if_index (fib_proto, sw_if_index);
785  tcp_init_buffer (vm, b);
786 
787  /* Make and write options */
788  tcp_hdr_len = sizeof (tcp_header_t);
789 
790  if (is_ip4)
791  {
792  pkt_ih4 = vlib_buffer_get_current (pkt);
793  pkt_th = ip4_next_header (pkt_ih4);
794  }
795  else
796  {
797  pkt_ih6 = vlib_buffer_get_current (pkt);
798  pkt_th = ip6_next_header (pkt_ih6);
799  }
800 
801  if (tcp_ack (pkt_th))
802  {
803  flags = TCP_FLAG_RST;
804  seq = pkt_th->ack_number;
805  ack = (tc->state >= TCP_STATE_SYN_RCVD) ? tc->rcv_nxt : 0;
806  }
807  else
808  {
809  flags = TCP_FLAG_RST | TCP_FLAG_ACK;
810  seq = 0;
811  ack = clib_host_to_net_u32 (vnet_buffer (pkt)->tcp.seq_end);
812  }
813 
814  th = vlib_buffer_push_tcp_net_order (b, pkt_th->dst_port, pkt_th->src_port,
815  seq, ack, tcp_hdr_len, flags, 0);
816 
817  /* Swap src and dst ip */
818  if (is_ip4)
819  {
820  ASSERT ((pkt_ih4->ip_version_and_header_length & 0xF0) == 0x40);
821  ih4 = vlib_buffer_push_ip4 (vm, b, &pkt_ih4->dst_address,
822  &pkt_ih4->src_address, IP_PROTOCOL_TCP, 1);
823  th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
824  }
825  else
826  {
827  int bogus = ~0;
828  ASSERT ((pkt_ih6->ip_version_traffic_class_and_flow_label & 0xF0) ==
829  0x60);
830  ih6 = vlib_buffer_push_ip6 (vm, b, &pkt_ih6->dst_address,
831  &pkt_ih6->src_address, IP_PROTOCOL_TCP);
832  th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
833  ASSERT (!bogus);
834  }
835 
836  tcp_enqueue_to_ip_lookup_now (wrk, b, bi, is_ip4, fib_index);
837  TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
838  vlib_node_increment_counter (vm, tcp_node_index (output, tc->c_is_ip4),
839  TCP_ERROR_RST_SENT, 1);
840 }
841 
842 /**
843  * Build and set reset packet for connection
844  */
845 void
847 {
848  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
849  vlib_main_t *vm = wrk->vm;
850  vlib_buffer_t *b;
851  u32 bi;
852  tcp_header_t *th;
853  u16 tcp_hdr_opts_len, advertise_wnd, opts_write_len;
854  u8 flags;
855 
856  if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
857  return;
858  b = vlib_get_buffer (vm, bi);
859  tcp_init_buffer (vm, b);
860 
861  tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
862  tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
863  advertise_wnd = tcp_window_to_advertise (tc, TCP_STATE_ESTABLISHED);
864  flags = TCP_FLAG_RST;
865  th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
866  tc->rcv_nxt, tcp_hdr_opts_len, flags,
867  advertise_wnd);
868  opts_write_len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
869  ASSERT (opts_write_len == tc->snd_opts_len);
870  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
871  if (tc->c_is_ip4)
872  {
873  ip4_header_t *ih4;
874  ih4 = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip.ip4,
875  &tc->c_rmt_ip.ip4, IP_PROTOCOL_TCP, 0);
876  th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
877  }
878  else
879  {
880  int bogus = ~0;
881  ip6_header_t *ih6;
882  ih6 = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip.ip6,
883  &tc->c_rmt_ip.ip6, IP_PROTOCOL_TCP);
884  th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
885  ASSERT (!bogus);
886  }
887  tcp_enqueue_to_ip_lookup_now (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
888  TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
889  vlib_node_increment_counter (vm, tcp_node_index (output, tc->c_is_ip4),
890  TCP_ERROR_RST_SENT, 1);
891 }
892 
893 static void
895  vlib_buffer_t * b)
896 {
898  vlib_main_t *vm = wrk->vm;
899  if (tc->c_is_ip4)
900  {
901  ip4_header_t *ih;
902  ih = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip4,
903  &tc->c_rmt_ip4, IP_PROTOCOL_TCP, 1);
904  th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih);
905  }
906  else
907  {
908  ip6_header_t *ih;
909  int bogus = ~0;
910 
911  ih = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip6,
912  &tc->c_rmt_ip6, IP_PROTOCOL_TCP);
913  th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih, &bogus);
914  ASSERT (!bogus);
915  }
916 }
917 
918 /**
919  * Send SYN
920  *
921  * Builds a SYN packet for a half-open connection and sends it to ipx_lookup.
922  * The packet is not forwarded through tcpx_output to avoid doing lookups
923  * in the half_open pool.
924  */
925 void
927 {
928  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
929  vlib_main_t *vm = wrk->vm;
930  vlib_buffer_t *b;
931  u32 bi;
932 
933  /*
934  * Setup retransmit and establish timers before requesting buffer
935  * such that we can return if we've ran out.
936  */
937  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
938  tc->rto * TCP_TO_TIMER_TICK);
939 
940  if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
941  {
942  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1);
943  return;
944  }
945 
946  b = vlib_get_buffer (vm, bi);
947  tcp_init_buffer (vm, b);
948  tcp_make_syn (tc, b);
949 
950  /* Measure RTT with this */
951  tc->rtt_ts = tcp_time_now_us (vlib_num_workers ()? 1 : 0);
952  tc->rtt_seq = tc->snd_nxt;
953  tc->rto_boff = 0;
954 
955  tcp_push_ip_hdr (wrk, tc, b);
956  tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
957  TCP_EVT_DBG (TCP_EVT_SYN_SENT, tc);
958 }
959 
960 void
962 {
963  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
964  vlib_main_t *vm = wrk->vm;
965  vlib_buffer_t *b;
966  u32 bi;
967 
969 
970  if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
971  {
972  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
973  return;
974  }
975 
976  tc->rtt_ts = tcp_time_now_us (tc->c_thread_index);
977  b = vlib_get_buffer (vm, bi);
978  tcp_init_buffer (vm, b);
979  tcp_make_synack (tc, b);
980  tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
981  TCP_EVT_DBG (TCP_EVT_SYNACK_SENT, tc);
982 }
983 
984 /**
985  * Flush tx frame populated by retransmits and timer pops
986  */
987 void
989 {
990  if (wrk->tx_frames[!is_ip4])
991  {
992  u32 next_index;
993  next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
994  vlib_put_frame_to_node (wrk->vm, next_index, wrk->tx_frames[!is_ip4]);
995  wrk->tx_frames[!is_ip4] = 0;
996  }
997 }
998 
999 /**
1000  * Flush ip lookup tx frames populated by timer pops
1001  */
1002 static void
1004 {
1005  if (wrk->ip_lookup_tx_frames[!is_ip4])
1006  {
1007  u32 next_index;
1008  next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
1009  vlib_put_frame_to_node (wrk->vm, next_index,
1010  wrk->ip_lookup_tx_frames[!is_ip4]);
1011  wrk->ip_lookup_tx_frames[!is_ip4] = 0;
1012  }
1013 }
1014 
1015 /**
1016  * Flush v4 and v6 tcp and ip-lookup tx frames for thread index
1017  */
1018 void
1020 {
1021  tcp_flush_frame_to_output (wrk, 1);
1022  tcp_flush_frame_to_output (wrk, 0);
1025 }
1026 
1027 /**
1028  * Send FIN
1029  */
1030 void
1032 {
1033  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
1034  vlib_main_t *vm = wrk->vm;
1035  vlib_buffer_t *b;
1036  u32 bi;
1037  u8 fin_snt = 0;
1038 
1039  fin_snt = tc->flags & TCP_CONN_FINSNT;
1040  if (fin_snt)
1041  tc->snd_nxt -= 1;
1042 
1043  if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
1044  {
1045  /* Out of buffers so program fin retransmit ASAP */
1046  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
1047  if (fin_snt)
1048  tc->snd_nxt += 1;
1049  else
1050  /* Make sure retransmit retries a fin not data */
1051  tc->flags |= TCP_CONN_FINSNT;
1052  return;
1053  }
1054 
1056  b = vlib_get_buffer (vm, bi);
1057  tcp_init_buffer (vm, b);
1058  tcp_make_fin (tc, b);
1059  tcp_enqueue_to_output_now (wrk, b, bi, tc->c_is_ip4);
1060  TCP_EVT_DBG (TCP_EVT_FIN_SENT, tc);
1061  /* Account for the FIN */
1062  tc->snd_nxt += 1;
1063  if (!fin_snt)
1064  {
1065  tc->flags |= TCP_CONN_FINSNT;
1066  tc->flags &= ~TCP_CONN_FINPNDG;
1067  tc->snd_una_max = seq_max (tc->snd_una_max, tc->snd_nxt);
1068  }
1069 }
1070 
1071 /**
1072  * Push TCP header and update connection variables. Should only be called
1073  * for segments with data, not for 'control' packets.
1074  */
1075 always_inline void
1077  u8 compute_opts, u8 maybe_burst, u8 update_snd_nxt)
1078 {
1079  u8 tcp_hdr_opts_len, flags = TCP_FLAG_ACK;
1080  u32 advertise_wnd, data_len;
1081  tcp_main_t *tm = &tcp_main;
1082  tcp_header_t *th;
1083 
1084  data_len = b->current_length;
1085  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NEXT_PRESENT))
1087 
1088  vnet_buffer (b)->tcp.flags = 0;
1089  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
1090 
1091  if (compute_opts)
1092  tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1093 
1094  tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
1095 
1096  if (maybe_burst)
1097  advertise_wnd = tc->rcv_wnd >> tc->rcv_wscale;
1098  else
1099  advertise_wnd = tcp_window_to_advertise (tc, TCP_STATE_ESTABLISHED);
1100 
1101  if (PREDICT_FALSE (tc->flags & TCP_CONN_PSH_PENDING))
1102  {
1103  if (seq_geq (tc->psh_seq, snd_nxt)
1104  && seq_lt (tc->psh_seq, snd_nxt + data_len))
1105  flags |= TCP_FLAG_PSH;
1106  }
1107  th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, snd_nxt,
1108  tc->rcv_nxt, tcp_hdr_opts_len, flags,
1109  advertise_wnd);
1110 
1111  if (maybe_burst)
1112  {
1113  clib_memcpy_fast ((u8 *) (th + 1),
1114  tm->wrk_ctx[tc->c_thread_index].cached_opts,
1115  tc->snd_opts_len);
1116  }
1117  else
1118  {
1119  u8 len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
1120  ASSERT (len == tc->snd_opts_len);
1121  }
1122 
1123  /*
1124  * Update connection variables
1125  */
1126 
1127  if (update_snd_nxt)
1128  tc->snd_nxt += data_len;
1129  tc->rcv_las = tc->rcv_nxt;
1130 
1131  tc->bytes_out += data_len;
1132  tc->data_segs_out += 1;
1133 
1134  TCP_EVT_DBG (TCP_EVT_PKTIZE, tc);
1135 }
1136 
1137 u32
1139 {
1140  tcp_connection_t *tc = (tcp_connection_t *) tconn;
1141 
1142  if (tc->flags & TCP_CONN_TRACK_BURST)
1143  {
1145  tcp_bt_track_tx (tc);
1146  tc->flags &= ~TCP_CONN_TRACK_BURST;
1147  }
1148 
1149  tcp_push_hdr_i (tc, b, tc->snd_nxt, /* compute opts */ 0, /* burst */ 1,
1150  /* update_snd_nxt */ 1);
1151 
1152  tc->snd_una_max = seq_max (tc->snd_nxt, tc->snd_una_max);
1153  tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
1154  /* If not tracking an ACK, start tracking */
1155  if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
1156  {
1157  tc->rtt_ts = tcp_time_now_us (tc->c_thread_index);
1158  tc->rtt_seq = tc->snd_nxt;
1159  }
1160  if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
1161  {
1163  tc->rto_boff = 0;
1164  }
1165  tcp_trajectory_add_start (b, 3);
1166  return 0;
1167 }
1168 
1169 void
1171 {
1172  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
1173  vlib_main_t *vm = wrk->vm;
1174  vlib_buffer_t *b;
1175  u32 bi;
1176 
1177  if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
1178  {
1179  tcp_update_rcv_wnd (tc);
1180  return;
1181  }
1182  b = vlib_get_buffer (vm, bi);
1183  tcp_init_buffer (vm, b);
1184  tcp_make_ack (tc, b);
1185  tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1186 }
1187 
1188 void
1190 {
1191  if (!(tc->flags & TCP_CONN_SNDACK))
1192  {
1193  session_add_self_custom_tx_evt (&tc->connection, 1);
1194  tc->flags |= TCP_CONN_SNDACK;
1195  }
1196 }
1197 
1198 void
1200 {
1201  if (!(tc->flags & TCP_CONN_SNDACK))
1202  {
1203  session_add_self_custom_tx_evt (&tc->connection, 1);
1204  tc->flags |= TCP_CONN_SNDACK;
1205  }
1206  if (tc->pending_dupacks < 255)
1207  tc->pending_dupacks += 1;
1208 }
1209 
1210 void
1212 {
1213  if (!(tc->flags & TCP_CONN_FRXT_PENDING))
1214  {
1215  session_add_self_custom_tx_evt (&tc->connection, 0);
1216  tc->flags |= TCP_CONN_FRXT_PENDING;
1217  }
1218 }
1219 
1220 /**
1221  * Delayed ack timer handler
1222  *
1223  * Sends delayed ACK when timer expires
1224  */
1225 void
1227 {
1228  u32 thread_index = vlib_get_thread_index ();
1229  tcp_connection_t *tc;
1230 
1231  tc = tcp_connection_get (index, thread_index);
1232  tc->timers[TCP_TIMER_DELACK] = TCP_TIMER_HANDLE_INVALID;
1233  tcp_send_ack (tc);
1234 }
1235 
1236 /**
1237  * Send Window Update ACK,
1238  * ensuring that it will be sent once, if RWND became non-zero,
1239  * after zero RWND has been advertised in ACK before
1240  */
1241 void
1243 {
1244  u32 win;
1245 
1246  if (tcp_zero_rwnd_sent (tc))
1247  {
1248  win = tcp_window_to_advertise (tc, tc->state);
1249  if (win > 0)
1250  {
1252  tcp_program_ack (tc);
1253  }
1254  }
1255 }
1256 
1257 /**
1258  * Allocate a new buffer and build a new tcp segment
1259  *
1260  * @param wrk tcp worker
1261  * @param tc connection for which the segment will be allocated
1262  * @param offset offset of the first byte in the tx fifo
1263  * @param max_deq_byte segment size
1264  * @param[out] b pointer to buffer allocated
1265  *
1266  * @return the number of bytes in the segment or 0 if buffer cannot be
1267  * allocated or no data available
1268  */
1269 static int
1271  u32 offset, u32 max_deq_bytes, vlib_buffer_t ** b)
1272 {
1273  u32 bytes_per_buffer = vnet_get_tcp_main ()->bytes_per_buffer;
1274  vlib_main_t *vm = wrk->vm;
1275  u32 bi, seg_size;
1276  int n_bytes = 0;
1277  u8 *data;
1278 
1279  seg_size = max_deq_bytes + TRANSPORT_MAX_HDRS_LEN;
1280 
1281  /*
1282  * Prepare options
1283  */
1284  tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1285 
1286  /*
1287  * Allocate and fill in buffer(s)
1288  */
1289 
1290  /* Easy case, buffer size greater than mss */
1291  if (PREDICT_TRUE (seg_size <= bytes_per_buffer))
1292  {
1293  if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
1294  return 0;
1295  *b = vlib_get_buffer (vm, bi);
1296  data = tcp_init_buffer (vm, *b);
1297  n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset,
1298  max_deq_bytes);
1299  ASSERT (n_bytes == max_deq_bytes);
1300  b[0]->current_length = n_bytes;
1301  tcp_push_hdr_i (tc, *b, tc->snd_una + offset, /* compute opts */ 0,
1302  /* burst */ 0, /* update_snd_nxt */ 0);
1303  }
1304  /* Split mss into multiple buffers */
1305  else
1306  {
1307  u32 chain_bi = ~0, n_bufs_per_seg, n_bufs;
1308  u16 n_peeked, len_to_deq;
1309  vlib_buffer_t *chain_b, *prev_b;
1310  int i;
1311 
1312  /* Make sure we have enough buffers */
1313  n_bufs_per_seg = ceil ((double) seg_size / bytes_per_buffer);
1314  vec_validate_aligned (wrk->tx_buffers, n_bufs_per_seg - 1,
1316  n_bufs = vlib_buffer_alloc (vm, wrk->tx_buffers, n_bufs_per_seg);
1317  if (PREDICT_FALSE (n_bufs != n_bufs_per_seg))
1318  {
1319  if (n_bufs)
1320  vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
1321  return 0;
1322  }
1323 
1324  *b = vlib_get_buffer (vm, wrk->tx_buffers[--n_bufs]);
1325  data = tcp_init_buffer (vm, *b);
1326  n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset,
1327  bytes_per_buffer -
1328  TRANSPORT_MAX_HDRS_LEN);
1329  b[0]->current_length = n_bytes;
1330  b[0]->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1332  max_deq_bytes -= n_bytes;
1333 
1334  chain_b = *b;
1335  for (i = 1; i < n_bufs_per_seg; i++)
1336  {
1337  prev_b = chain_b;
1338  len_to_deq = clib_min (max_deq_bytes, bytes_per_buffer);
1339  chain_bi = wrk->tx_buffers[--n_bufs];
1340  chain_b = vlib_get_buffer (vm, chain_bi);
1341  chain_b->current_data = 0;
1342  data = vlib_buffer_get_current (chain_b);
1343  n_peeked = session_tx_fifo_peek_bytes (&tc->connection, data,
1344  offset + n_bytes,
1345  len_to_deq);
1346  ASSERT (n_peeked == len_to_deq);
1347  n_bytes += n_peeked;
1348  chain_b->current_length = n_peeked;
1349  chain_b->next_buffer = 0;
1350 
1351  /* update previous buffer */
1352  prev_b->next_buffer = chain_bi;
1353  prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
1354 
1355  max_deq_bytes -= n_peeked;
1356  b[0]->total_length_not_including_first_buffer += n_peeked;
1357  }
1358 
1359  tcp_push_hdr_i (tc, *b, tc->snd_una + offset, /* compute opts */ 0,
1360  /* burst */ 0, /* update_snd_nxt */ 0);
1361 
1362  if (PREDICT_FALSE (n_bufs))
1363  {
1364  clib_warning ("not all buffers consumed");
1365  vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
1366  }
1367  }
1368 
1369  ASSERT (n_bytes > 0);
1370  ASSERT (((*b)->current_data + (*b)->current_length) <= bytes_per_buffer);
1371 
1372  return n_bytes;
1373 }
1374 
1375 /**
1376  * Build a retransmit segment
1377  *
1378  * @return the number of bytes in the segment or 0 if there's nothing to
1379  * retransmit
1380  */
1381 static u32
1383  tcp_connection_t * tc, u32 offset,
1384  u32 max_deq_bytes, vlib_buffer_t ** b)
1385 {
1386  u32 start, available_bytes;
1387  int n_bytes = 0;
1388 
1389  ASSERT (tc->state >= TCP_STATE_ESTABLISHED);
1390  ASSERT (max_deq_bytes != 0);
1391 
1392  /*
1393  * Make sure we can retransmit something
1394  */
1395  available_bytes = transport_max_tx_dequeue (&tc->connection);
1396  ASSERT (available_bytes >= offset);
1397  available_bytes -= offset;
1398  if (!available_bytes)
1399  return 0;
1400 
1401  max_deq_bytes = clib_min (tc->snd_mss, max_deq_bytes);
1402  max_deq_bytes = clib_min (available_bytes, max_deq_bytes);
1403 
1404  /* Start is beyond snd_congestion */
1405  start = tc->snd_una + offset;
1406  if (seq_geq (start, tc->snd_congestion))
1407  return 0;
1408 
1409  /* Don't overshoot snd_congestion */
1410  if (seq_gt (start + max_deq_bytes, tc->snd_congestion))
1411  {
1412  max_deq_bytes = tc->snd_congestion - start;
1413  if (max_deq_bytes == 0)
1414  return 0;
1415  }
1416 
1417  n_bytes = tcp_prepare_segment (wrk, tc, offset, max_deq_bytes, b);
1418  if (!n_bytes)
1419  return 0;
1420 
1421  if (tcp_in_fastrecovery (tc))
1422  {
1423  tc->snd_rxt_bytes += n_bytes;
1424  if (tc->flags & TCP_CONN_RATE_SAMPLE)
1425  tcp_bt_track_rxt (tc, start, start + n_bytes);
1426  }
1427 
1428  tc->bytes_retrans += n_bytes;
1429  tc->segs_retrans += 1;
1430  TCP_EVT_DBG (TCP_EVT_CC_RTX, tc, offset, n_bytes);
1431  return n_bytes;
1432 }
1433 
1434 /**
1435  * Reset congestion control, switch cwnd to loss window and try again.
1436  */
1437 static void
1439 {
1440  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 6);
1441  tc->prev_ssthresh = tc->ssthresh;
1442  tc->prev_cwnd = tc->cwnd;
1443 
1444  /* Clear fast recovery state if needed */
1445  if (tcp_in_fastrecovery (tc))
1447 
1448  /* Let cc algo decide loss cwnd and ssthresh */
1449  tcp_cc_loss (tc);
1450 
1451  /* Start again from the beginning */
1452  tc->snd_congestion = tc->snd_nxt;
1453  tc->rcv_dupacks = 0;
1454  tc->rtt_ts = 0;
1455  tc->cwnd_acc_bytes = 0;
1456  tc->tr_occurences += 1;
1457  tcp_connection_tx_pacer_reset (tc, tc->cwnd, 2 * tc->snd_mss);
1458  tcp_recovery_on (tc);
1459 }
1460 
1461 void
1463 {
1464  u32 thread_index = vlib_get_thread_index ();
1465  tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1466  vlib_main_t *vm = wrk->vm;
1467  tcp_connection_t *tc;
1468  vlib_buffer_t *b = 0;
1469  u32 bi, n_bytes;
1470 
1471  tc = tcp_connection_get (tc_index, thread_index);
1472 
1473  /* Note: the connection may have been closed and pool_put */
1474  if (PREDICT_FALSE (tc == 0 || tc->state == TCP_STATE_SYN_SENT))
1475  return;
1476 
1477  tc->timers[TCP_TIMER_RETRANSMIT] = TCP_TIMER_HANDLE_INVALID;
1478 
1479  /* Wait-close and retransmit could pop at the same time */
1480  if (tc->state == TCP_STATE_CLOSED)
1481  return;
1482 
1483  if (tc->state >= TCP_STATE_ESTABLISHED)
1484  {
1485  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1486 
1487  /* Lost FIN, retransmit and return */
1488  if (tc->flags & TCP_CONN_FINSNT)
1489  {
1490  tcp_send_fin (tc);
1491  tc->rto_boff += 1;
1492  tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1493  return;
1494  }
1495 
1496  /* Shouldn't be here. This condition is tricky because it has to take
1497  * into account boff > 0 due to persist timeout. */
1498  if ((tc->rto_boff == 0 && tc->snd_una == tc->snd_nxt)
1499  || (tc->rto_boff > 0 && seq_geq (tc->snd_una, tc->snd_congestion)
1500  && !tcp_flight_size (tc)))
1501  {
1502  ASSERT (!tcp_in_recovery (tc));
1503  tc->rto_boff = 0;
1504  return;
1505  }
1506 
1507  /* We're not in recovery so make sure rto_boff is 0. Can be non 0 due
1508  * to persist timer timeout */
1509  if (!tcp_in_recovery (tc) && tc->rto_boff > 0)
1510  {
1511  tc->rto_boff = 0;
1512  tcp_update_rto (tc);
1513  }
1514 
1515  /* Peer is dead or network connectivity is lost. Close connection.
1516  * RFC 1122 section 4.2.3.5 recommends a value of at least 100s. For
1517  * a min rto of 0.2s we need to retry about 8 times. */
1518  if (tc->rto_boff >= TCP_RTO_BOFF_MAX)
1519  {
1520  tcp_send_reset (tc);
1521  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1522  session_transport_closing_notify (&tc->connection);
1524  tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time);
1525  return;
1526  }
1527 
1528  /* Increment RTO backoff (also equal to number of retries) and go back
1529  * to first un-acked byte */
1530  tc->rto_boff += 1;
1531 
1532  /* TODO be less aggressive about clearing scoreboard */
1533  scoreboard_clear (&tc->sack_sb);
1534 
1535  /* First retransmit timeout */
1536  if (tc->rto_boff == 1)
1537  {
1539  /* Record timestamp. Eifel detection algorithm RFC3522 */
1540  tc->snd_rxt_ts = tcp_tstamp (tc);
1541  }
1542 
1543  if (tc->flags & TCP_CONN_RATE_SAMPLE)
1544  tcp_bt_flush_samples (tc);
1545 
1546  /* If we've sent beyond snd_congestion, update it */
1547  tc->snd_congestion = seq_max (tc->snd_nxt, tc->snd_congestion);
1548  tc->snd_nxt = tc->snd_una;
1549 
1550  /* Send one segment. n_bytes may be zero due to buffer shortfall */
1551  n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
1552  if (!n_bytes)
1553  {
1554  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
1555  return;
1556  }
1557 
1558  bi = vlib_get_buffer_index (vm, b);
1559  tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1560 
1561  tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1563  }
1564  /* Retransmit SYN-ACK */
1565  else if (tc->state == TCP_STATE_SYN_RCVD)
1566  {
1567  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1568 
1569  tc->rtt_ts = 0;
1570 
1571  /* Passive open establish timeout */
1572  if (tc->rto > TCP_ESTABLISH_TIME >> 1)
1573  {
1574  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1576  tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.cleanup_time);
1577  return;
1578  }
1579 
1580  if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
1581  {
1582  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
1583  return;
1584  }
1585 
1586  tc->rto_boff += 1;
1587  if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1588  tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1589 
1591 
1592  b = vlib_get_buffer (vm, bi);
1593  tcp_init_buffer (vm, b);
1594  tcp_make_synack (tc, b);
1595  TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 1);
1596 
1597  /* Retransmit timer already updated, just enqueue to output */
1598  tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1599  }
1600  else
1601  {
1602  ASSERT (tc->state == TCP_STATE_CLOSED);
1603  return;
1604  }
1605 }
1606 
1607 /**
1608  * SYN retransmit timer handler. Active open only.
1609  */
1610 void
1612 {
1613  u32 thread_index = vlib_get_thread_index ();
1614  tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1615  vlib_main_t *vm = wrk->vm;
1616  tcp_connection_t *tc;
1617  vlib_buffer_t *b = 0;
1618  u32 bi;
1619 
1620  tc = tcp_half_open_connection_get (tc_index);
1621 
1622  /* Note: the connection may have transitioned to ESTABLISHED... */
1623  if (PREDICT_FALSE (tc == 0 || tc->state != TCP_STATE_SYN_SENT))
1624  return;
1625 
1626  tc->timers[TCP_TIMER_RETRANSMIT_SYN] = TCP_TIMER_HANDLE_INVALID;
1627 
1628  /* Half-open connection actually moved to established but we were
1629  * waiting for syn retransmit to pop to call cleanup from the right
1630  * thread. */
1631  if (tc->flags & TCP_CONN_HALF_OPEN_DONE)
1632  {
1634  TCP_DBG ("could not remove half-open connection");
1635  return;
1636  }
1637 
1638  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1639  tc->rtt_ts = 0;
1640 
1641  /* Active open establish timeout */
1642  if (tc->rto >= TCP_ESTABLISH_TIME >> 1)
1643  {
1644  session_stream_connect_notify (&tc->connection, 1 /* fail */ );
1646  return;
1647  }
1648 
1649  if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
1650  {
1651  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1);
1652  return;
1653  }
1654 
1655  /* Try without increasing RTO a number of times. If this fails,
1656  * start growing RTO exponentially */
1657  tc->rto_boff += 1;
1658  if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1659  tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1660 
1661  b = vlib_get_buffer (vm, bi);
1662  tcp_init_buffer (vm, b);
1663  tcp_make_syn (tc, b);
1664 
1665  TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 0);
1666 
1667  /* This goes straight to ipx_lookup */
1668  tcp_push_ip_hdr (wrk, tc, b);
1669  tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
1670 
1671  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
1672  tc->rto * TCP_TO_TIMER_TICK);
1673 }
1674 
1675 /**
1676  * Got 0 snd_wnd from peer, try to do something about it.
1677  *
1678  */
1679 void
1681 {
1682  u32 thread_index = vlib_get_thread_index ();
1683  tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1684  u32 bi, max_snd_bytes, available_bytes, offset;
1685  tcp_main_t *tm = vnet_get_tcp_main ();
1686  vlib_main_t *vm = wrk->vm;
1687  tcp_connection_t *tc;
1688  vlib_buffer_t *b;
1689  int n_bytes = 0;
1690  u8 *data;
1691 
1692  tc = tcp_connection_get_if_valid (index, thread_index);
1693  if (!tc)
1694  return;
1695 
1696  /* Make sure timer handle is set to invalid */
1697  tc->timers[TCP_TIMER_PERSIST] = TCP_TIMER_HANDLE_INVALID;
1698 
1699  /* Problem already solved or worse */
1700  if (tc->state == TCP_STATE_CLOSED || tc->snd_wnd > tc->snd_mss
1701  || (tc->flags & TCP_CONN_FINSNT))
1702  return;
1703 
1704  available_bytes = transport_max_tx_dequeue (&tc->connection);
1705  offset = tc->snd_nxt - tc->snd_una;
1706 
1707  /* Reprogram persist if no new bytes available to send. We may have data
1708  * next time */
1709  if (!available_bytes)
1710  {
1711  tcp_persist_timer_set (tc);
1712  return;
1713  }
1714 
1715  if (available_bytes <= offset)
1716  {
1717  ASSERT (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT));
1718  return;
1719  }
1720 
1721  /* Increment RTO backoff */
1722  tc->rto_boff += 1;
1723  tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1724 
1725  /*
1726  * Try to force the first unsent segment (or buffer)
1727  */
1728  if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
1729  {
1730  tcp_persist_timer_set (tc);
1731  return;
1732  }
1733  b = vlib_get_buffer (vm, bi);
1734  data = tcp_init_buffer (vm, b);
1735 
1736  tcp_validate_txf_size (tc, offset);
1737  tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1738  max_snd_bytes = clib_min (tc->snd_mss,
1739  tm->bytes_per_buffer - TRANSPORT_MAX_HDRS_LEN);
1740  n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset,
1741  max_snd_bytes);
1742  b->current_length = n_bytes;
1743  ASSERT (n_bytes != 0 && (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)
1744  || tc->snd_nxt == tc->snd_una_max
1745  || tc->rto_boff > 1));
1746 
1747  tcp_push_hdr_i (tc, b, tc->snd_nxt, /* compute opts */ 0,
1748  /* burst */ 0, /* update_snd_nxt */ 1);
1749  tc->snd_una_max = seq_max (tc->snd_nxt, tc->snd_una_max);
1750  tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
1751  tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1752 
1753  /* Just sent new data, enable retransmit */
1755 }
1756 
1757 /**
1758  * Retransmit first unacked segment
1759  */
1760 int
1762 {
1763  vlib_main_t *vm = wrk->vm;
1764  vlib_buffer_t *b;
1765  u32 bi, n_bytes;
1766 
1767  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 1);
1768 
1769  n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
1770  if (!n_bytes)
1771  return -1;
1772 
1773  bi = vlib_get_buffer_index (vm, b);
1774  tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1775 
1776  return 0;
1777 }
1778 
1779 static int
1781  u32 burst_size)
1782 {
1783  u32 offset, n_segs = 0, n_written, bi, available_wnd;
1784  vlib_main_t *vm = wrk->vm;
1785  vlib_buffer_t *b = 0;
1786 
1787  offset = tc->snd_nxt - tc->snd_una;
1788  available_wnd = tc->snd_wnd - offset;
1789  burst_size = clib_min (burst_size, available_wnd / tc->snd_mss);
1790 
1791  while (n_segs < burst_size)
1792  {
1793  n_written = tcp_prepare_segment (wrk, tc, offset, tc->snd_mss, &b);
1794  if (!n_written)
1795  goto done;
1796 
1797  bi = vlib_get_buffer_index (vm, b);
1798  tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1799  offset += n_written;
1800  n_segs += 1;
1801 
1802  tc->snd_nxt += n_written;
1803  tc->snd_una_max = seq_max (tc->snd_nxt, tc->snd_una_max);
1804  }
1805 
1806 done:
1807  return n_segs;
1808 }
1809 
1810 #define scoreboard_rescue_rxt_valid(_sb, _tc) \
1811  (seq_geq (_sb->rescue_rxt, _tc->snd_una) \
1812  && seq_leq (_sb->rescue_rxt, _tc->snd_congestion))
1813 
1814 /**
1815  * Do fast retransmit with SACKs
1816  */
1817 int
1819  u32 burst_size)
1820 {
1821  u32 n_written = 0, offset, max_bytes, n_segs = 0, n_segs_now;
1822  sack_scoreboard_hole_t *hole;
1823  vlib_main_t *vm = wrk->vm;
1824  vlib_buffer_t *b = 0;
1825  sack_scoreboard_t *sb;
1826  u32 bi, max_deq;
1827  int snd_space;
1828  u8 snd_limited = 0, can_rescue = 0;
1829 
1830  ASSERT (tcp_in_fastrecovery (tc));
1831 
1832  snd_space = tcp_available_cc_snd_space (tc);
1833  if (snd_space < tc->snd_mss)
1834  {
1836  return 0;
1837  }
1838 
1839  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
1840  sb = &tc->sack_sb;
1841  hole = scoreboard_get_hole (sb, sb->cur_rxt_hole);
1842 
1843  max_deq = transport_max_tx_dequeue (&tc->connection);
1844  max_deq -= tc->snd_nxt - tc->snd_una;
1845 
1846  while (snd_space > 0 && n_segs < burst_size)
1847  {
1848  hole = scoreboard_next_rxt_hole (sb, hole, max_deq, &can_rescue,
1849  &snd_limited);
1850  if (!hole)
1851  {
1852  if (max_deq)
1853  {
1854  snd_space = clib_min (max_deq, snd_space);
1855  burst_size = clib_min (burst_size - n_segs,
1856  snd_space / tc->snd_mss);
1857  n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size);
1858  if (max_deq > n_segs_now * tc->snd_mss)
1860  n_segs += n_segs_now;
1861  goto done;
1862  }
1863 
1864  if (!can_rescue || scoreboard_rescue_rxt_valid (sb, tc))
1865  break;
1866 
1867  /* If rescue rxt undefined or less than snd_una then one segment of
1868  * up to SMSS octets that MUST include the highest outstanding
1869  * unSACKed sequence number SHOULD be returned, and RescueRxt set to
1870  * RecoveryPoint. HighRxt MUST NOT be updated.
1871  */
1872  max_bytes = clib_min (tc->snd_mss,
1873  tc->snd_congestion - tc->snd_una);
1874  max_bytes = clib_min (max_bytes, snd_space);
1875  offset = tc->snd_congestion - tc->snd_una - max_bytes;
1876  sb->rescue_rxt = tc->snd_congestion;
1877  n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
1878  max_bytes, &b);
1879  if (!n_written)
1880  goto done;
1881 
1882  bi = vlib_get_buffer_index (vm, b);
1883  tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1884  n_segs += 1;
1885  break;
1886  }
1887 
1888  max_bytes = clib_min (hole->end - sb->high_rxt, snd_space);
1889  max_bytes = snd_limited ? clib_min (max_bytes, tc->snd_mss) : max_bytes;
1890  if (max_bytes == 0)
1891  break;
1892 
1893  offset = sb->high_rxt - tc->snd_una;
1894  n_written = tcp_prepare_retransmit_segment (wrk, tc, offset, max_bytes,
1895  &b);
1896  ASSERT (n_written <= snd_space);
1897 
1898  /* Nothing left to retransmit */
1899  if (n_written == 0)
1900  break;
1901 
1902  bi = vlib_get_buffer_index (vm, b);
1903  tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1904 
1905  sb->high_rxt += n_written;
1906  snd_space -= n_written;
1907  n_segs += 1;
1908  }
1909 
1910  if (hole)
1912 
1913 done:
1914  return n_segs;
1915 }
1916 
1917 /**
1918  * Fast retransmit without SACK info
1919  */
1920 int
1922  u32 burst_size)
1923 {
1924  u32 n_written = 0, offset = 0, bi, max_deq, n_segs_now;
1925  vlib_main_t *vm = wrk->vm;
1926  int snd_space, n_segs = 0;
1927  vlib_buffer_t *b;
1928 
1929  ASSERT (tcp_in_fastrecovery (tc));
1930  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
1931 
1932  snd_space = tcp_available_cc_snd_space (tc);
1933 
1934  if (!tcp_fastrecovery_first (tc))
1935  goto send_unsent;
1936 
1937  /* RFC 6582: [If a partial ack], retransmit the first unacknowledged
1938  * segment. */
1939  while (snd_space > 0 && n_segs < burst_size)
1940  {
1941  n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
1942  tc->snd_mss, &b);
1943 
1944  /* Nothing left to retransmit */
1945  if (n_written == 0)
1946  break;
1947 
1948  bi = vlib_get_buffer_index (vm, b);
1949  tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1950  snd_space -= n_written;
1951  offset += n_written;
1952  n_segs += 1;
1953  }
1954 
1955  if (n_segs == burst_size)
1956  goto done;
1957 
1958 send_unsent:
1959 
1960  /* RFC 6582: Send a new segment if permitted by the new value of cwnd. */
1961  if (snd_space < tc->snd_mss || tc->snd_mss == 0)
1962  goto done;
1963 
1964  max_deq = transport_max_tx_dequeue (&tc->connection);
1965  max_deq -= tc->snd_nxt - tc->snd_una;
1966  if (max_deq)
1967  {
1968  snd_space = clib_min (max_deq, snd_space);
1969  burst_size = clib_min (burst_size - n_segs, snd_space / tc->snd_mss);
1970  n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size);
1971  if (max_deq > n_segs_now * tc->snd_mss)
1973  n_segs += n_segs_now;
1974  }
1975 
1976 done:
1978  return n_segs;
1979 }
1980 
1981 /**
1982  * Do fast retransmit
1983  */
1984 int
1986  u32 burst_size)
1987 {
1988  if (tcp_opts_sack_permitted (&tc->rcv_opts))
1989  return tcp_fast_retransmit_sack (wrk, tc, burst_size);
1990  else
1991  return tcp_fast_retransmit_no_sack (wrk, tc, burst_size);
1992 }
1993 
1994 static int
1995 tcp_send_acks (tcp_connection_t * tc, u32 max_burst_size)
1996 {
1997  int j, n_acks;
1998 
1999  if (!tc->pending_dupacks)
2000  {
2001  tcp_send_ack (tc);
2002  return 1;
2003  }
2004 
2005  /* If we're supposed to send dupacks but have no ooo data
2006  * send only one ack */
2007  if (!vec_len (tc->snd_sacks))
2008  {
2009  tcp_send_ack (tc);
2010  return 1;
2011  }
2012 
2013  /* Start with first sack block */
2014  tc->snd_sack_pos = 0;
2015 
2016  /* Generate enough dupacks to cover all sack blocks. Do not generate
2017  * more sacks than the number of packets received. But do generate at
2018  * least 3, i.e., the number needed to signal congestion, if needed. */
2019  n_acks = vec_len (tc->snd_sacks) / TCP_OPTS_MAX_SACK_BLOCKS;
2020  n_acks = clib_min (n_acks, tc->pending_dupacks);
2021  n_acks = clib_max (n_acks, clib_min (tc->pending_dupacks, 3));
2022  for (j = 0; j < clib_min (n_acks, max_burst_size); j++)
2023  tcp_send_ack (tc);
2024 
2025  if (n_acks < max_burst_size)
2026  {
2027  tc->pending_dupacks = 0;
2028  tc->snd_sack_pos = 0;
2029  tc->dupacks_out += n_acks;
2030  return n_acks;
2031  }
2032  else
2033  {
2034  TCP_DBG ("constrained by burst size");
2035  tc->pending_dupacks = n_acks - max_burst_size;
2036  tc->dupacks_out += max_burst_size;
2037  tcp_program_dupack (tc);
2038  return max_burst_size;
2039  }
2040 }
2041 
2042 static int
2044 {
2045  u32 n_segs = 0, burst_size, sent_bytes, burst_bytes;
2046  tcp_worker_ctx_t *wrk;
2047 
2048  wrk = tcp_get_worker (tc->c_thread_index);
2049  burst_bytes = transport_connection_tx_pacer_burst (&tc->connection,
2050  wrk->vm->
2051  clib_time.last_cpu_time);
2052  burst_size = clib_min (max_burst_size, burst_bytes / tc->snd_mss);
2053  if (!burst_size)
2054  {
2056  return 0;
2057  }
2058 
2059  n_segs = tcp_fast_retransmit (wrk, tc, burst_size);
2060  sent_bytes = clib_min (n_segs * tc->snd_mss, burst_bytes);
2061  transport_connection_tx_pacer_update_bytes (&tc->connection, sent_bytes);
2062  return n_segs;
2063 }
2064 
2065 int
2066 tcp_session_custom_tx (void *conn, u32 max_burst_size)
2067 {
2068  tcp_connection_t *tc = (tcp_connection_t *) conn;
2069  u32 n_segs = 0;
2070 
2071  if (tcp_in_fastrecovery (tc) && (tc->flags & TCP_CONN_FRXT_PENDING))
2072  {
2073  tc->flags &= ~TCP_CONN_FRXT_PENDING;
2074  n_segs = tcp_do_fastretransmit (tc, max_burst_size);
2075  max_burst_size -= n_segs;
2076  }
2077 
2078  if (!(tc->flags & TCP_CONN_SNDACK))
2079  return n_segs;
2080 
2081  tc->flags &= ~TCP_CONN_SNDACK;
2082 
2083  /* We have retransmitted packets and no dupack */
2084  if (n_segs && !tc->pending_dupacks)
2085  return n_segs;
2086 
2087  if (!max_burst_size)
2088  {
2089  tcp_program_ack (tc);
2090  return max_burst_size;
2091  }
2092 
2093  n_segs += tcp_send_acks (tc, max_burst_size);
2094 
2095  return n_segs;
2096 }
2097 #endif /* CLIB_MARCH_VARIANT */
2098 
2099 static void
2101  u16 * next0, u32 * error0)
2102 {
2103  ip_adjacency_t *adj;
2104  adj_index_t ai;
2105 
2106  /* Not thread safe but as long as the connection exists the adj should
2107  * not be removed */
2108  ai = adj_nbr_find (FIB_PROTOCOL_IP6, VNET_LINK_IP6, &tc0->c_rmt_ip,
2109  tc0->sw_if_index);
2110  if (ai == ADJ_INDEX_INVALID)
2111  {
2112  vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
2113  *next0 = TCP_OUTPUT_NEXT_DROP;
2114  *error0 = TCP_ERROR_LINK_LOCAL_RW;
2115  return;
2116  }
2117 
2118  adj = adj_get (ai);
2120  *next0 = TCP_OUTPUT_NEXT_IP_REWRITE;
2121  else if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP)
2122  *next0 = TCP_OUTPUT_NEXT_IP_ARP;
2123  else
2124  {
2125  *next0 = TCP_OUTPUT_NEXT_DROP;
2126  *error0 = TCP_ERROR_LINK_LOCAL_RW;
2127  }
2128  vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ai;
2129 }
2130 
2131 static void
2133  u32 * to_next, u32 n_bufs)
2134 {
2135  u32 n_trace = vlib_get_trace_count (vm, node);
2136  tcp_connection_t *tc;
2137  tcp_tx_trace_t *t;
2138  vlib_buffer_t *b;
2139  tcp_header_t *th;
2140  int i;
2141 
2142  for (i = 0; i < clib_min (n_trace, n_bufs); i++)
2143  {
2144  b = vlib_get_buffer (vm, to_next[i]);
2145  th = vlib_buffer_get_current (b);
2146  tc = tcp_connection_get (vnet_buffer (b)->tcp.connection_index,
2147  vm->thread_index);
2148  t = vlib_add_trace (vm, node, b, sizeof (*t));
2149  clib_memcpy_fast (&t->tcp_header, th, sizeof (t->tcp_header));
2150  clib_memcpy_fast (&t->tcp_connection, tc, sizeof (t->tcp_connection));
2151  }
2152 }
2153 
2154 always_inline void
2156  tcp_connection_t * tc0, u8 is_ip4)
2157 {
2158  tcp_header_t *th0 = 0;
2159 
2160  th0 = vlib_buffer_get_current (b0);
2161  TCP_EVT_DBG (TCP_EVT_OUTPUT, tc0, th0->flags, b0->current_length);
2162  if (is_ip4)
2163  {
2164  vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, &tc0->c_rmt_ip4,
2165  IP_PROTOCOL_TCP, 1);
2166  b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
2167  vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
2168  th0->checksum = 0;
2169  }
2170  else
2171  {
2172  ip6_header_t *ih0;
2173  ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6,
2174  &tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
2175  b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
2176  vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
2177  vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
2178  th0->checksum = 0;
2179  }
2180 }
2181 
2182 always_inline void
2184  u32 * error0, u16 * next0, u8 is_ip4)
2185 {
2186 
2187  if (PREDICT_FALSE (tc0->state == TCP_STATE_CLOSED))
2188  {
2189  *error0 = TCP_ERROR_INVALID_CONNECTION;
2190  *next0 = TCP_OUTPUT_NEXT_DROP;
2191  return;
2192  }
2193 
2194  /* If next_index is not drop use it */
2195  if (tc0->next_node_index)
2196  {
2197  *next0 = tc0->next_node_index;
2198  vnet_buffer (b0)->tcp.next_node_opaque = tc0->next_node_opaque;
2199  }
2200 
2201  vnet_buffer (b0)->sw_if_index[VLIB_TX] = tc0->c_fib_index;
2202  vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
2203 
2204  if (!is_ip4)
2205  {
2206  if (PREDICT_FALSE (ip6_address_is_link_local_unicast (&tc0->c_rmt_ip6)))
2207  tcp_output_handle_link_local (tc0, b0, next0, error0);
2208  }
2209 
2210  if (!TCP_ALWAYS_ACK)
2211  tcp_timer_reset (tc0, TCP_TIMER_DELACK);
2212 
2213  tc0->segs_out += 1;
2214 }
2215 
2218  vlib_frame_t * frame, int is_ip4)
2219 {
2220  u32 n_left_from, *from, thread_index = vm->thread_index;
2221  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
2222  u16 nexts[VLIB_FRAME_SIZE], *next;
2223 
2224  from = vlib_frame_vector_args (frame);
2225  n_left_from = frame->n_vectors;
2226  tcp_set_time_now (tcp_get_worker (thread_index));
2227 
2229  tcp46_output_trace_frame (vm, node, from, n_left_from);
2230 
2231  vlib_get_buffers (vm, from, bufs, n_left_from);
2232  b = bufs;
2233  next = nexts;
2234 
2235  while (n_left_from >= 4)
2236  {
2237  u32 error0 = TCP_ERROR_PKTS_SENT, error1 = TCP_ERROR_PKTS_SENT;
2238  tcp_connection_t *tc0, *tc1;
2239 
2240  {
2241  vlib_prefetch_buffer_header (b[2], STORE);
2242  CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2243 
2244  vlib_prefetch_buffer_header (b[3], STORE);
2245  CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2246  }
2247 
2248  next[0] = next[1] = TCP_OUTPUT_NEXT_IP_LOOKUP;
2249 
2250  tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
2251  thread_index);
2252  tc1 = tcp_connection_get (vnet_buffer (b[1])->tcp.connection_index,
2253  thread_index);
2254 
2255  tcp_output_push_ip (vm, b[0], tc0, is_ip4);
2256  tcp_output_push_ip (vm, b[1], tc1, is_ip4);
2257 
2258  tcp_output_handle_packet (tc0, b[0], &error0, &next[0], is_ip4);
2259  tcp_output_handle_packet (tc1, b[1], &error1, &next[1], is_ip4);
2260 
2261  b += 2;
2262  next += 2;
2263  n_left_from -= 2;
2264  }
2265  while (n_left_from > 0)
2266  {
2267  u32 error0 = TCP_ERROR_PKTS_SENT;
2268  tcp_connection_t *tc0;
2269 
2270  if (n_left_from > 1)
2271  {
2272  vlib_prefetch_buffer_header (b[1], STORE);
2273  CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2274  }
2275 
2276  next[0] = TCP_OUTPUT_NEXT_IP_LOOKUP;
2277  tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
2278  thread_index);
2279 
2280  tcp_output_push_ip (vm, b[0], tc0, is_ip4);
2281  tcp_output_handle_packet (tc0, b[0], &error0, &next[0], is_ip4);
2282 
2283  b += 1;
2284  next += 1;
2285  n_left_from -= 1;
2286  }
2287 
2288  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
2289  return frame->n_vectors;
2290 }
2291 
2293  vlib_frame_t * from_frame)
2294 {
2295  return tcp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */ );
2296 }
2297 
2299  vlib_frame_t * from_frame)
2300 {
2301  return tcp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */ );
2302 }
2303 
2304 /* *INDENT-OFF* */
2306 {
2307  .name = "tcp4-output",
2308  /* Takes a vector of packets. */
2309  .vector_size = sizeof (u32),
2310  .n_errors = TCP_N_ERROR,
2311  .protocol_hint = VLIB_NODE_PROTO_HINT_TCP,
2312  .error_strings = tcp_error_strings,
2313  .n_next_nodes = TCP_OUTPUT_N_NEXT,
2314  .next_nodes = {
2315 #define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
2317 #undef _
2318  },
2319  .format_buffer = format_tcp_header,
2320  .format_trace = format_tcp_tx_trace,
2321 };
2322 /* *INDENT-ON* */
2323 
2324 /* *INDENT-OFF* */
2326 {
2327  .name = "tcp6-output",
2328  /* Takes a vector of packets. */
2329  .vector_size = sizeof (u32),
2330  .n_errors = TCP_N_ERROR,
2331  .protocol_hint = VLIB_NODE_PROTO_HINT_TCP,
2332  .error_strings = tcp_error_strings,
2333  .n_next_nodes = TCP_OUTPUT_N_NEXT,
2334  .next_nodes = {
2335 #define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
2337 #undef _
2338  },
2339  .format_buffer = format_tcp_header,
2340  .format_trace = format_tcp_tx_trace,
2341 };
2342 /* *INDENT-ON* */
2343 
2344 typedef enum _tcp_reset_next
2345 {
2350 
2351 #define foreach_tcp4_reset_next \
2352  _(DROP, "error-drop") \
2353  _(IP_LOOKUP, "ip4-lookup")
2354 
2355 #define foreach_tcp6_reset_next \
2356  _(DROP, "error-drop") \
2357  _(IP_LOOKUP, "ip6-lookup")
2358 
2359 static uword
2361  vlib_frame_t * from_frame, u8 is_ip4)
2362 {
2363  u32 n_left_from, next_index, *from, *to_next;
2364  u32 my_thread_index = vm->thread_index;
2365 
2366  from = vlib_frame_vector_args (from_frame);
2367  n_left_from = from_frame->n_vectors;
2368 
2369  next_index = node->cached_next_index;
2370 
2371  while (n_left_from > 0)
2372  {
2373  u32 n_left_to_next;
2374 
2375  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2376 
2377  while (n_left_from > 0 && n_left_to_next > 0)
2378  {
2379  u32 bi0;
2380  vlib_buffer_t *b0;
2381  tcp_tx_trace_t *t0;
2382  tcp_header_t *th0;
2383  u32 error0 = TCP_ERROR_RST_SENT, next0 = TCP_RESET_NEXT_IP_LOOKUP;
2384 
2385  bi0 = from[0];
2386  to_next[0] = bi0;
2387  from += 1;
2388  to_next += 1;
2389  n_left_from -= 1;
2390  n_left_to_next -= 1;
2391 
2392  b0 = vlib_get_buffer (vm, bi0);
2393 
2394  if (tcp_make_reset_in_place (vm, b0, vnet_buffer (b0)->tcp.flags,
2395  my_thread_index, is_ip4))
2396  {
2397  error0 = TCP_ERROR_LOOKUP_DROPS;
2398  next0 = TCP_RESET_NEXT_DROP;
2399  goto done;
2400  }
2401 
2402  /* Prepare to send to IP lookup */
2403  vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
2404  next0 = TCP_RESET_NEXT_IP_LOOKUP;
2405 
2406  done:
2407  b0->error = node->errors[error0];
2408  b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
2409  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2410  {
2411  th0 = vlib_buffer_get_current (b0);
2412  if (is_ip4)
2413  th0 = ip4_next_header ((ip4_header_t *) th0);
2414  else
2415  th0 = ip6_next_header ((ip6_header_t *) th0);
2416  t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
2417  clib_memcpy_fast (&t0->tcp_header, th0,
2418  sizeof (t0->tcp_header));
2419  }
2420 
2421  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2422  n_left_to_next, bi0, next0);
2423  }
2424  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2425  }
2426  return from_frame->n_vectors;
2427 }
2428 
2430  vlib_frame_t * from_frame)
2431 {
2432  return tcp46_send_reset_inline (vm, node, from_frame, 1);
2433 }
2434 
2436  vlib_frame_t * from_frame)
2437 {
2438  return tcp46_send_reset_inline (vm, node, from_frame, 0);
2439 }
2440 
2441 /* *INDENT-OFF* */
2443  .name = "tcp4-reset",
2444  .vector_size = sizeof (u32),
2445  .n_errors = TCP_N_ERROR,
2446  .error_strings = tcp_error_strings,
2447  .n_next_nodes = TCP_RESET_N_NEXT,
2448  .next_nodes = {
2449 #define _(s,n) [TCP_RESET_NEXT_##s] = n,
2451 #undef _
2452  },
2453  .format_trace = format_tcp_tx_trace,
2454 };
2455 /* *INDENT-ON* */
2456 
2457 /* *INDENT-OFF* */
2459  .name = "tcp6-reset",
2460  .vector_size = sizeof (u32),
2461  .n_errors = TCP_N_ERROR,
2462  .error_strings = tcp_error_strings,
2463  .n_next_nodes = TCP_RESET_N_NEXT,
2464  .next_nodes = {
2465 #define _(s,n) [TCP_RESET_NEXT_##s] = n,
2467 #undef _
2468  },
2469  .format_trace = format_tcp_tx_trace,
2470 };
2471 /* *INDENT-ON* */
2472 
2473 /*
2474  * fd.io coding-style-patch-verification: ON
2475  *
2476  * Local Variables:
2477  * eval: (c-set-style "gnu")
2478  * End:
2479  */
void tcp_make_fin(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to FIN-ACK.
Definition: tcp_output.c:525
#define tcp_in_cong_recovery(tc)
Definition: tcp.h:438
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
void session_flush_frames_main_thread(vlib_main_t *vm)
Definition: session.c:1428
End of options.
Definition: tcp_packet.h:104
static u32 tcp_options_write(u8 *data, tcp_options_t *opts)
Write TCP options to segment.
Definition: tcp_output.c:185
u32 flags
Definition: vhost_user.h:141
#define clib_min(x, y)
Definition: clib.h:295
static int tcp_send_acks(tcp_connection_t *tc, u32 max_burst_size)
Definition: tcp_output.c:1995
#define TCP_OPTION_LEN_EOL
Definition: tcp_packet.h:163
#define CLIB_UNUSED(x)
Definition: clib.h:82
#define tcp_in_recovery(tc)
Definition: tcp.h:429
void scoreboard_clear(sack_scoreboard_t *sb)
Definition: tcp_input.c:906
static f64 tcp_time_now_us(u32 thread_index)
Definition: tcp.h:993
static void tcp_retransmit_timer_set(tcp_connection_t *tc)
Definition: tcp.h:1097
static u32 transport_rx_fifo_size(transport_connection_t *tc)
Definition: session.h:453
#define TCP_OPTION_LEN_SACK_PERMITTED
Definition: tcp_packet.h:167
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:171
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
ip4_address_t src_address
Definition: ip4_packet.h:170
#define tcp_node_index(node_id, is_ip4)
Definition: tcp.h:635
int session_tx_fifo_peek_bytes(transport_connection_t *tc, u8 *buffer, u32 offset, u32 max_bytes)
Definition: session.c:475
#define TCP_TO_TIMER_TICK
Factor for converting ticks to timer ticks.
Definition: tcp.h:93
Selective Ack permitted.
Definition: tcp_packet.h:108
#define TCP_FLAG_SYN
Definition: fa_node.h:13
#define tcp_opts_tstamp(_to)
Definition: tcp_packet.h:157
void tcp_make_synack(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to SYN-ACK.
Definition: tcp_output.c:559
#define PREDICT_TRUE(x)
Definition: clib.h:112
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
static void tcp_flush_frame_to_ip_lookup(tcp_worker_ctx_t *wrk, u8 is_ip4)
Flush ip lookup tx frames populated by timer pops.
Definition: tcp_output.c:1003
static tcp_connection_t * tcp_connection_get_if_valid(u32 conn_index, u32 thread_index)
Definition: tcp.h:681
static int tcp_make_syn_options(tcp_options_t *opts, u8 wnd_scale)
Definition: tcp_output.c:266
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
struct _sack_scoreboard sack_scoreboard_t
IP unicast adjacency.
Definition: adj.h:221
u32 fib_table_get_index_for_sw_if_index(fib_protocol_t proto, u32 sw_if_index)
Get the index of the FIB bound to the interface.
Definition: fib_table.c:967
static tcp_connection_t * tcp_half_open_connection_get(u32 conn_index)
Definition: tcp.h:726
#define tcp_zero_rwnd_sent_off(tc)
Definition: tcp.h:450
void tcp_update_rcv_mss(tcp_connection_t *tc)
Update max segment size we&#39;re able to process.
Definition: tcp_output.c:85
vlib_frame_t * tx_frames[2]
tx frames for tcp 4/6 output nodes
Definition: tcp.h:478
void session_add_self_custom_tx_evt(transport_connection_t *tc, u8 has_prio)
Definition: session.c:122
void tcp_timer_retransmit_handler(u32 tc_index)
Definition: tcp_output.c:1462
struct _tcp_main tcp_main_t
u32 thread_index
Definition: main.h:197
void tcp_connection_timers_reset(tcp_connection_t *tc)
Stop all connection timers.
Definition: tcp.c:488
This packet is to be rewritten and forwarded to the next processing node.
Definition: adj.h:73
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
u8 data[0]
Packet data.
Definition: buffer.h:181
#define TCP_OPTS_ALIGN
Definition: tcp_packet.h:174
static u32 tcp_initial_wnd_unscaled(tcp_connection_t *tc)
TCP&#39;s initial window.
Definition: tcp_output.c:95
enum _tcp_output_next tcp_output_next_t
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
clib_memset(h->entries, 0, sizeof(h->entries[0])*entries)
struct _tcp_connection tcp_connection_t
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
static u32 tcp_available_cc_snd_space(const tcp_connection_t *tc)
Estimate of how many bytes we can still push into the network.
Definition: tcp.h:933
#define tcp_opts_sack(_to)
Definition: tcp_packet.h:159
u8 data[128]
Definition: ipsec.api:249
#define VLIB_NODE_FN(node)
Definition: node.h:201
static void tcp_push_ip_hdr(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, vlib_buffer_t *b)
Definition: tcp_output.c:894
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
static uword tcp46_send_reset_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, u8 is_ip4)
Definition: tcp_output.c:2360
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:468
No operation.
Definition: tcp_packet.h:105
int tcp_fast_retransmit_no_sack(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 burst_size)
Fast retransmit without SACK info.
Definition: tcp_output.c:1921
u8 n_sack_blocks
Number of SACKs blocks.
Definition: tcp_packet.h:152
static void tcp_enqueue_to_output_i(tcp_worker_ctx_t *wrk, vlib_buffer_t *b, u32 bi, u8 is_ip4, u8 flush)
Definition: tcp_output.c:632
struct _tcp_header tcp_header_t
int tcp_half_open_connection_cleanup(tcp_connection_t *tc)
Try to cleanup half-open connection.
Definition: tcp.c:211
#define scoreboard_rescue_rxt_valid(_sb, _tc)
Definition: tcp_output.c:1810
ip6_address_t src_address
Definition: ip6_packet.h:383
unsigned char u8
Definition: types.h:56
struct _sack_scoreboard_hole sack_scoreboard_hole_t
u8 wscale
Window scale advertised.
Definition: tcp_packet.h:148
u32 transport_connection_tx_pacer_burst(transport_connection_t *tc, u64 time_now)
Get tx pacer max burst.
Definition: transport.c:679
enum fib_protocol_t_ fib_protocol_t
Protocol Type.
#define TCP_OPTS_MAX_SACK_BLOCKS
Definition: tcp_packet.h:175
vlib_node_registration_t ip4_lookup_node
(constructor) VLIB_REGISTER_NODE (ip4_lookup_node)
Definition: ip4_forward.c:101
#define foreach_tcp4_reset_next
Definition: tcp_output.c:2351
static u32 tcp_prepare_retransmit_segment(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 offset, u32 max_deq_bytes, vlib_buffer_t **b)
Build a retransmit segment.
Definition: tcp_output.c:1382
u16 src_port
Definition: udp.api:41
Limit MSS.
Definition: tcp_packet.h:106
#define tcp_zero_rwnd_sent_on(tc)
Definition: tcp.h:449
void session_transport_closing_notify(transport_connection_t *tc)
Notification from transport that connection is being closed.
Definition: session.c:829
static uword tcp46_output_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_ip4)
Definition: tcp_output.c:2217
static void * tcp_init_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Definition: tcp_output.c:468
static ip_adjacency_t * adj_get(adj_index_t adj_index)
Get a pointer to an adjacency object from its index.
Definition: adj.h:433
void tcp_make_syn(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to SYN.
Definition: tcp_output.c:534
void tcp_timer_retransmit_syn_handler(u32 tc_index)
SYN retransmit timer handler.
Definition: tcp_output.c:1611
static int tcp_prepare_segment(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 offset, u32 max_deq_bytes, vlib_buffer_t **b)
Allocate a new buffer and build a new tcp segment.
Definition: tcp_output.c:1270
#define seq_gt(_s1, _s2)
Definition: tcp.h:824
static void tcp_connection_set_state(tcp_connection_t *tc, tcp_state_t state)
Definition: tcp.h:697
#define tcp_cfg
Definition: tcp.h:634
vl_api_interface_index_t sw_if_index
Definition: gre.api:50
sack_scoreboard_hole_t * scoreboard_get_hole(sack_scoreboard_t *sb, u32 index)
Definition: tcp_input.c:662
#define always_inline
Definition: clib.h:98
#define TCP_OPTION_LEN_SACK_BLOCK
Definition: tcp_packet.h:169
ip4_address_t dst_address
Definition: ip4_packet.h:170
#define TCP_FLAG_ACK
Definition: fa_node.h:16
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
static void tcp_cc_loss(tcp_connection_t *tc)
Definition: tcp.h:1032
tcp_main_t tcp_main
Definition: tcp.c:30
static tcp_header_t * tcp_buffer_hdr(vlib_buffer_t *b)
Definition: tcp.h:651
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:187
enum _tcp_state tcp_state_t
#define TCP_ALWAYS_ACK
On/off delayed acks.
Definition: tcp.h:37
#define TCP_RTO_MAX
Definition: tcp.h:97
vhost_vring_state_t state
Definition: vhost_user.h:146
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:241
static u32 tcp_time_now(void)
Definition: tcp.h:971
sack_block_t * sacks
SACK blocks.
Definition: tcp_packet.h:151
unsigned int u32
Definition: types.h:88
static void tcp46_output_trace_frame(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *to_next, u32 n_bufs)
Definition: tcp_output.c:2132
#define TCP_ESTABLISH_TIME
Definition: tcp.h:103
sack_scoreboard_hole_t * scoreboard_next_rxt_hole(sack_scoreboard_t *sb, sack_scoreboard_hole_t *start, u8 have_sent_1_smss, u8 *can_rescue, u8 *snd_limited)
Figure out the next hole to retransmit.
Definition: tcp_input.c:828
#define tcp_validate_txf_size(_tc, _a)
Definition: tcp.h:1158
#define VLIB_FRAME_SIZE
Definition: node.h:376
static void tcp_enqueue_to_ip_lookup_now(tcp_worker_ctx_t *wrk, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index)
Definition: tcp_output.c:616
static void tcp_push_hdr_i(tcp_connection_t *tc, vlib_buffer_t *b, u32 snd_nxt, u8 compute_opts, u8 maybe_burst, u8 update_snd_nxt)
Push TCP header and update connection variables.
Definition: tcp_output.c:1076
#define TCP_EVT_DBG(_evt, _args...)
Definition: tcp_debug.h:243
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:257
u32 tcp_session_push_header(transport_connection_t *tconn, vlib_buffer_t *b)
Definition: tcp_output.c:1138
#define TCP_OPTION_LEN_WINDOW_SCALE
Definition: tcp_packet.h:166
vlib_node_registration_t tcp6_reset_node
(constructor) VLIB_REGISTER_NODE (tcp6_reset_node)
Definition: tcp_output.c:2458
#define TCP_RTO_SYN_RETRIES
Definition: tcp.h:100
#define tcp_zero_rwnd_sent(tc)
Definition: tcp.h:448
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
int tcp_fast_retransmit_sack(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 burst_size)
Do fast retransmit with SACKs.
Definition: tcp_output.c:1818
#define tcp_trajectory_add_start(b, start)
Definition: tcp.h:664
#define TRANSPORT_MAX_HDRS_LEN
vlib_main_t * vm
convenience pointer to this thread&#39;s vlib main
Definition: tcp.h:490
void tcp_send_reset(tcp_connection_t *tc)
Build and set reset packet for connection.
Definition: tcp_output.c:846
void tcp_send_synack(tcp_connection_t *tc)
Definition: tcp_output.c:961
#define ADJ_INDEX_INVALID
Invalid ADJ index - used when no adj is known likewise blazoned capitals INVALID speak volumes where ...
Definition: adj_types.h:36
static int tcp_make_synack_options(tcp_connection_t *tc, tcp_options_t *opts)
Definition: tcp_output.c:295
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
Definition: buffer.h:350
#define tcp_in_fastrecovery(tc)
Definition: tcp.h:428
void tcp_connection_tx_pacer_reset(tcp_connection_t *tc, u32 window, u32 start_bucket)
Definition: tcp.c:1280
static void * vlib_buffer_push_tcp_net_order(vlib_buffer_t *b, u16 sp, u16 dp, u32 seq, u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
Push TCP header to buffer.
Definition: tcp.h:1202
#define tcp_opts_mss(_to)
Definition: tcp_packet.h:156
unsigned short u16
Definition: types.h:57
void tcp_flush_frames_to_output(tcp_worker_ctx_t *wrk)
Flush v4 and v6 tcp and ip-lookup tx frames for thread index.
Definition: tcp_output.c:1019
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:196
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
#define TCP_TIMER_HANDLE_INVALID
Definition: tcp.h:90
static void tcp_output_handle_link_local(tcp_connection_t *tc0, vlib_buffer_t *b0, u16 *next0, u32 *error0)
Definition: tcp_output.c:2100
#define foreach_tcp6_output_next
Definition: tcp_output.c:34
static u32 tcp_flight_size(const tcp_connection_t *tc)
Our estimate of the number of bytes in flight (pipe size)
Definition: tcp.h:848
#define PREDICT_FALSE(x)
Definition: clib.h:111
void tcp_program_dupack(tcp_connection_t *tc)
Definition: tcp_output.c:1199
static int tcp_make_reset_in_place(vlib_main_t *vm, vlib_buffer_t *b0, tcp_state_t state, u8 thread_index, u8 is_ip4)
Definition: tcp_output.c:679
#define TCP_FLAG_FIN
Definition: fa_node.h:12
static u8 tcp_window_compute_scale(u32 window)
Definition: tcp_output.c:70
int tcp_session_custom_tx(void *conn, u32 max_burst_size)
Definition: tcp_output.c:2066
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:338
#define TCP_OPTION_LEN_TIMESTAMP
Definition: tcp_packet.h:168
#define foreach_tcp4_output_next
Definition: tcp_output.c:28
#define TCP_WND_MAX
Definition: tcp_packet.h:172
static void tcp_enqueue_to_ip_lookup(tcp_worker_ctx_t *wrk, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index)
Definition: tcp_output.c:623
Selective Ack block.
Definition: tcp_packet.h:109
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1150
#define TCP_FLAG_RST
Definition: fa_node.h:14
#define TCP_DBG(_fmt, _args...)
Definition: tcp_debug.h:244
u8 len
Definition: ip_types.api:90
#define TCP_MAX_WND_SCALE
Definition: tcp_packet.h:173
static void tcp_timer_reset(tcp_connection_t *tc, u8 timer_id)
Definition: tcp.h:1069
This packet matches an "incomplete adjacency" and packets need to be passed to ARP to find rewrite st...
Definition: adj.h:63
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
static void * vlib_buffer_push_tcp(vlib_buffer_t *b, u16 sp_net, u16 dp_net, u32 seq, u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
Push TCP header to buffer.
Definition: tcp.h:1237
tcp_header_t tcp_header
Definition: tcp_output.c:48
u16 n_vectors
Definition: node.h:395
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:213
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
vlib_main_t * vm
Definition: buffer.c:312
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_node.h:332
void tcp_send_window_update_ack(tcp_connection_t *tc)
Send Window Update ACK, ensuring that it will be sent once, if RWND became non-zero, after zero RWND has been advertised in ACK before.
Definition: tcp_output.c:1242
static u32 tcp_tstamp(tcp_connection_t *tc)
Generate timestamp for tcp connection.
Definition: tcp.h:986
void tcp_bt_track_tx(tcp_connection_t *tc)
Track a tcp tx burst.
Definition: tcp_bt.c:266
void tcp_send_reset_w_pkt(tcp_connection_t *tc, vlib_buffer_t *pkt, u32 thread_index, u8 is_ip4)
Send reset without reusing existing buffer.
Definition: tcp_output.c:764
#define clib_warning(format, args...)
Definition: error.h:59
format_function_t format_tcp_header
Definition: format.h:101
struct _transport_connection transport_connection_t
#define TCP_USE_SACKS
Disable only for testing.
Definition: tcp.h:38
#define tcp_recovery_on(tc)
Definition: tcp.h:426
static u32 tcp_window_to_advertise(tcp_connection_t *tc, tcp_state_t state)
Compute and return window to advertise, scaled as per RFC1323.
Definition: tcp_output.c:172
#define tcp_fastrecovery_first(tc)
Definition: tcp.h:434
u32 adj_index_t
An index for adjacencies.
Definition: adj_types.h:30
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:458
u16 mss
Option flags, see above.
Definition: tcp_packet.h:147
static void tcp_output_handle_packet(tcp_connection_t *tc0, vlib_buffer_t *b0, u32 *error0, u16 *next0, u8 is_ip4)
Definition: tcp_output.c:2183
static void * ip6_next_header(ip6_header_t *i)
Definition: ip6_packet.h:410
static void tcp_make_ack(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to ACK.
Definition: tcp_output.c:514
static int tcp_do_fastretransmit(tcp_connection_t *tc, u32 max_burst_size)
Definition: tcp_output.c:2043
static u32 transport_max_tx_dequeue(transport_connection_t *tc)
Definition: session.h:439
static void tcp_timer_update(tcp_connection_t *tc, u8 timer_id, u32 interval)
Definition: tcp.h:1082
u16 ip6_tcp_udp_icmp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip6_header_t *ip0, int *bogus_lengthp)
Definition: ip6_forward.c:906
signed int i32
Definition: types.h:77
vlib_node_registration_t ip6_lookup_node
(constructor) VLIB_REGISTER_NODE (ip6_lookup_node)
Definition: ip6_forward.c:552
static int tcp_make_established_options(tcp_connection_t *tc, tcp_options_t *opts)
Definition: tcp_output.c:330
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:513
#define ASSERT(truth)
static void tcp_cc_init_rxt_timeout(tcp_connection_t *tc)
Reset congestion control, switch cwnd to loss window and try again.
Definition: tcp_output.c:1438
static void tcp_output_push_ip(vlib_main_t *vm, vlib_buffer_t *b0, tcp_connection_t *tc0, u8 is_ip4)
Definition: tcp_output.c:2155
#define tcp_syn(_th)
Definition: tcp_packet.h:80
static u8 * format_tcp_tx_trace(u8 *s, va_list *args)
Definition: tcp_output.c:53
u16 ip4_tcp_udp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip4_header_t *ip0)
Definition: ip4_forward.c:1144
void tcp_update_burst_snd_vars(tcp_connection_t *tc)
Update burst send vars.
Definition: tcp_output.c:400
#define seq_geq(_s1, _s2)
Definition: tcp.h:825
void tcp_init_mss(tcp_connection_t *tc)
Definition: tcp_output.c:426
void tcp_program_fastretransmit(tcp_connection_t *tc)
Definition: tcp_output.c:1211
static uword ip6_address_is_link_local_unicast(const ip6_address_t *a)
Definition: ip6_packet.h:326
#define tcp_fastrecovery_first_off(tc)
Definition: tcp.h:436
static void tcp_update_rcv_wnd(tcp_connection_t *tc)
Definition: tcp_output.c:126
void tcp_send_fin(tcp_connection_t *tc)
Send FIN.
Definition: tcp_output.c:1031
#define clib_max(x, y)
Definition: clib.h:288
void tcp_send_ack(tcp_connection_t *tc)
Definition: tcp_output.c:1170
void tcp_cc_fastrecovery_clear(tcp_connection_t *tc)
Definition: tcp_input.c:1198
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
void transport_connection_tx_pacer_update_bytes(transport_connection_t *tc, u32 bytes)
Definition: transport.c:718
#define seq_lt(_s1, _s2)
Definition: tcp.h:822
int tcp_retransmit_first_unacked(tcp_worker_ctx_t *wrk, tcp_connection_t *tc)
Retransmit first unacked segment.
Definition: tcp_output.c:1761
template key/value backing page structure
Definition: bihash_doc.h:44
u32 ip_version_traffic_class_and_flow_label
Definition: ip6_packet.h:370
#define tcp_opts_wscale(_to)
Definition: tcp_packet.h:158
Definition: defs.h:47
void tcp_bt_check_app_limited(tcp_connection_t *tc)
Check if sample to be generated is app limited.
Definition: tcp_bt.c:251
u32 tsval
Timestamp value.
Definition: tcp_packet.h:149
u32 tsecr
Echoed/reflected time stamp.
Definition: tcp_packet.h:150
static void * vlib_buffer_push_ip6(vlib_main_t *vm, vlib_buffer_t *b, ip6_address_t *src, ip6_address_t *dst, int proto)
Push IPv6 header to buffer.
Definition: ip6.h:662
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
ip_lookup_next_t lookup_next_index
Next hop after ip4-lookup.
Definition: adj.h:236
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
#define foreach_tcp6_reset_next
Definition: tcp_output.c:2355
static int tcp_fast_retransmit_unsent(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 burst_size)
Definition: tcp_output.c:1780
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:489
static tcp_worker_ctx_t * tcp_get_worker(u32 thread_index)
Definition: tcp.h:645
static void tcp_retransmit_timer_update(tcp_connection_t *tc)
Definition: tcp.h:1139
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
#define seq_max(_s1, _s2)
Definition: tcp.h:826
static void tcp_enqueue_to_ip_lookup_i(tcp_worker_ctx_t *wrk, vlib_buffer_t *b, u32 bi, u8 is_ip4, u32 fib_index, u8 flush)
Definition: tcp_output.c:580
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
static void tcp_make_ack_i(tcp_connection_t *tc, vlib_buffer_t *b, tcp_state_t state, u8 flags)
Prepare ACK.
Definition: tcp_output.c:484
void tcp_timer_delack_handler(u32 index)
Delayed ack timer handler.
Definition: tcp_output.c:1226
#define TCP_OPTION_LEN_MSS
Definition: tcp_packet.h:165
void tcp_flush_frame_to_output(tcp_worker_ctx_t *wrk, u8 is_ip4)
Flush tx frame populated by retransmits and timer pops.
Definition: tcp_output.c:988
struct clib_bihash_value offset
template key/value backing page structure
static void tcp_retransmit_timer_force_update(tcp_connection_t *tc)
Definition: tcp.h:1111
int tcp_fast_retransmit(tcp_worker_ctx_t *wrk, tcp_connection_t *tc, u32 burst_size)
Do fast retransmit.
Definition: tcp_output.c:1985
u8 * format_tcp_connection(u8 *s, va_list *args)
Definition: tcp.c:954
u32 tcp_initial_window_to_advertise(tcp_connection_t *tc)
Compute initial window and scale factor.
Definition: tcp_output.c:114
#define vnet_buffer(b)
Definition: buffer.h:361
static tcp_connection_t * tcp_connection_get(u32 conn_index, u32 thread_index)
Definition: tcp.h:672
static void tcp_cc_event(tcp_connection_t *tc, tcp_cc_event_t evt)
Definition: tcp.h:1051
void tcp_update_rto(tcp_connection_t *tc)
Definition: tcp_input.c:469
int session_stream_connect_notify(transport_connection_t *tc, u8 is_fail)
Definition: session.c:729
static u32 vlib_num_workers()
Definition: threads.h:367
void tcp_connection_cleanup(tcp_connection_t *tc)
Cleans up connection state.
Definition: tcp.c:239
#define TCP_OPTION_LEN_NOOP
Definition: tcp_packet.h:164
void tcp_send_syn(tcp_connection_t *tc)
Send SYN.
Definition: tcp_output.c:926
vlib_node_registration_t tcp6_output_node
(constructor) VLIB_REGISTER_NODE (tcp6_output_node)
Definition: tcp_output.c:2325
u16 flags
Copy of main node flags.
Definition: node.h:507
Window scale.
Definition: tcp_packet.h:107
enum _tcp_reset_next tcp_reset_next_t
static u32 transport_max_rx_enqueue(transport_connection_t *tc)
Definition: session.h:432
#define tcp_opts_sack_permitted(_to)
Definition: tcp_packet.h:160
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:898
tcp_connection_t tcp_connection
Definition: tcp_output.c:49
void tcp_program_ack(tcp_connection_t *tc)
Definition: tcp_output.c:1189
u16 dst_port
Definition: udp.api:42
vlib_frame_t * ip_lookup_tx_frames[2]
tx frames for ip 4/6 lookup nodes
Definition: tcp.h:481
static void * tcp_reuse_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Definition: tcp_output.c:451
u8 ip_version_and_header_length
Definition: ip4_packet.h:138
Timestamps.
Definition: tcp_packet.h:110
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:244
static void tcp_enqueue_to_output_now(tcp_worker_ctx_t *wrk, vlib_buffer_t *b, u32 bi, u8 is_ip4)
Definition: tcp_output.c:671
vlib_node_registration_t tcp4_reset_node
(constructor) VLIB_REGISTER_NODE (tcp4_reset_node)
Definition: tcp_output.c:2442
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:301
vlib_node_registration_t tcp4_output_node
(constructor) VLIB_REGISTER_NODE (tcp4_output_node)
Definition: tcp_output.c:2305
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
static void tcp_enqueue_to_output(tcp_worker_ctx_t *wrk, vlib_buffer_t *b, u32 bi, u8 is_ip4)
Definition: tcp_output.c:664
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:612
static void tcp_persist_timer_set(tcp_connection_t *tc)
Definition: tcp.h:1118
static tcp_main_t * vnet_get_tcp_main()
Definition: tcp.h:639
#define TCP_RTO_BOFF_MAX
Definition: tcp.h:102
static char * tcp_error_strings[]
Definition: tcp_output.c:40
static void * vlib_buffer_push_ip4(vlib_main_t *vm, vlib_buffer_t *b, ip4_address_t *src, ip4_address_t *dst, int proto, u8 csum_offload)
Push IPv4 header to buffer.
Definition: ip4.h:387
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static u32 tcp_set_time_now(tcp_worker_ctx_t *wrk)
Definition: tcp.h:999
void tcp_timer_persist_handler(u32 index)
Got 0 snd_wnd from peer, try to do something about it.
Definition: tcp_output.c:1680
#define tcp_ack(_th)
Definition: tcp_packet.h:83
void tcp_bt_track_rxt(tcp_connection_t *tc, u32 start, u32 end)
Track a tcp retransmission.
Definition: tcp_bt.c:291
void tcp_bt_flush_samples(tcp_connection_t *tc)
Flush byte tracker samples.
Definition: tcp_bt.c:540
static u8 tcp_timer_is_active(tcp_connection_t *tc, tcp_timers_e timer)
Definition: tcp.h:1153
Definition: defs.h:46
ip6_address_t dst_address
Definition: ip6_packet.h:383
u32 * tx_buffers
tx buffer free list
Definition: tcp.h:475
adj_index_t adj_nbr_find(fib_protocol_t nh_proto, vnet_link_t link_type, const ip46_address_t *nh_addr, u32 sw_if_index)
Lookup neighbor adjancency.
Definition: adj_nbr.c:99
static int tcp_make_options(tcp_connection_t *tc, tcp_options_t *opts, tcp_state_t state)
Definition: tcp_output.c:365