FD.io VPP  v18.10-32-g1161dda
Vector Packet Processing
tcp_input.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vppinfra/sparse_vec.h>
17 #include <vnet/tcp/tcp_packet.h>
18 #include <vnet/tcp/tcp.h>
19 #include <vnet/session/session.h>
20 #include <math.h>
21 
22 static char *tcp_error_strings[] = {
23 #define tcp_error(n,s) s,
24 #include <vnet/tcp/tcp_error.def>
25 #undef tcp_error
26 };
27 
28 /* All TCP nodes have the same outgoing arcs */
29 #define foreach_tcp_state_next \
30  _ (DROP4, "ip4-drop") \
31  _ (DROP6, "ip6-drop") \
32  _ (TCP4_OUTPUT, "tcp4-output") \
33  _ (TCP6_OUTPUT, "tcp6-output")
34 
35 typedef enum _tcp_established_next
36 {
37 #define _(s,n) TCP_ESTABLISHED_NEXT_##s,
39 #undef _
42 
43 typedef enum _tcp_rcv_process_next
44 {
45 #define _(s,n) TCP_RCV_PROCESS_NEXT_##s,
47 #undef _
50 
51 typedef enum _tcp_syn_sent_next
52 {
53 #define _(s,n) TCP_SYN_SENT_NEXT_##s,
55 #undef _
58 
59 typedef enum _tcp_listen_next
60 {
61 #define _(s,n) TCP_LISTEN_NEXT_##s,
63 #undef _
66 
67 /* Generic, state independent indices */
68 typedef enum _tcp_state_next
69 {
70 #define _(s,n) TCP_NEXT_##s,
72 #undef _
75 
76 #define tcp_next_output(is_ip4) (is_ip4 ? TCP_NEXT_TCP4_OUTPUT \
77  : TCP_NEXT_TCP6_OUTPUT)
78 
79 #define tcp_next_drop(is_ip4) (is_ip4 ? TCP_NEXT_DROP4 \
80  : TCP_NEXT_DROP6)
81 
84 
85 /**
86  * Validate segment sequence number. As per RFC793:
87  *
88  * Segment Receive Test
89  * Length Window
90  * ------- ------- -------------------------------------------
91  * 0 0 SEG.SEQ = RCV.NXT
92  * 0 >0 RCV.NXT =< SEG.SEQ < RCV.NXT+RCV.WND
93  * >0 0 not acceptable
94  * >0 >0 RCV.NXT =< SEG.SEQ < RCV.NXT+RCV.WND
95  * or RCV.NXT =< SEG.SEQ+SEG.LEN-1 < RCV.NXT+RCV.WND
96  *
97  * This ultimately consists in checking if segment falls within the window.
98  * The one important difference compared to RFC793 is that we use rcv_las,
99  * or the rcv_nxt at last ack sent instead of rcv_nxt since that's the
100  * peer's reference when computing our receive window.
101  *
102  * This:
103  * seq_leq (end_seq, tc->rcv_las + tc->rcv_wnd) && seq_geq (seq, tc->rcv_las)
104  * however, is too strict when we have retransmits. Instead we just check that
105  * the seq is not beyond the right edge and that the end of the segment is not
106  * less than the left edge.
107  *
108  * N.B. rcv_nxt and rcv_wnd are both updated in this node if acks are sent, so
109  * use rcv_nxt in the right edge window test instead of rcv_las.
110  *
111  */
114 {
115  return (seq_geq (end_seq, tc->rcv_las)
116  && seq_leq (seq, tc->rcv_nxt + tc->rcv_wnd));
117 }
118 
119 /**
120  * Parse TCP header options.
121  *
122  * @param th TCP header
123  * @param to TCP options data structure to be populated
124  * @return -1 if parsing failed
125  */
126 static int
128 {
129  const u8 *data;
130  u8 opt_len, opts_len, kind;
131  int j;
132  sack_block_t b;
133 
134  opts_len = (tcp_doff (th) << 2) - sizeof (tcp_header_t);
135  data = (const u8 *) (th + 1);
136 
137  /* Zero out all flags but those set in SYN */
138  to->flags &= (TCP_OPTS_FLAG_SACK_PERMITTED | TCP_OPTS_FLAG_WSCALE
139  | TCP_OPTS_FLAG_SACK);
140 
141  for (; opts_len > 0; opts_len -= opt_len, data += opt_len)
142  {
143  kind = data[0];
144 
145  /* Get options length */
146  if (kind == TCP_OPTION_EOL)
147  break;
148  else if (kind == TCP_OPTION_NOOP)
149  {
150  opt_len = 1;
151  continue;
152  }
153  else
154  {
155  /* broken options */
156  if (opts_len < 2)
157  return -1;
158  opt_len = data[1];
159 
160  /* weird option length */
161  if (opt_len < 2 || opt_len > opts_len)
162  return -1;
163  }
164 
165  /* Parse options */
166  switch (kind)
167  {
168  case TCP_OPTION_MSS:
169  if ((opt_len == TCP_OPTION_LEN_MSS) && tcp_syn (th))
170  {
171  to->flags |= TCP_OPTS_FLAG_MSS;
172  to->mss = clib_net_to_host_u16 (*(u16 *) (data + 2));
173  }
174  break;
176  if ((opt_len == TCP_OPTION_LEN_WINDOW_SCALE) && tcp_syn (th))
177  {
178  to->flags |= TCP_OPTS_FLAG_WSCALE;
179  to->wscale = data[2];
180  if (to->wscale > TCP_MAX_WND_SCALE)
181  {
182  clib_warning ("Illegal window scaling value: %d",
183  to->wscale);
185  }
186  }
187  break;
189  if (opt_len == TCP_OPTION_LEN_TIMESTAMP)
190  {
191  to->flags |= TCP_OPTS_FLAG_TSTAMP;
192  to->tsval = clib_net_to_host_u32 (*(u32 *) (data + 2));
193  to->tsecr = clib_net_to_host_u32 (*(u32 *) (data + 6));
194  }
195  break;
197  if (opt_len == TCP_OPTION_LEN_SACK_PERMITTED && tcp_syn (th))
198  to->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
199  break;
201  /* If SACK permitted was not advertised or a SYN, break */
202  if ((to->flags & TCP_OPTS_FLAG_SACK_PERMITTED) == 0 || tcp_syn (th))
203  break;
204 
205  /* If too short or not correctly formatted, break */
206  if (opt_len < 10 || ((opt_len - 2) % TCP_OPTION_LEN_SACK_BLOCK))
207  break;
208 
209  to->flags |= TCP_OPTS_FLAG_SACK;
210  to->n_sack_blocks = (opt_len - 2) / TCP_OPTION_LEN_SACK_BLOCK;
211  vec_reset_length (to->sacks);
212  for (j = 0; j < to->n_sack_blocks; j++)
213  {
214  b.start = clib_net_to_host_u32 (*(u32 *) (data + 2 + 8 * j));
215  b.end = clib_net_to_host_u32 (*(u32 *) (data + 6 + 8 * j));
216  vec_add1 (to->sacks, b);
217  }
218  break;
219  default:
220  /* Nothing to see here */
221  continue;
222  }
223  }
224  return 0;
225 }
226 
227 /**
228  * RFC1323: Check against wrapped sequence numbers (PAWS). If we have
229  * timestamp to echo and it's less than tsval_recent, drop segment
230  * but still send an ACK in order to retain TCP's mechanism for detecting
231  * and recovering from half-open connections
232  *
233  * Or at least that's what the theory says. It seems that this might not work
234  * very well with packet reordering and fast retransmit. XXX
235  */
236 always_inline int
238 {
239  return tcp_opts_tstamp (&tc->rcv_opts) && tc->tsval_recent
240  && timestamp_lt (tc->rcv_opts.tsval, tc->tsval_recent);
241 }
242 
243 /**
244  * Update tsval recent
245  */
246 always_inline void
248 {
249  /*
250  * RFC1323: If Last.ACK.sent falls within the range of sequence numbers
251  * of an incoming segment:
252  * SEG.SEQ <= Last.ACK.sent < SEG.SEQ + SEG.LEN
253  * then the TSval from the segment is copied to TS.Recent;
254  * otherwise, the TSval is ignored.
255  */
256  if (tcp_opts_tstamp (&tc->rcv_opts) && seq_leq (seq, tc->rcv_las)
257  && seq_leq (tc->rcv_las, seq_end))
258  {
259  ASSERT (timestamp_leq (tc->tsval_recent, tc->rcv_opts.tsval));
260  tc->tsval_recent = tc->rcv_opts.tsval;
261  tc->tsval_recent_age = tcp_time_now ();
262  }
263 }
264 
265 /**
266  * Validate incoming segment as per RFC793 p. 69 and RFC1323 p. 19
267  *
268  * It first verifies if segment has a wrapped sequence number (PAWS) and then
269  * does the processing associated to the first four steps (ignoring security
270  * and precedence): sequence number, rst bit and syn bit checks.
271  *
272  * @return 0 if segments passes validation.
273  */
274 static int
276  vlib_buffer_t * b0, tcp_header_t * th0,
277  u32 * next0, u32 * error0)
278 {
279  /* We could get a burst of RSTs interleaved with acks */
280  if (PREDICT_FALSE (tc0->state == TCP_STATE_CLOSED))
281  {
282  tcp_send_reset (tc0);
283  *error0 = TCP_ERROR_CONNECTION_CLOSED;
284  goto drop;
285  }
286 
287  if (PREDICT_FALSE (!tcp_ack (th0) && !tcp_rst (th0) && !tcp_syn (th0)))
288  {
289  *error0 = TCP_ERROR_SEGMENT_INVALID;
290  goto drop;
291  }
292 
293  if (PREDICT_FALSE (tcp_options_parse (th0, &tc0->rcv_opts)))
294  {
295  clib_warning ("options parse error");
296  *error0 = TCP_ERROR_OPTIONS;
297  goto drop;
298  }
299 
301  {
302  *error0 = TCP_ERROR_PAWS;
303  if (CLIB_DEBUG > 2)
304  clib_warning ("paws failed\n%U", format_tcp_connection, tc0, 2);
305  TCP_EVT_DBG (TCP_EVT_PAWS_FAIL, tc0, vnet_buffer (b0)->tcp.seq_number,
306  vnet_buffer (b0)->tcp.seq_end);
307 
308  /* If it just so happens that a segment updates tsval_recent for a
309  * segment over 24 days old, invalidate tsval_recent. */
310  if (timestamp_lt (tc0->tsval_recent_age + TCP_PAWS_IDLE,
311  tcp_time_now ()))
312  {
313  /* Age isn't reset until we get a valid tsval (bsd inspired) */
314  tc0->tsval_recent = 0;
315  clib_warning ("paws failed - really old segment. REALLY?");
316  }
317  else
318  {
319  /* Drop after ack if not rst */
320  if (!tcp_rst (th0))
321  {
322  tcp_make_ack (tc0, b0);
323  TCP_EVT_DBG (TCP_EVT_DUPACK_SENT, tc0, vnet_buffer (b0)->tcp);
324  goto error;
325  }
326  }
327  goto drop;
328  }
329 
330  /* 1st: check sequence number */
331  if (!tcp_segment_in_rcv_wnd (tc0, vnet_buffer (b0)->tcp.seq_number,
332  vnet_buffer (b0)->tcp.seq_end))
333  {
334  *error0 = TCP_ERROR_RCV_WND;
335  /* If our window is 0 and the packet is in sequence, let it pass
336  * through for ack processing. It should be dropped later. */
337  if (!(tc0->rcv_wnd == 0
338  && tc0->rcv_nxt == vnet_buffer (b0)->tcp.seq_number))
339  {
340  /* If not RST, send dup ack */
341  if (!tcp_rst (th0))
342  {
343  tcp_make_ack (tc0, b0);
344  TCP_EVT_DBG (TCP_EVT_DUPACK_SENT, tc0, vnet_buffer (b0)->tcp);
345  goto error;
346  }
347  goto drop;
348  }
349  }
350 
351  /* 2nd: check the RST bit */
352  if (PREDICT_FALSE (tcp_rst (th0)))
353  {
354  tcp_connection_reset (tc0);
355  *error0 = TCP_ERROR_RST_RCVD;
356  goto drop;
357  }
358 
359  /* 3rd: check security and precedence (skip) */
360 
361  /* 4th: check the SYN bit */
362  if (PREDICT_FALSE (tcp_syn (th0)))
363  {
364  /* TODO implement RFC 5961 */
365  if (tc0->state == TCP_STATE_SYN_RCVD)
366  {
367  tcp_make_synack (tc0, b0);
368  TCP_EVT_DBG (TCP_EVT_SYN_RCVD, tc0, 0);
369  }
370  else
371  {
372  tcp_make_ack (tc0, b0);
373  TCP_EVT_DBG (TCP_EVT_SYNACK_RCVD, tc0);
374  }
375  goto error;
376  }
377 
378  /* If segment in window, save timestamp */
379  tcp_update_timestamp (tc0, vnet_buffer (b0)->tcp.seq_number,
380  vnet_buffer (b0)->tcp.seq_end);
381  return 0;
382 
383 drop:
384  *next0 = tcp_next_drop (tc0->c_is_ip4);
385  return -1;
386 error:
387  *next0 = tcp_next_output (tc0->c_is_ip4);
388  return -1;
389 }
390 
391 always_inline int
393 {
394  /* SND.UNA =< SEG.ACK =< SND.NXT */
395  return (seq_leq (tc0->snd_una, vnet_buffer (tb0)->tcp.ack_number)
396  && seq_leq (vnet_buffer (tb0)->tcp.ack_number, tc0->snd_nxt));
397 }
398 
399 /**
400  * Compute smoothed RTT as per VJ's '88 SIGCOMM and RFC6298
401  *
402  * Note that although the original article, srtt and rttvar are scaled
403  * to minimize round-off errors, here we don't. Instead, we rely on
404  * better precision time measurements.
405  *
406  * TODO support us rtt resolution
407  */
408 static void
410 {
411  int err, diff;
412 
413  if (tc->srtt != 0)
414  {
415  err = mrtt - tc->srtt;
416 
417  /* XXX Drop in RTT results in RTTVAR increase and bigger RTO.
418  * The increase should be bound */
419  tc->srtt = clib_max ((int) tc->srtt + (err >> 3), 1);
420  diff = (clib_abs (err) - (int) tc->rttvar) >> 2;
421  tc->rttvar = clib_max ((int) tc->rttvar + diff, 1);
422  }
423  else
424  {
425  /* First measurement. */
426  tc->srtt = mrtt;
427  tc->rttvar = mrtt >> 1;
428  }
429 }
430 
431 void
433 {
434  tc->rto = clib_min (tc->srtt + (tc->rttvar << 2), TCP_RTO_MAX);
435  tc->rto = clib_max (tc->rto, TCP_RTO_MIN);
436 }
437 
438 /**
439  * Update RTT estimate and RTO timer
440  *
441  * Measure RTT: We have two sources of RTT measurements: TSOPT and ACK
442  * timing. Middle boxes are known to fiddle with TCP options so we
443  * should give higher priority to ACK timing.
444  *
445  * This should be called only if previously sent bytes have been acked.
446  *
447  * return 1 if valid rtt 0 otherwise
448  */
449 static int
451 {
452  u32 mrtt = 0;
453 
454  /* Karn's rule, part 1. Don't use retransmitted segments to estimate
455  * RTT because they're ambiguous. */
456  if (tcp_in_cong_recovery (tc) || tc->sack_sb.sacked_bytes)
457  {
458  if (tcp_in_recovery (tc))
459  return 0;
460  goto done;
461  }
462 
463  if (tc->rtt_ts && seq_geq (ack, tc->rtt_seq))
464  {
465  mrtt = tcp_time_now () - tc->rtt_ts;
466  }
467  /* As per RFC7323 TSecr can be used for RTTM only if the segment advances
468  * snd_una, i.e., the left side of the send window:
469  * seq_lt (tc->snd_una, ack). This is a condition for calling update_rtt */
470  else if (tcp_opts_tstamp (&tc->rcv_opts) && tc->rcv_opts.tsecr)
471  {
472  mrtt = tcp_time_now () - tc->rcv_opts.tsecr;
473  }
474 
475  /* Ignore dubious measurements */
476  if (mrtt == 0 || mrtt > TCP_RTT_MAX)
477  goto done;
478 
479  tcp_estimate_rtt (tc, mrtt);
480 
481 done:
482 
483  /* Allow measuring of a new RTT */
484  tc->rtt_ts = 0;
485 
486  /* If we got here something must've been ACKed so make sure boff is 0,
487  * even if mrtt is not valid since we update the rto lower */
488  tc->rto_boff = 0;
489  tcp_update_rto (tc);
490 
491  return 0;
492 }
493 
494 /**
495  * Dequeue bytes that have been acked and while at it update RTT estimates.
496  */
497 static void
499 {
500  /* Dequeue the newly ACKed add SACKed bytes */
501  stream_session_dequeue_drop (&tc->connection,
502  tc->bytes_acked + tc->sack_sb.snd_una_adv);
503 
504  tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
505 
506  /* Update rtt and rto */
507  tcp_update_rtt (tc, ack);
508 
509  /* If everything has been acked, stop retransmit timer
510  * otherwise update. */
512 }
513 
514 /**
515  * Check if duplicate ack as per RFC5681 Sec. 2
516  */
517 static u8
519  u32 prev_snd_una)
520 {
521  return ((vnet_buffer (b)->tcp.ack_number == prev_snd_una)
522  && seq_gt (tc->snd_una_max, tc->snd_una)
523  && (vnet_buffer (b)->tcp.seq_end == vnet_buffer (b)->tcp.seq_number)
524  && (prev_snd_wnd == tc->snd_wnd));
525 }
526 
527 /**
528  * Checks if ack is a congestion control event.
529  */
530 static u8
532  u32 prev_snd_wnd, u32 prev_snd_una, u8 * is_dack)
533 {
534  /* Check if ack is duplicate. Per RFC 6675, ACKs that SACK new data are
535  * defined to be 'duplicate' */
536  *is_dack = tc->sack_sb.last_sacked_bytes
537  || tcp_ack_is_dupack (tc, b, prev_snd_wnd, prev_snd_una);
538 
539  return ((*is_dack || tcp_in_cong_recovery (tc)) && !tcp_is_lost_fin (tc));
540 }
541 
542 static u32
544 {
545  ASSERT (!pool_is_free_index (sb->holes, hole - sb->holes));
546  return hole - sb->holes;
547 }
548 
549 static u32
551 {
552  return hole->end - hole->start;
553 }
554 
557 {
558  if (index != TCP_INVALID_SACK_HOLE_INDEX)
559  return pool_elt_at_index (sb->holes, index);
560  return 0;
561 }
562 
565 {
566  if (hole->next != TCP_INVALID_SACK_HOLE_INDEX)
567  return pool_elt_at_index (sb->holes, hole->next);
568  return 0;
569 }
570 
573 {
574  if (hole->prev != TCP_INVALID_SACK_HOLE_INDEX)
575  return pool_elt_at_index (sb->holes, hole->prev);
576  return 0;
577 }
578 
581 {
582  if (sb->head != TCP_INVALID_SACK_HOLE_INDEX)
583  return pool_elt_at_index (sb->holes, sb->head);
584  return 0;
585 }
586 
589 {
590  if (sb->tail != TCP_INVALID_SACK_HOLE_INDEX)
591  return pool_elt_at_index (sb->holes, sb->tail);
592  return 0;
593 }
594 
595 static void
597 {
598  sack_scoreboard_hole_t *next, *prev;
599 
600  if (hole->next != TCP_INVALID_SACK_HOLE_INDEX)
601  {
602  next = pool_elt_at_index (sb->holes, hole->next);
603  next->prev = hole->prev;
604  }
605  else
606  {
607  sb->tail = hole->prev;
608  }
609 
610  if (hole->prev != TCP_INVALID_SACK_HOLE_INDEX)
611  {
612  prev = pool_elt_at_index (sb->holes, hole->prev);
613  prev->next = hole->next;
614  }
615  else
616  {
617  sb->head = hole->next;
618  }
619 
620  if (scoreboard_hole_index (sb, hole) == sb->cur_rxt_hole)
621  sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
622 
623  /* Poison the entry */
624  if (CLIB_DEBUG > 0)
625  memset (hole, 0xfe, sizeof (*hole));
626 
627  pool_put (sb->holes, hole);
628 }
629 
630 static sack_scoreboard_hole_t *
632  u32 start, u32 end)
633 {
634  sack_scoreboard_hole_t *hole, *next, *prev;
635  u32 hole_index;
636 
637  pool_get (sb->holes, hole);
638  memset (hole, 0, sizeof (*hole));
639 
640  hole->start = start;
641  hole->end = end;
642  hole_index = scoreboard_hole_index (sb, hole);
643 
644  prev = scoreboard_get_hole (sb, prev_index);
645  if (prev)
646  {
647  hole->prev = prev_index;
648  hole->next = prev->next;
649 
650  if ((next = scoreboard_next_hole (sb, hole)))
651  next->prev = hole_index;
652  else
653  sb->tail = hole_index;
654 
655  prev->next = hole_index;
656  }
657  else
658  {
659  sb->head = hole_index;
660  hole->prev = TCP_INVALID_SACK_HOLE_INDEX;
661  hole->next = TCP_INVALID_SACK_HOLE_INDEX;
662  }
663 
664  return hole;
665 }
666 
667 static void
669 {
671  u32 bytes = 0, blks = 0;
672 
673  sb->lost_bytes = 0;
674  sb->sacked_bytes = 0;
675  left = scoreboard_last_hole (sb);
676  if (!left)
677  return;
678 
679  if (seq_gt (sb->high_sacked, left->end))
680  {
681  bytes = sb->high_sacked - left->end;
682  blks = 1;
683  }
684 
685  while ((right = left)
686  && bytes < (TCP_DUPACK_THRESHOLD - 1) * tc->snd_mss
687  && blks < TCP_DUPACK_THRESHOLD
688  /* left not updated if above conditions fail */
689  && (left = scoreboard_prev_hole (sb, right)))
690  {
691  bytes += right->start - left->end;
692  blks++;
693  }
694 
695  /* left is first lost */
696  if (left)
697  {
698  do
699  {
700  sb->lost_bytes += scoreboard_hole_bytes (right);
701  left->is_lost = 1;
702  left = scoreboard_prev_hole (sb, right);
703  if (left)
704  bytes += right->start - left->end;
705  }
706  while ((right = left));
707  }
708 
709  sb->sacked_bytes = bytes;
710 }
711 
712 /**
713  * Figure out the next hole to retransmit
714  *
715  * Follows logic proposed in RFC6675 Sec. 4, NextSeg()
716  */
719  sack_scoreboard_hole_t * start,
720  u8 have_sent_1_smss,
721  u8 * can_rescue, u8 * snd_limited)
722 {
723  sack_scoreboard_hole_t *hole = 0;
724 
725  hole = start ? start : scoreboard_first_hole (sb);
726  while (hole && seq_leq (hole->end, sb->high_rxt) && hole->is_lost)
727  hole = scoreboard_next_hole (sb, hole);
728 
729  /* Nothing, return */
730  if (!hole)
731  {
732  sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
733  return 0;
734  }
735 
736  /* Rule (1): if higher than rxt, less than high_sacked and lost */
737  if (hole->is_lost && seq_lt (hole->start, sb->high_sacked))
738  {
739  sb->cur_rxt_hole = scoreboard_hole_index (sb, hole);
740  }
741  else
742  {
743  /* Rule (2): output takes care of transmitting new data */
744  if (!have_sent_1_smss)
745  {
746  hole = 0;
747  sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
748  }
749  /* Rule (3): if hole not lost */
750  else if (seq_lt (hole->start, sb->high_sacked))
751  {
752  *snd_limited = 1;
753  sb->cur_rxt_hole = scoreboard_hole_index (sb, hole);
754  }
755  /* Rule (4): if hole beyond high_sacked */
756  else
757  {
758  ASSERT (seq_geq (hole->start, sb->high_sacked));
759  *snd_limited = 1;
760  *can_rescue = 1;
761  /* HighRxt MUST NOT be updated */
762  return 0;
763  }
764  }
765 
766  if (hole && seq_lt (sb->high_rxt, hole->start))
767  sb->high_rxt = hole->start;
768 
769  return hole;
770 }
771 
772 static void
774 {
776  hole = scoreboard_first_hole (sb);
777  if (hole)
778  {
779  seq = seq_gt (seq, hole->start) ? seq : hole->start;
780  sb->cur_rxt_hole = sb->head;
781  }
782  sb->high_rxt = seq;
783 }
784 
785 void
787 {
788  sb->head = TCP_INVALID_SACK_HOLE_INDEX;
789  sb->tail = TCP_INVALID_SACK_HOLE_INDEX;
790  sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
791 }
792 
793 void
795 {
797  while ((hole = scoreboard_first_hole (sb)))
798  {
799  scoreboard_remove_hole (sb, hole);
800  }
801  ASSERT (sb->head == sb->tail && sb->head == TCP_INVALID_SACK_HOLE_INDEX);
802  ASSERT (pool_elts (sb->holes) == 0);
803  sb->sacked_bytes = 0;
804  sb->last_sacked_bytes = 0;
805  sb->last_bytes_delivered = 0;
806  sb->snd_una_adv = 0;
807  sb->high_sacked = 0;
808  sb->high_rxt = 0;
809  sb->lost_bytes = 0;
810  sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
811 }
812 
813 /**
814  * Test that scoreboard is sane after recovery
815  *
816  * Returns 1 if scoreboard is empty or if first hole beyond
817  * snd_una.
818  */
819 static u8
821 {
823  hole = scoreboard_first_hole (&tc->sack_sb);
824  return (!hole || (seq_geq (hole->start, tc->snd_una)
825  && seq_lt (hole->end, tc->snd_una_max)));
826 }
827 
828 void
830 {
831  sack_scoreboard_t *sb = &tc->sack_sb;
832  sack_block_t *blk, tmp;
833  sack_scoreboard_hole_t *hole, *next_hole, *last_hole;
834  u32 blk_index = 0, old_sacked_bytes, hole_index;
835  int i, j;
836 
837  sb->last_sacked_bytes = 0;
838  sb->last_bytes_delivered = 0;
839  sb->snd_una_adv = 0;
840 
841  if (!tcp_opts_sack (&tc->rcv_opts)
842  && sb->head == TCP_INVALID_SACK_HOLE_INDEX)
843  return;
844 
845  old_sacked_bytes = sb->sacked_bytes;
846 
847  /* Remove invalid blocks */
848  blk = tc->rcv_opts.sacks;
849  while (blk < vec_end (tc->rcv_opts.sacks))
850  {
851  if (seq_lt (blk->start, blk->end)
852  && seq_gt (blk->start, tc->snd_una)
853  && seq_gt (blk->start, ack) && seq_leq (blk->end, tc->snd_una_max))
854  {
855  blk++;
856  continue;
857  }
858  vec_del1 (tc->rcv_opts.sacks, blk - tc->rcv_opts.sacks);
859  }
860 
861  /* Add block for cumulative ack */
862  if (seq_gt (ack, tc->snd_una))
863  {
864  tmp.start = tc->snd_una;
865  tmp.end = ack;
866  vec_add1 (tc->rcv_opts.sacks, tmp);
867  }
868 
869  if (vec_len (tc->rcv_opts.sacks) == 0)
870  return;
871 
872  tcp_scoreboard_trace_add (tc, ack);
873 
874  /* Make sure blocks are ordered */
875  for (i = 0; i < vec_len (tc->rcv_opts.sacks); i++)
876  for (j = i + 1; j < vec_len (tc->rcv_opts.sacks); j++)
877  if (seq_lt (tc->rcv_opts.sacks[j].start, tc->rcv_opts.sacks[i].start))
878  {
879  tmp = tc->rcv_opts.sacks[i];
880  tc->rcv_opts.sacks[i] = tc->rcv_opts.sacks[j];
881  tc->rcv_opts.sacks[j] = tmp;
882  }
883 
884  if (sb->head == TCP_INVALID_SACK_HOLE_INDEX)
885  {
886  /* If no holes, insert the first that covers all outstanding bytes */
888  tc->snd_una, tc->snd_una_max);
889  sb->tail = scoreboard_hole_index (sb, last_hole);
890  tmp = tc->rcv_opts.sacks[vec_len (tc->rcv_opts.sacks) - 1];
891  sb->high_sacked = tmp.end;
892  }
893  else
894  {
895  /* If we have holes but snd_una_max is beyond the last hole, update
896  * last hole end */
897  tmp = tc->rcv_opts.sacks[vec_len (tc->rcv_opts.sacks) - 1];
898  last_hole = scoreboard_last_hole (sb);
899  if (seq_gt (tc->snd_una_max, last_hole->end))
900  {
901  if (seq_geq (last_hole->start, sb->high_sacked))
902  {
903  last_hole->end = tc->snd_una_max;
904  }
905  /* New hole after high sacked block */
906  else if (seq_lt (sb->high_sacked, tc->snd_una_max))
907  {
908  scoreboard_insert_hole (sb, sb->tail, sb->high_sacked,
909  tc->snd_una_max);
910  }
911  }
912  /* Keep track of max byte sacked for when the last hole
913  * is acked */
914  if (seq_gt (tmp.end, sb->high_sacked))
915  sb->high_sacked = tmp.end;
916  }
917 
918  /* Walk the holes with the SACK blocks */
919  hole = pool_elt_at_index (sb->holes, sb->head);
920  while (hole && blk_index < vec_len (tc->rcv_opts.sacks))
921  {
922  blk = &tc->rcv_opts.sacks[blk_index];
923  if (seq_leq (blk->start, hole->start))
924  {
925  /* Block covers hole. Remove hole */
926  if (seq_geq (blk->end, hole->end))
927  {
928  next_hole = scoreboard_next_hole (sb, hole);
929 
930  /* Byte accounting: snd_una needs to be advanced */
931  if (blk->end == ack)
932  {
933  if (next_hole)
934  {
935  if (seq_lt (ack, next_hole->start))
936  sb->snd_una_adv = next_hole->start - ack;
937  sb->last_bytes_delivered +=
938  next_hole->start - hole->end;
939  }
940  else
941  {
942  ASSERT (seq_geq (sb->high_sacked, ack));
943  sb->snd_una_adv = sb->high_sacked - ack;
944  sb->last_bytes_delivered += sb->high_sacked - hole->end;
945  }
946  }
947 
948  scoreboard_remove_hole (sb, hole);
949  hole = next_hole;
950  }
951  /* Partial 'head' overlap */
952  else
953  {
954  if (seq_gt (blk->end, hole->start))
955  {
956  hole->start = blk->end;
957  }
958  blk_index++;
959  }
960  }
961  else
962  {
963  /* Hole must be split */
964  if (seq_lt (blk->end, hole->end))
965  {
966  hole_index = scoreboard_hole_index (sb, hole);
967  next_hole = scoreboard_insert_hole (sb, hole_index, blk->end,
968  hole->end);
969 
970  /* Pool might've moved */
971  hole = scoreboard_get_hole (sb, hole_index);
972  hole->end = blk->start;
973  blk_index++;
974  ASSERT (hole->next == scoreboard_hole_index (sb, next_hole));
975  }
976  else if (seq_lt (blk->start, hole->end))
977  {
978  hole->end = blk->start;
979  }
980  hole = scoreboard_next_hole (sb, hole);
981  }
982  }
983 
984  if (pool_elts (sb->holes) == 1)
985  {
986  hole = scoreboard_first_hole (sb);
987  if (hole->start == ack + sb->snd_una_adv
988  && hole->end == tc->snd_una_max)
989  scoreboard_remove_hole (sb, hole);
990  }
991 
992  scoreboard_update_bytes (tc, sb);
993  sb->last_sacked_bytes = sb->sacked_bytes
994  - (old_sacked_bytes - sb->last_bytes_delivered);
995  ASSERT (sb->last_sacked_bytes <= sb->sacked_bytes || tcp_in_recovery (tc));
996  ASSERT (sb->sacked_bytes == 0
997  || sb->sacked_bytes < tc->snd_una_max - seq_max (tc->snd_una, ack));
998  ASSERT (sb->last_sacked_bytes + sb->lost_bytes <= tc->snd_una_max
999  - seq_max (tc->snd_una, ack));
1001  || sb->holes[sb->head].start == ack + sb->snd_una_adv);
1002  TCP_EVT_DBG (TCP_EVT_CC_SCOREBOARD, tc);
1003 }
1004 
1005 /**
1006  * Try to update snd_wnd based on feedback received from peer.
1007  *
1008  * If successful, and new window is 'effectively' 0, activate persist
1009  * timer.
1010  */
1011 static void
1012 tcp_update_snd_wnd (tcp_connection_t * tc, u32 seq, u32 ack, u32 snd_wnd)
1013 {
1014  /* If (SND.WL1 < SEG.SEQ or (SND.WL1 = SEG.SEQ and SND.WL2 =< SEG.ACK)), set
1015  * SND.WND <- SEG.WND, set SND.WL1 <- SEG.SEQ, and set SND.WL2 <- SEG.ACK */
1016  if (seq_lt (tc->snd_wl1, seq)
1017  || (tc->snd_wl1 == seq && seq_leq (tc->snd_wl2, ack)))
1018  {
1019  tc->snd_wnd = snd_wnd;
1020  tc->snd_wl1 = seq;
1021  tc->snd_wl2 = ack;
1022  TCP_EVT_DBG (TCP_EVT_SND_WND, tc);
1023 
1024  if (tc->snd_wnd < tc->snd_mss)
1025  {
1026  /* Set persist timer if not set and we just got 0 wnd */
1027  if (!tcp_timer_is_active (tc, TCP_TIMER_PERSIST)
1028  && !tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT))
1029  tcp_persist_timer_set (tc);
1030  }
1031  else
1032  {
1034  if (!tcp_in_recovery (tc) && tc->rto_boff > 0)
1035  {
1036  tc->rto_boff = 0;
1037  tcp_update_rto (tc);
1038  }
1039  }
1040  }
1041 }
1042 
1043 /**
1044  * Init loss recovery/fast recovery.
1045  *
1046  * Triggered by dup acks as opposed to timer timeout. Note that cwnd is
1047  * updated in @ref tcp_cc_handle_event after fast retransmit
1048  */
1049 void
1051 {
1052  tcp_fastrecovery_on (tc);
1053  tc->snd_congestion = tc->snd_una_max;
1054  tc->cwnd_acc_bytes = 0;
1055  tc->cc_algo->congestion (tc);
1056  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 4);
1057 }
1058 
1059 static void
1061 {
1062  tc->rto_boff = 0;
1063  tcp_update_rto (tc);
1064  tc->snd_rxt_ts = 0;
1065  tc->snd_nxt = tc->snd_una_max;
1066  tcp_recovery_off (tc);
1067  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 3);
1068 }
1069 
1070 void
1072 {
1073  tc->cc_algo->recovered (tc);
1074  tc->snd_rxt_bytes = 0;
1075  tc->rcv_dupacks = 0;
1076  tc->snd_nxt = tc->snd_una_max;
1077  tcp_fastrecovery_off (tc);
1079  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 3);
1080 }
1081 
1082 static void
1084 {
1085  tc->cwnd = tc->prev_cwnd;
1086  tc->ssthresh = tc->prev_ssthresh;
1087  tc->snd_nxt = tc->snd_una_max;
1088  tc->rcv_dupacks = 0;
1089  if (tcp_in_recovery (tc))
1090  tcp_cc_recovery_exit (tc);
1091  ASSERT (tc->rto_boff == 0);
1092  TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 5);
1093  /* TODO extend for fastrecovery */
1094 }
1095 
1096 static u8
1098 {
1099  return (tcp_in_recovery (tc) && tc->rto_boff == 1
1100  && tc->snd_rxt_ts
1101  && tcp_opts_tstamp (&tc->rcv_opts)
1102  && timestamp_lt (tc->rcv_opts.tsecr, tc->snd_rxt_ts));
1103 }
1104 
1105 static int
1107 {
1110  {
1112  return 1;
1113  }
1114 
1115  if (tcp_in_recovery (tc))
1116  tcp_cc_recovery_exit (tc);
1117  else if (tcp_in_fastrecovery (tc))
1119 
1120  ASSERT (tc->rto_boff == 0);
1121  ASSERT (!tcp_in_cong_recovery (tc));
1123  return 0;
1124 }
1125 
1126 static void
1128 {
1130 
1131  /* Congestion avoidance */
1132  tc->cc_algo->rcv_ack (tc);
1133  tc->tsecr_last_ack = tc->rcv_opts.tsecr;
1134 
1135  /* If a cumulative ack, make sure dupacks is 0 */
1136  tc->rcv_dupacks = 0;
1137 
1138  /* When dupacks hits the threshold we only enter fast retransmit if
1139  * cumulative ack covers more than snd_congestion. Should snd_una
1140  * wrap this test may fail under otherwise valid circumstances.
1141  * Therefore, proactively update snd_congestion when wrap detected. */
1142  if (PREDICT_FALSE
1143  (seq_leq (tc->snd_congestion, tc->snd_una - tc->bytes_acked)
1144  && seq_gt (tc->snd_congestion, tc->snd_una)))
1145  tc->snd_congestion = tc->snd_una - 1;
1146 }
1147 
1148 static u8
1150 {
1151  return (TCP_DUPACK_THRESHOLD - 1) * tc->snd_mss < tc->sack_sb.sacked_bytes;
1152 }
1153 
1154 static u8
1156 {
1157  return (tc->rcv_dupacks == TCP_DUPACK_THRESHOLD
1158  || tcp_should_fastrecover_sack (tc));
1159 }
1160 
1161 /**
1162  * One function to rule them all ... and in the darkness bind them
1163  */
1164 static void
1166 {
1167  u32 rxt_delivered;
1168 
1169  if (tcp_in_fastrecovery (tc) && tcp_opts_sack_permitted (&tc->rcv_opts))
1170  {
1171  if (tc->bytes_acked)
1172  goto partial_ack;
1173  tcp_fast_retransmit (tc);
1174  return;
1175  }
1176  /*
1177  * Duplicate ACK. Check if we should enter fast recovery, or if already in
1178  * it account for the bytes that left the network.
1179  */
1180  else if (is_dack && !tcp_in_recovery (tc))
1181  {
1182  TCP_EVT_DBG (TCP_EVT_DUPACK_RCVD, tc, 1);
1183  ASSERT (tc->snd_una != tc->snd_una_max
1184  || tc->sack_sb.last_sacked_bytes);
1185 
1186  tc->rcv_dupacks++;
1187 
1188  /* Pure duplicate ack. If some data got acked, it's handled lower */
1189  if (tc->rcv_dupacks > TCP_DUPACK_THRESHOLD && !tc->bytes_acked)
1190  {
1191  ASSERT (tcp_in_fastrecovery (tc));
1192  tc->cc_algo->rcv_cong_ack (tc, TCP_CC_DUPACK);
1193  return;
1194  }
1195  else if (tcp_should_fastrecover (tc))
1196  {
1197  ASSERT (!tcp_in_fastrecovery (tc));
1198 
1199  /* If of of the two conditions lower hold, reset dupacks because
1200  * we're probably after timeout (RFC6582 heuristics).
1201  * If Cumulative ack does not cover more than congestion threshold,
1202  * and:
1203  * 1) The following doesn't hold: The congestion window is greater
1204  * than SMSS bytes and the difference between highest_ack
1205  * and prev_highest_ack is at most 4*SMSS bytes
1206  * 2) Echoed timestamp in the last non-dup ack does not equal the
1207  * stored timestamp
1208  */
1209  if (seq_leq (tc->snd_una, tc->snd_congestion)
1210  && ((!(tc->cwnd > tc->snd_mss
1211  && tc->bytes_acked <= 4 * tc->snd_mss))
1212  || (tc->rcv_opts.tsecr != tc->tsecr_last_ack)))
1213  {
1214  tc->rcv_dupacks = 0;
1215  return;
1216  }
1217 
1219  tc->cc_algo->rcv_cong_ack (tc, TCP_CC_DUPACK);
1220 
1221  /* The first segment MUST be retransmitted */
1223 
1224  /* Post retransmit update cwnd to ssthresh and account for the
1225  * three segments that have left the network and should've been
1226  * buffered at the receiver XXX */
1227  tc->cwnd = tc->ssthresh + tc->rcv_dupacks * tc->snd_mss;
1228  ASSERT (tc->cwnd >= tc->snd_mss);
1229 
1230  /* If cwnd allows, send more data */
1231  if (tcp_opts_sack_permitted (&tc->rcv_opts))
1232  {
1233  scoreboard_init_high_rxt (&tc->sack_sb,
1234  tc->snd_una + tc->snd_mss);
1236  }
1237  else
1238  {
1240  }
1241  return;
1242  }
1243  else if (!tc->bytes_acked
1244  || (tc->bytes_acked && !tcp_in_cong_recovery (tc)))
1245  {
1246  tc->cc_algo->rcv_cong_ack (tc, TCP_CC_DUPACK);
1247  return;
1248  }
1249  else
1250  goto partial_ack;
1251  }
1252 
1253  if (!tc->bytes_acked)
1254  return;
1255 
1256 partial_ack:
1257  TCP_EVT_DBG (TCP_EVT_CC_PACK, tc);
1258 
1259  /*
1260  * Legitimate ACK. 1) See if we can exit recovery
1261  */
1262  /* XXX limit this only to first partial ack? */
1263  if (seq_lt (tc->snd_una, tc->snd_congestion))
1265  else
1267 
1268  if (seq_geq (tc->snd_una, tc->snd_congestion))
1269  {
1270  /* If spurious return, we've already updated everything */
1271  if (tcp_cc_recover (tc))
1272  {
1273  tc->tsecr_last_ack = tc->rcv_opts.tsecr;
1274  return;
1275  }
1276 
1277  tc->snd_nxt = tc->snd_una_max;
1278 
1279  /* Treat as congestion avoidance ack */
1280  tc->cc_algo->rcv_ack (tc);
1281  tc->tsecr_last_ack = tc->rcv_opts.tsecr;
1282  return;
1283  }
1284 
1285  /*
1286  * Legitimate ACK. 2) If PARTIAL ACK try to retransmit
1287  */
1288 
1289  /* RFC6675: If the incoming ACK is a cumulative acknowledgment,
1290  * reset dupacks to 0. Also needed if in congestion recovery */
1291  tc->rcv_dupacks = 0;
1292 
1293  /* Post RTO timeout don't try anything fancy */
1294  if (tcp_in_recovery (tc))
1295  {
1296  tc->cc_algo->rcv_ack (tc);
1297  tc->tsecr_last_ack = tc->rcv_opts.tsecr;
1298  transport_add_tx_event (&tc->connection);
1299  return;
1300  }
1301 
1302  /* Remove retransmitted bytes that have been delivered */
1303  ASSERT (tc->bytes_acked + tc->sack_sb.snd_una_adv
1304  >= tc->sack_sb.last_bytes_delivered
1305  || (tc->flags & TCP_CONN_FINSNT));
1306 
1307  if (seq_lt (tc->snd_una, tc->sack_sb.high_rxt))
1308  {
1309  /* If we have sacks and we haven't gotten an ack beyond high_rxt,
1310  * remove sacked bytes delivered */
1311  rxt_delivered = tc->bytes_acked + tc->sack_sb.snd_una_adv
1312  - tc->sack_sb.last_bytes_delivered;
1313  ASSERT (tc->snd_rxt_bytes >= rxt_delivered);
1314  tc->snd_rxt_bytes -= rxt_delivered;
1315  }
1316  else
1317  {
1318  /* Either all retransmitted holes have been acked, or we're
1319  * "in the blind" and retransmitting segment by segment */
1320  tc->snd_rxt_bytes = 0;
1321  }
1322 
1323  tc->cc_algo->rcv_cong_ack (tc, TCP_CC_PARTIALACK);
1324 
1325  /*
1326  * Since this was a partial ack, try to retransmit some more data
1327  */
1328  tcp_fast_retransmit (tc);
1329 }
1330 
1331 /**
1332  * Process incoming ACK
1333  */
1334 static int
1336  tcp_header_t * th, u32 * next, u32 * error)
1337 {
1338  u32 prev_snd_wnd, prev_snd_una;
1339  u8 is_dack;
1340 
1341  TCP_EVT_DBG (TCP_EVT_CC_STAT, tc);
1342 
1343  /* If the ACK acks something not yet sent (SEG.ACK > SND.NXT) */
1344  if (PREDICT_FALSE (seq_gt (vnet_buffer (b)->tcp.ack_number, tc->snd_nxt)))
1345  {
1346  /* When we entered recovery, we reset snd_nxt to snd_una. Seems peer
1347  * still has the data so accept the ack */
1348  if (tcp_in_recovery (tc)
1349  && seq_leq (vnet_buffer (b)->tcp.ack_number, tc->snd_congestion))
1350  {
1351  tc->snd_nxt = vnet_buffer (b)->tcp.ack_number;
1352  if (seq_gt (tc->snd_nxt, tc->snd_una_max))
1353  tc->snd_una_max = tc->snd_nxt;
1354  goto process_ack;
1355  }
1356 
1357  /* If we have outstanding data and this is within the window, accept it,
1358  * probably retransmit has timed out. Otherwise ACK segment and then
1359  * drop it */
1360  if (seq_gt (vnet_buffer (b)->tcp.ack_number, tc->snd_una_max))
1361  {
1362  tcp_make_ack (tc, b);
1363  *next = tcp_next_output (tc->c_is_ip4);
1364  *error = TCP_ERROR_ACK_INVALID;
1365  TCP_EVT_DBG (TCP_EVT_ACK_RCV_ERR, tc, 0,
1366  vnet_buffer (b)->tcp.ack_number);
1367  return -1;
1368  }
1369 
1370  TCP_EVT_DBG (TCP_EVT_ACK_RCV_ERR, tc, 2,
1371  vnet_buffer (b)->tcp.ack_number);
1372 
1373  tc->snd_nxt = vnet_buffer (b)->tcp.ack_number;
1374  *error = TCP_ERROR_ACK_FUTURE;
1375  }
1376 
1377  /* If old ACK, probably it's an old dupack */
1378  if (PREDICT_FALSE (seq_lt (vnet_buffer (b)->tcp.ack_number, tc->snd_una)))
1379  {
1380  *error = TCP_ERROR_ACK_OLD;
1381  TCP_EVT_DBG (TCP_EVT_ACK_RCV_ERR, tc, 1,
1382  vnet_buffer (b)->tcp.ack_number);
1383  if (tcp_in_fastrecovery (tc) && tc->rcv_dupacks == TCP_DUPACK_THRESHOLD)
1384  tcp_cc_handle_event (tc, 1);
1385  /* Don't drop yet */
1386  return 0;
1387  }
1388 
1389  /*
1390  * Looks okay, process feedback
1391  */
1392 process_ack:
1393  if (tcp_opts_sack_permitted (&tc->rcv_opts))
1394  tcp_rcv_sacks (tc, vnet_buffer (b)->tcp.ack_number);
1395 
1396  prev_snd_wnd = tc->snd_wnd;
1397  prev_snd_una = tc->snd_una;
1398  tcp_update_snd_wnd (tc, vnet_buffer (b)->tcp.seq_number,
1399  vnet_buffer (b)->tcp.ack_number,
1400  clib_net_to_host_u16 (th->window) << tc->snd_wscale);
1401  tc->bytes_acked = vnet_buffer (b)->tcp.ack_number - tc->snd_una;
1402  tc->snd_una = vnet_buffer (b)->tcp.ack_number + tc->sack_sb.snd_una_adv;
1403  tcp_validate_txf_size (tc, tc->bytes_acked);
1404 
1405  if (tc->bytes_acked)
1406  tcp_dequeue_acked (tc, vnet_buffer (b)->tcp.ack_number);
1407 
1408  TCP_EVT_DBG (TCP_EVT_ACK_RCVD, tc);
1409 
1410  /*
1411  * Check if we have congestion event
1412  */
1413 
1414  if (tcp_ack_is_cc_event (tc, b, prev_snd_wnd, prev_snd_una, &is_dack))
1415  {
1416  tcp_cc_handle_event (tc, is_dack);
1417  if (!tcp_in_cong_recovery (tc))
1418  return 0;
1419  *error = TCP_ERROR_ACK_DUP;
1420  if (vnet_buffer (b)->tcp.data_len || tcp_is_fin (th))
1421  return 0;
1422  return -1;
1423  }
1424 
1425  /*
1426  * Update congestion control (slow start/congestion avoidance)
1427  */
1428  tcp_cc_update (tc, b);
1429  *error = TCP_ERROR_ACK_OK;
1430  return 0;
1431 }
1432 
1433 static u8
1435 {
1436  int i;
1437  for (i = 1; i < vec_len (sacks); i++)
1438  {
1439  if (sacks[i - 1].end == sacks[i].start)
1440  return 0;
1441  }
1442  return 1;
1443 }
1444 
1445 /**
1446  * Build SACK list as per RFC2018.
1447  *
1448  * Makes sure the first block contains the segment that generated the current
1449  * ACK and the following ones are the ones most recently reported in SACK
1450  * blocks.
1451  *
1452  * @param tc TCP connection for which the SACK list is updated
1453  * @param start Start sequence number of the newest SACK block
1454  * @param end End sequence of the newest SACK block
1455  */
1456 void
1458 {
1459  sack_block_t *new_list = 0, *block = 0;
1460  int i;
1461 
1462  /* If the first segment is ooo add it to the list. Last write might've moved
1463  * rcv_nxt over the first segment. */
1464  if (seq_lt (tc->rcv_nxt, start))
1465  {
1466  vec_add2 (new_list, block, 1);
1467  block->start = start;
1468  block->end = end;
1469  }
1470 
1471  /* Find the blocks still worth keeping. */
1472  for (i = 0; i < vec_len (tc->snd_sacks); i++)
1473  {
1474  /* Discard if rcv_nxt advanced beyond current block */
1475  if (seq_leq (tc->snd_sacks[i].start, tc->rcv_nxt))
1476  continue;
1477 
1478  /* Merge or drop if segment overlapped by the new segment */
1479  if (block && (seq_geq (tc->snd_sacks[i].end, new_list[0].start)
1480  && seq_leq (tc->snd_sacks[i].start, new_list[0].end)))
1481  {
1482  if (seq_lt (tc->snd_sacks[i].start, new_list[0].start))
1483  new_list[0].start = tc->snd_sacks[i].start;
1484  if (seq_lt (new_list[0].end, tc->snd_sacks[i].end))
1485  new_list[0].end = tc->snd_sacks[i].end;
1486  continue;
1487  }
1488 
1489  /* Save to new SACK list if we have space. */
1490  if (vec_len (new_list) < TCP_MAX_SACK_BLOCKS)
1491  {
1492  vec_add1 (new_list, tc->snd_sacks[i]);
1493  }
1494  else
1495  {
1496  clib_warning ("sack discarded");
1497  }
1498  }
1499 
1500  ASSERT (vec_len (new_list) <= TCP_MAX_SACK_BLOCKS);
1501 
1502  /* Replace old vector with new one */
1503  vec_free (tc->snd_sacks);
1504  tc->snd_sacks = new_list;
1505 
1506  /* Segments should not 'touch' */
1507  ASSERT (tcp_sack_vector_is_sane (tc->snd_sacks));
1508 }
1509 
1510 u32
1512 {
1513  u32 bytes = 0, i;
1514  for (i = 0; i < vec_len (tc->snd_sacks); i++)
1515  bytes += tc->snd_sacks[i].end - tc->snd_sacks[i].start;
1516  return bytes;
1517 }
1518 
1519 /** Enqueue data for delivery to application */
1520 static int
1522  u16 data_len)
1523 {
1524  int written, error = TCP_ERROR_ENQUEUED;
1525 
1526  ASSERT (seq_geq (vnet_buffer (b)->tcp.seq_number, tc->rcv_nxt));
1527  ASSERT (data_len);
1528  written = session_enqueue_stream_connection (&tc->connection, b, 0,
1529  1 /* queue event */ , 1);
1530 
1531  TCP_EVT_DBG (TCP_EVT_INPUT, tc, 0, data_len, written);
1532 
1533  /* Update rcv_nxt */
1534  if (PREDICT_TRUE (written == data_len))
1535  {
1536  tc->rcv_nxt += written;
1537  }
1538  /* If more data written than expected, account for out-of-order bytes. */
1539  else if (written > data_len)
1540  {
1541  tc->rcv_nxt += written;
1542 
1543  /* Send ACK confirming the update */
1544  tc->flags |= TCP_CONN_SNDACK;
1545  TCP_EVT_DBG (TCP_EVT_CC_INPUT, tc, data_len, written);
1546  }
1547  else if (written > 0)
1548  {
1549  /* We've written something but FIFO is probably full now */
1550  tc->rcv_nxt += written;
1551 
1552  /* Depending on how fast the app is, all remaining buffers in burst will
1553  * not be enqueued. Inform peer */
1554  tc->flags |= TCP_CONN_SNDACK;
1555 
1556  error = TCP_ERROR_PARTIALLY_ENQUEUED;
1557  }
1558  else
1559  {
1560  tc->flags |= TCP_CONN_SNDACK;
1561  return TCP_ERROR_FIFO_FULL;
1562  }
1563 
1564  /* Update SACK list if need be */
1565  if (tcp_opts_sack_permitted (&tc->rcv_opts))
1566  {
1567  /* Remove SACK blocks that have been delivered */
1568  tcp_update_sack_list (tc, tc->rcv_nxt, tc->rcv_nxt);
1569  }
1570 
1571  return error;
1572 }
1573 
1574 /** Enqueue out-of-order data */
1575 static int
1577  u16 data_len)
1578 {
1579  stream_session_t *s0;
1580  int rv, offset;
1581 
1582  ASSERT (seq_gt (vnet_buffer (b)->tcp.seq_number, tc->rcv_nxt));
1583  ASSERT (data_len);
1584 
1585  /* Enqueue out-of-order data with relative offset */
1586  rv = session_enqueue_stream_connection (&tc->connection, b,
1587  vnet_buffer (b)->tcp.seq_number -
1588  tc->rcv_nxt, 0 /* queue event */ ,
1589  0);
1590 
1591  /* Nothing written */
1592  if (rv)
1593  {
1594  TCP_EVT_DBG (TCP_EVT_INPUT, tc, 1, data_len, 0);
1595  return TCP_ERROR_FIFO_FULL;
1596  }
1597 
1598  TCP_EVT_DBG (TCP_EVT_INPUT, tc, 1, data_len, data_len);
1599 
1600  /* Update SACK list if in use */
1601  if (tcp_opts_sack_permitted (&tc->rcv_opts))
1602  {
1603  ooo_segment_t *newest;
1604  u32 start, end;
1605 
1606  s0 = session_get (tc->c_s_index, tc->c_thread_index);
1607 
1608  /* Get the newest segment from the fifo */
1609  newest = svm_fifo_newest_ooo_segment (s0->server_rx_fifo);
1610  if (newest)
1611  {
1612  offset = ooo_segment_offset (s0->server_rx_fifo, newest);
1613  ASSERT (offset <= vnet_buffer (b)->tcp.seq_number - tc->rcv_nxt);
1614  start = tc->rcv_nxt + offset;
1615  end = start + ooo_segment_length (s0->server_rx_fifo, newest);
1616  tcp_update_sack_list (tc, start, end);
1617  svm_fifo_newest_ooo_segment_reset (s0->server_rx_fifo);
1618  TCP_EVT_DBG (TCP_EVT_CC_SACKS, tc);
1619  }
1620  }
1621 
1622  return TCP_ERROR_ENQUEUED_OOO;
1623 }
1624 
1625 /**
1626  * Check if ACK could be delayed. If ack can be delayed, it should return
1627  * true for a full frame. If we're always acking return 0.
1628  */
1629 always_inline int
1631 {
1632  /* Send ack if ... */
1633  if (TCP_ALWAYS_ACK
1634  /* just sent a rcv wnd 0 */
1635  || (tc->flags & TCP_CONN_SENT_RCV_WND0) != 0
1636  /* constrained to send ack */
1637  || (tc->flags & TCP_CONN_SNDACK) != 0
1638  /* we're almost out of tx wnd */
1639  || tcp_available_cc_snd_space (tc) < 4 * tc->snd_mss)
1640  return 0;
1641 
1642  return 1;
1643 }
1644 
1645 static int
1647 {
1648  u32 discard, first = b->current_length;
1650 
1651  /* Handle multi-buffer segments */
1652  if (n_bytes_to_drop > b->current_length)
1653  {
1654  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
1655  return -1;
1656  do
1657  {
1658  discard = clib_min (n_bytes_to_drop, b->current_length);
1659  vlib_buffer_advance (b, discard);
1660  b = vlib_get_buffer (vm, b->next_buffer);
1661  n_bytes_to_drop -= discard;
1662  }
1663  while (n_bytes_to_drop);
1664  if (n_bytes_to_drop > first)
1665  b->total_length_not_including_first_buffer -= n_bytes_to_drop - first;
1666  }
1667  else
1668  vlib_buffer_advance (b, n_bytes_to_drop);
1669  vnet_buffer (b)->tcp.data_len -= n_bytes_to_drop;
1670  return 0;
1671 }
1672 
1673 /**
1674  * Receive buffer for connection and handle acks
1675  *
1676  * It handles both in order or out-of-order data.
1677  */
1678 static int
1680 {
1681  u32 error, n_bytes_to_drop, n_data_bytes;
1682 
1683  vlib_buffer_advance (b, vnet_buffer (b)->tcp.data_offset);
1684  n_data_bytes = vnet_buffer (b)->tcp.data_len;
1685  ASSERT (n_data_bytes);
1686 
1687  /* Handle out-of-order data */
1688  if (PREDICT_FALSE (vnet_buffer (b)->tcp.seq_number != tc->rcv_nxt))
1689  {
1690  /* Old sequence numbers allowed through because they overlapped
1691  * the rx window */
1692  if (seq_lt (vnet_buffer (b)->tcp.seq_number, tc->rcv_nxt))
1693  {
1694  /* Completely in the past (possible retransmit). Ack
1695  * retransmissions since we may not have any data to send */
1696  if (seq_leq (vnet_buffer (b)->tcp.seq_end, tc->rcv_nxt))
1697  {
1698  tcp_make_ack (tc, b);
1699  error = TCP_ERROR_SEGMENT_OLD;
1700  *next0 = tcp_next_output (tc->c_is_ip4);
1701  goto done;
1702  }
1703 
1704  /* Chop off the bytes in the past and see if what is left
1705  * can be enqueued in order */
1706  n_bytes_to_drop = tc->rcv_nxt - vnet_buffer (b)->tcp.seq_number;
1707  n_data_bytes -= n_bytes_to_drop;
1708  vnet_buffer (b)->tcp.seq_number = tc->rcv_nxt;
1709  if (tcp_buffer_discard_bytes (b, n_bytes_to_drop))
1710  {
1711  error = TCP_ERROR_SEGMENT_OLD;
1712  *next0 = tcp_next_drop (tc->c_is_ip4);
1713  goto done;
1714  }
1715  goto in_order;
1716  }
1717 
1718  /* RFC2581: Enqueue and send DUPACK for fast retransmit */
1719  error = tcp_session_enqueue_ooo (tc, b, n_data_bytes);
1720  *next0 = tcp_next_output (tc->c_is_ip4);
1721  tcp_make_ack (tc, b);
1722  vnet_buffer (b)->tcp.flags = TCP_BUF_FLAG_DUPACK;
1723  TCP_EVT_DBG (TCP_EVT_DUPACK_SENT, tc, vnet_buffer (b)->tcp);
1724  goto done;
1725  }
1726 
1727 in_order:
1728 
1729  /* In order data, enqueue. Fifo figures out by itself if any out-of-order
1730  * segments can be enqueued after fifo tail offset changes. */
1731  error = tcp_session_enqueue_data (tc, b, n_data_bytes);
1732  if (tcp_can_delack (tc))
1733  {
1734  *next0 = tcp_next_drop (tc->c_is_ip4);
1735  if (!tcp_timer_is_active (tc, TCP_TIMER_DELACK))
1736  tcp_timer_set (tc, TCP_TIMER_DELACK, TCP_DELACK_TIME);
1737  goto done;
1738  }
1739 
1740  *next0 = tcp_next_output (tc->c_is_ip4);
1741  tcp_make_ack (tc, b);
1742 
1743 done:
1744  return error;
1745 }
1746 
1747 typedef struct
1748 {
1751 } tcp_rx_trace_t;
1752 
1753 static u8 *
1754 format_tcp_rx_trace (u8 * s, va_list * args)
1755 {
1756  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1757  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1758  tcp_rx_trace_t *t = va_arg (*args, tcp_rx_trace_t *);
1759  u32 indent = format_get_indent (s);
1760 
1761  s = format (s, "%U\n%U%U",
1762  format_tcp_header, &t->tcp_header, 128,
1763  format_white_space, indent,
1765 
1766  return s;
1767 }
1768 
1769 static u8 *
1770 format_tcp_rx_trace_short (u8 * s, va_list * args)
1771 {
1772  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1773  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1774  tcp_rx_trace_t *t = va_arg (*args, tcp_rx_trace_t *);
1775 
1776  s = format (s, "%d -> %d (%U)",
1777  clib_net_to_host_u16 (t->tcp_header.dst_port),
1778  clib_net_to_host_u16 (t->tcp_header.src_port), format_tcp_state,
1779  t->tcp_connection.state);
1780 
1781  return s;
1782 }
1783 
1784 static void
1786  tcp_header_t * th0, vlib_buffer_t * b0, u8 is_ip4)
1787 {
1788  if (tc0)
1789  {
1790  clib_memcpy (&t0->tcp_connection, tc0, sizeof (t0->tcp_connection));
1791  }
1792  else
1793  {
1794  th0 = tcp_buffer_hdr (b0);
1795  }
1796  clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header));
1797 }
1798 
1799 static void
1801  vlib_frame_t * frame, u8 is_ip4)
1802 {
1803  u32 *from, n_left;
1804 
1805  n_left = frame->n_vectors;
1806  from = vlib_frame_vector_args (frame);
1807 
1808  while (n_left >= 1)
1809  {
1810  tcp_connection_t *tc0;
1811  tcp_rx_trace_t *t0;
1812  tcp_header_t *th0;
1813  vlib_buffer_t *b0;
1814  u32 bi0;
1815 
1816  bi0 = from[0];
1817  b0 = vlib_get_buffer (vm, bi0);
1818 
1819  if (b0->flags & VLIB_BUFFER_IS_TRACED)
1820  {
1821  t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
1822  tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
1823  vm->thread_index);
1824  th0 = tcp_buffer_hdr (b0);
1825  tcp_set_rx_trace_data (t0, tc0, th0, b0, is_ip4);
1826  }
1827 
1828  from += 1;
1829  n_left -= 1;
1830  }
1831 }
1832 
1833 always_inline void
1834 tcp_node_inc_counter_i (vlib_main_t * vm, u32 tcp4_node, u32 tcp6_node,
1835  u8 is_ip4, u32 evt, u32 val)
1836 {
1837  if (is_ip4)
1838  vlib_node_increment_counter (vm, tcp4_node, evt, val);
1839  else
1840  vlib_node_increment_counter (vm, tcp6_node, evt, val);
1841 }
1842 
1843 #define tcp_maybe_inc_counter(node_id, err, count) \
1844 { \
1845  if (next0 != tcp_next_drop (is_ip4)) \
1846  tcp_node_inc_counter_i (vm, tcp4_##node_id##_node.index, \
1847  tcp6_##node_id##_node.index, is_ip4, err, \
1848  1); \
1849 }
1850 #define tcp_inc_counter(node_id, err, count) \
1851  tcp_node_inc_counter_i (vm, tcp4_##node_id##_node.index, \
1852  tcp6_##node_id##_node.index, is_ip4, \
1853  err, count)
1854 #define tcp_maybe_inc_err_counter(cnts, err) \
1855 { \
1856  cnts[err] += (next0 != tcp_next_drop (is_ip4)); \
1857 }
1858 #define tcp_inc_err_counter(cnts, err, val) \
1859 { \
1860  cnts[err] += val; \
1861 }
1862 #define tcp_store_err_counters(node_id, cnts) \
1863 { \
1864  int i; \
1865  for (i = 0; i < TCP_N_ERROR; i++) \
1866  if (cnts[i]) \
1867  tcp_inc_counter(node_id, i, cnts[i]); \
1868 }
1869 
1870 
1873  vlib_frame_t * frame, int is_ip4)
1874 {
1875  u32 thread_index = vm->thread_index, errors = 0;
1876  u32 n_left_from, next_index, *from, *to_next;
1877  u16 err_counters[TCP_N_ERROR] = { 0 };
1878  u8 is_fin = 0;
1879 
1880  if (node->flags & VLIB_NODE_FLAG_TRACE)
1881  tcp_established_trace_frame (vm, node, frame, is_ip4);
1882 
1883  from = vlib_frame_vector_args (frame);
1884  n_left_from = frame->n_vectors;
1885  next_index = node->cached_next_index;
1886 
1887  while (n_left_from > 0)
1888  {
1889  u32 n_left_to_next;
1890 
1891  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1892  while (n_left_from > 0 && n_left_to_next > 0)
1893  {
1894  u32 bi0;
1895  vlib_buffer_t *b0;
1896  tcp_header_t *th0 = 0;
1897  tcp_connection_t *tc0;
1898  u32 next0 = tcp_next_drop (is_ip4), error0 = TCP_ERROR_ACK_OK;
1899 
1900  if (n_left_from > 1)
1901  {
1902  vlib_buffer_t *pb;
1903  pb = vlib_get_buffer (vm, from[1]);
1904  vlib_prefetch_buffer_header (pb, LOAD);
1905  CLIB_PREFETCH (pb->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
1906  }
1907 
1908  bi0 = from[0];
1909  to_next[0] = bi0;
1910  from += 1;
1911  to_next += 1;
1912  n_left_from -= 1;
1913  n_left_to_next -= 1;
1914 
1915  b0 = vlib_get_buffer (vm, bi0);
1916  tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
1917  thread_index);
1918 
1919  if (PREDICT_FALSE (tc0 == 0))
1920  {
1921  error0 = TCP_ERROR_INVALID_CONNECTION;
1922  goto done;
1923  }
1924 
1925  th0 = tcp_buffer_hdr (b0);
1926  /* N.B. buffer is rewritten if segment is ooo. Thus, th0 becomes a
1927  * dangling reference. */
1928  is_fin = tcp_is_fin (th0);
1929 
1930  /* SYNs, FINs and data consume sequence numbers */
1931  vnet_buffer (b0)->tcp.seq_end = vnet_buffer (b0)->tcp.seq_number
1932  + tcp_is_syn (th0) + is_fin + vnet_buffer (b0)->tcp.data_len;
1933 
1934  /* TODO header prediction fast path */
1935 
1936  /* 1-4: check SEQ, RST, SYN */
1937  if (PREDICT_FALSE (tcp_segment_validate (vm, tc0, b0, th0, &next0,
1938  &error0)))
1939  {
1940  tcp_maybe_inc_err_counter (err_counters, error0);
1941  TCP_EVT_DBG (TCP_EVT_SEG_INVALID, tc0, vnet_buffer (b0)->tcp);
1942  goto done;
1943  }
1944 
1945  /* 5: check the ACK field */
1946  if (PREDICT_FALSE (tcp_rcv_ack (tc0, b0, th0, &next0, &error0)))
1947  {
1948  tcp_maybe_inc_err_counter (err_counters, error0);
1949  goto done;
1950  }
1951 
1952  /* 6: check the URG bit TODO */
1953 
1954  /* 7: process the segment text */
1955  if (vnet_buffer (b0)->tcp.data_len)
1956  {
1957  error0 = tcp_segment_rcv (tc0, b0, &next0);
1958  tcp_maybe_inc_err_counter (err_counters, error0);
1959  }
1960 
1961  /* 8: check the FIN bit */
1962  if (PREDICT_FALSE (is_fin))
1963  {
1964  /* Enter CLOSE-WAIT and notify session. To avoid lingering
1965  * in CLOSE-WAIT, set timer (reuse WAITCLOSE). */
1966  /* Account for the FIN if nothing else was received */
1967  if (vnet_buffer (b0)->tcp.data_len == 0)
1968  tc0->rcv_nxt += 1;
1969  tcp_make_ack (tc0, b0);
1970  next0 = tcp_next_output (tc0->c_is_ip4);
1971  tc0->state = TCP_STATE_CLOSE_WAIT;
1972  stream_session_disconnect_notify (&tc0->connection);
1973  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_CLOSEWAIT_TIME);
1974  TCP_EVT_DBG (TCP_EVT_FIN_RCVD, tc0);
1975  tcp_inc_err_counter (err_counters, TCP_ERROR_FIN_RCVD, 1);
1976  }
1977 
1978  done:
1979  b0->error = node->errors[error0];
1980  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1981  n_left_to_next, bi0, next0);
1982  }
1983 
1984  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1985  }
1986 
1988  thread_index);
1989  err_counters[TCP_ERROR_EVENT_FIFO_FULL] = errors;
1990  tcp_store_err_counters (established, err_counters);
1991  tcp_flush_frame_to_output (vm, thread_index, is_ip4);
1992 
1993  return frame->n_vectors;
1994 }
1995 
1996 static uword
1998  vlib_frame_t * from_frame)
1999 {
2000  return tcp46_established_inline (vm, node, from_frame, 1 /* is_ip4 */ );
2001 }
2002 
2003 static uword
2005  vlib_frame_t * from_frame)
2006 {
2007  return tcp46_established_inline (vm, node, from_frame, 0 /* is_ip4 */ );
2008 }
2009 
2010 /* *INDENT-OFF* */
2012 {
2013  .function = tcp4_established,
2014  .name = "tcp4-established",
2015  /* Takes a vector of packets. */
2016  .vector_size = sizeof (u32),
2017  .n_errors = TCP_N_ERROR,
2018  .error_strings = tcp_error_strings,
2019  .n_next_nodes = TCP_ESTABLISHED_N_NEXT,
2020  .next_nodes =
2021  {
2022 #define _(s,n) [TCP_ESTABLISHED_NEXT_##s] = n,
2024 #undef _
2025  },
2026  .format_trace = format_tcp_rx_trace_short,
2027 };
2028 /* *INDENT-ON* */
2029 
2031 
2032 /* *INDENT-OFF* */
2034 {
2035  .function = tcp6_established,
2036  .name = "tcp6-established",
2037  /* Takes a vector of packets. */
2038  .vector_size = sizeof (u32),
2039  .n_errors = TCP_N_ERROR,
2040  .error_strings = tcp_error_strings,
2041  .n_next_nodes = TCP_ESTABLISHED_N_NEXT,
2042  .next_nodes =
2043  {
2044 #define _(s,n) [TCP_ESTABLISHED_NEXT_##s] = n,
2046 #undef _
2047  },
2048  .format_trace = format_tcp_rx_trace_short,
2049 };
2050 /* *INDENT-ON* */
2051 
2052 
2054 
2057 
2058 static u8
2060 {
2061  transport_connection_t *tmp = 0;
2062  u64 handle;
2063 
2064  if (!tc)
2065  return 1;
2066 
2067  /* Proxy case */
2068  if (tc->c_lcl_port == 0 && tc->state == TCP_STATE_LISTEN)
2069  return 1;
2070 
2071  u8 is_valid = (tc->c_lcl_port == hdr->dst_port
2072  && (tc->state == TCP_STATE_LISTEN
2073  || tc->c_rmt_port == hdr->src_port));
2074 
2075  if (!is_valid)
2076  {
2077  handle = session_lookup_half_open_handle (&tc->connection);
2078  tmp = session_lookup_half_open_connection (handle & 0xFFFFFFFF,
2079  tc->c_proto, tc->c_is_ip4);
2080 
2081  if (tmp)
2082  {
2083  if (tmp->lcl_port == hdr->dst_port
2084  && tmp->rmt_port == hdr->src_port)
2085  {
2086  TCP_DBG ("half-open is valid!");
2087  }
2088  }
2089  }
2090  return is_valid;
2091 }
2092 
2093 /**
2094  * Lookup transport connection
2095  */
2096 static tcp_connection_t *
2097 tcp_lookup_connection (u32 fib_index, vlib_buffer_t * b, u8 thread_index,
2098  u8 is_ip4)
2099 {
2100  tcp_header_t *tcp;
2101  transport_connection_t *tconn;
2102  tcp_connection_t *tc;
2103  u8 is_filtered = 0;
2104  if (is_ip4)
2105  {
2106  ip4_header_t *ip4;
2107  ip4 = vlib_buffer_get_current (b);
2108  tcp = ip4_next_header (ip4);
2109  tconn = session_lookup_connection_wt4 (fib_index,
2110  &ip4->dst_address,
2111  &ip4->src_address,
2112  tcp->dst_port,
2113  tcp->src_port,
2115  thread_index, &is_filtered);
2116  tc = tcp_get_connection_from_transport (tconn);
2117  ASSERT (tcp_lookup_is_valid (tc, tcp));
2118  }
2119  else
2120  {
2121  ip6_header_t *ip6;
2122  ip6 = vlib_buffer_get_current (b);
2123  tcp = ip6_next_header (ip6);
2124  tconn = session_lookup_connection_wt6 (fib_index,
2125  &ip6->dst_address,
2126  &ip6->src_address,
2127  tcp->dst_port,
2128  tcp->src_port,
2130  thread_index, &is_filtered);
2131  tc = tcp_get_connection_from_transport (tconn);
2132  ASSERT (tcp_lookup_is_valid (tc, tcp));
2133  }
2134  return tc;
2135 }
2136 
2139  vlib_frame_t * from_frame, int is_ip4)
2140 {
2141  tcp_main_t *tm = vnet_get_tcp_main ();
2142  u32 n_left_from, next_index, *from, *to_next;
2143  u32 my_thread_index = vm->thread_index, errors = 0;
2144 
2145  from = vlib_frame_vector_args (from_frame);
2146  n_left_from = from_frame->n_vectors;
2147 
2148  next_index = node->cached_next_index;
2149 
2150  while (n_left_from > 0)
2151  {
2152  u32 n_left_to_next;
2153 
2154  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2155 
2156  while (n_left_from > 0 && n_left_to_next > 0)
2157  {
2158  u32 bi0, ack0, seq0;
2159  vlib_buffer_t *b0;
2160  tcp_rx_trace_t *t0;
2161  tcp_header_t *tcp0 = 0;
2162  tcp_connection_t *tc0;
2163  tcp_connection_t *new_tc0;
2164  u32 next0 = tcp_next_drop (is_ip4), error0 = TCP_ERROR_ENQUEUED;
2165 
2166  bi0 = from[0];
2167  to_next[0] = bi0;
2168  from += 1;
2169  to_next += 1;
2170  n_left_from -= 1;
2171  n_left_to_next -= 1;
2172 
2173  b0 = vlib_get_buffer (vm, bi0);
2174  tc0 =
2176  tcp.connection_index);
2177  if (PREDICT_FALSE (tc0 == 0))
2178  {
2179  error0 = TCP_ERROR_INVALID_CONNECTION;
2180  goto drop;
2181  }
2182 
2183  /* Half-open completed recently but the connection was't removed
2184  * yet by the owning thread */
2185  if (PREDICT_FALSE (tc0->flags & TCP_CONN_HALF_OPEN_DONE))
2186  {
2187  /* Make sure the connection actually exists */
2188  ASSERT (tcp_lookup_connection (tc0->c_fib_index, b0,
2189  my_thread_index, is_ip4));
2190  goto drop;
2191  }
2192 
2193  ack0 = vnet_buffer (b0)->tcp.ack_number;
2194  seq0 = vnet_buffer (b0)->tcp.seq_number;
2195  tcp0 = tcp_buffer_hdr (b0);
2196 
2197  /* Crude check to see if the connection handle does not match
2198  * the packet. Probably connection just switched to established */
2199  if (PREDICT_FALSE (tcp0->dst_port != tc0->c_lcl_port
2200  || tcp0->src_port != tc0->c_rmt_port))
2201  goto drop;
2202 
2203  if (PREDICT_FALSE
2204  (!tcp_ack (tcp0) && !tcp_rst (tcp0) && !tcp_syn (tcp0)))
2205  goto drop;
2206 
2207  /* SYNs, FINs and data consume sequence numbers */
2208  vnet_buffer (b0)->tcp.seq_end = seq0 + tcp_is_syn (tcp0)
2209  + tcp_is_fin (tcp0) + vnet_buffer (b0)->tcp.data_len;
2210 
2211  /*
2212  * 1. check the ACK bit
2213  */
2214 
2215  /*
2216  * If the ACK bit is set
2217  * If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send a reset (unless
2218  * the RST bit is set, if so drop the segment and return)
2219  * <SEQ=SEG.ACK><CTL=RST>
2220  * and discard the segment. Return.
2221  * If SND.UNA =< SEG.ACK =< SND.NXT then the ACK is acceptable.
2222  */
2223  if (tcp_ack (tcp0))
2224  {
2225  if (seq_leq (ack0, tc0->iss) || seq_gt (ack0, tc0->snd_nxt))
2226  {
2227  clib_warning ("ack not in rcv wnd");
2228  if (!tcp_rst (tcp0))
2229  tcp_send_reset_w_pkt (tc0, b0, is_ip4);
2230  goto drop;
2231  }
2232 
2233  /* Make sure ACK is valid */
2234  if (seq_gt (tc0->snd_una, ack0))
2235  {
2236  clib_warning ("ack invalid");
2237  goto drop;
2238  }
2239  }
2240 
2241  /*
2242  * 2. check the RST bit
2243  */
2244 
2245  if (tcp_rst (tcp0))
2246  {
2247  /* If ACK is acceptable, signal client that peer is not
2248  * willing to accept connection and drop connection*/
2249  if (tcp_ack (tcp0))
2250  tcp_connection_reset (tc0);
2251  goto drop;
2252  }
2253 
2254  /*
2255  * 3. check the security and precedence (skipped)
2256  */
2257 
2258  /*
2259  * 4. check the SYN bit
2260  */
2261 
2262  /* No SYN flag. Drop. */
2263  if (!tcp_syn (tcp0))
2264  {
2265  clib_warning ("not synack");
2266  goto drop;
2267  }
2268 
2269  /* Parse options */
2270  if (tcp_options_parse (tcp0, &tc0->rcv_opts))
2271  {
2272  clib_warning ("options parse fail");
2273  goto drop;
2274  }
2275 
2276  /* Valid SYN or SYN-ACK. Move connection from half-open pool to
2277  * current thread pool. */
2278  pool_get (tm->connections[my_thread_index], new_tc0);
2279  clib_memcpy (new_tc0, tc0, sizeof (*new_tc0));
2280  new_tc0->c_c_index = new_tc0 - tm->connections[my_thread_index];
2281  new_tc0->c_thread_index = my_thread_index;
2282  new_tc0->rcv_nxt = vnet_buffer (b0)->tcp.seq_end;
2283  new_tc0->irs = seq0;
2284  new_tc0->timers[TCP_TIMER_ESTABLISH] = TCP_TIMER_HANDLE_INVALID;
2285  new_tc0->timers[TCP_TIMER_RETRANSMIT_SYN] =
2287  new_tc0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2288 
2289  /* If this is not the owning thread, wait for syn retransmit to
2290  * expire and cleanup then */
2292  tc0->flags |= TCP_CONN_HALF_OPEN_DONE;
2293 
2294  if (tcp_opts_tstamp (&new_tc0->rcv_opts))
2295  {
2296  new_tc0->tsval_recent = new_tc0->rcv_opts.tsval;
2297  new_tc0->tsval_recent_age = tcp_time_now ();
2298  }
2299 
2300  if (tcp_opts_wscale (&new_tc0->rcv_opts))
2301  new_tc0->snd_wscale = new_tc0->rcv_opts.wscale;
2302 
2303  new_tc0->snd_wnd = clib_net_to_host_u16 (tcp0->window)
2304  << new_tc0->snd_wscale;
2305  new_tc0->snd_wl1 = seq0;
2306  new_tc0->snd_wl2 = ack0;
2307 
2308  tcp_connection_init_vars (new_tc0);
2309 
2310  /* SYN-ACK: See if we can switch to ESTABLISHED state */
2311  if (PREDICT_TRUE (tcp_ack (tcp0)))
2312  {
2313  /* Our SYN is ACKed: we have iss < ack = snd_una */
2314 
2315  /* TODO Dequeue acknowledged segments if we support Fast Open */
2316  new_tc0->snd_una = ack0;
2317  new_tc0->state = TCP_STATE_ESTABLISHED;
2318 
2319  /* Make sure las is initialized for the wnd computation */
2320  new_tc0->rcv_las = new_tc0->rcv_nxt;
2321 
2322  /* Notify app that we have connection. If session layer can't
2323  * allocate session send reset */
2324  if (session_stream_connect_notify (&new_tc0->connection, 0))
2325  {
2326  clib_warning ("connect notify fail");
2327  tcp_send_reset_w_pkt (new_tc0, b0, is_ip4);
2328  tcp_connection_cleanup (new_tc0);
2329  goto drop;
2330  }
2331 
2332  /* Make sure after data segment processing ACK is sent */
2333  new_tc0->flags |= TCP_CONN_SNDACK;
2334 
2335  /* Update rtt with the syn-ack sample */
2336  tcp_update_rtt (new_tc0, vnet_buffer (b0)->tcp.ack_number);
2337  TCP_EVT_DBG (TCP_EVT_SYNACK_RCVD, new_tc0);
2338  }
2339  /* SYN: Simultaneous open. Change state to SYN-RCVD and send SYN-ACK */
2340  else
2341  {
2342  new_tc0->state = TCP_STATE_SYN_RCVD;
2343 
2344  /* Notify app that we have connection */
2345  if (session_stream_connect_notify (&new_tc0->connection, 0))
2346  {
2347  tcp_connection_cleanup (new_tc0);
2348  tcp_send_reset_w_pkt (tc0, b0, is_ip4);
2349  TCP_EVT_DBG (TCP_EVT_RST_SENT, tc0);
2350  goto drop;
2351  }
2352 
2353  tc0->rtt_ts = 0;
2354  tcp_init_snd_vars (tc0);
2355  tcp_make_synack (new_tc0, b0);
2356  next0 = tcp_next_output (is_ip4);
2357 
2358  goto drop;
2359  }
2360 
2361  /* Read data, if any */
2362  if (PREDICT_FALSE (vnet_buffer (b0)->tcp.data_len))
2363  {
2364  clib_warning ("rcvd data in syn-sent");
2365  error0 = tcp_segment_rcv (new_tc0, b0, &next0);
2366  if (error0 == TCP_ERROR_ACK_OK)
2367  error0 = TCP_ERROR_SYN_ACKS_RCVD;
2368  tcp_maybe_inc_counter (syn_sent, error0, 1);
2369  }
2370  else
2371  {
2372  tcp_make_ack (new_tc0, b0);
2373  next0 = tcp_next_output (new_tc0->c_is_ip4);
2374  }
2375 
2376  drop:
2377 
2378  b0->error = error0 ? node->errors[error0] : 0;
2379  if (PREDICT_FALSE
2380  ((b0->flags & VLIB_BUFFER_IS_TRACED) && tcp0 != 0))
2381  {
2382  t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
2383  clib_memcpy (&t0->tcp_header, tcp0, sizeof (t0->tcp_header));
2384  clib_memcpy (&t0->tcp_connection, tc0,
2385  sizeof (t0->tcp_connection));
2386  }
2387 
2388  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2389  n_left_to_next, bi0, next0);
2390  }
2391 
2392  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2393  }
2394 
2396  my_thread_index);
2397  tcp_inc_counter (syn_sent, TCP_ERROR_EVENT_FIFO_FULL, errors);
2398  return from_frame->n_vectors;
2399 }
2400 
2401 static uword
2403  vlib_frame_t * from_frame)
2404 {
2405  return tcp46_syn_sent_inline (vm, node, from_frame, 1 /* is_ip4 */ );
2406 }
2407 
2408 static uword
2410  vlib_frame_t * from_frame)
2411 {
2412  return tcp46_syn_sent_inline (vm, node, from_frame, 0 /* is_ip4 */ );
2413 }
2414 
2415 /* *INDENT-OFF* */
2417 {
2418  .function = tcp4_syn_sent,
2419  .name = "tcp4-syn-sent",
2420  /* Takes a vector of packets. */
2421  .vector_size = sizeof (u32),
2422  .n_errors = TCP_N_ERROR,
2423  .error_strings = tcp_error_strings,
2424  .n_next_nodes = TCP_SYN_SENT_N_NEXT,
2425  .next_nodes =
2426  {
2427 #define _(s,n) [TCP_SYN_SENT_NEXT_##s] = n,
2429 #undef _
2430  },
2431  .format_trace = format_tcp_rx_trace_short,
2432 };
2433 /* *INDENT-ON* */
2434 
2436 
2437 /* *INDENT-OFF* */
2439 {
2440  .function = tcp6_syn_sent_rcv,
2441  .name = "tcp6-syn-sent",
2442  /* Takes a vector of packets. */
2443  .vector_size = sizeof (u32),
2444  .n_errors = TCP_N_ERROR,
2445  .error_strings = tcp_error_strings,
2446  .n_next_nodes = TCP_SYN_SENT_N_NEXT,
2447  .next_nodes =
2448  {
2449 #define _(s,n) [TCP_SYN_SENT_NEXT_##s] = n,
2451 #undef _
2452  },
2453  .format_trace = format_tcp_rx_trace_short,
2454 };
2455 /* *INDENT-ON* */
2456 
2458 
2461 
2462 /**
2463  * Handles reception for all states except LISTEN, SYN-SENT and ESTABLISHED
2464  * as per RFC793 p. 64
2465  */
2468  vlib_frame_t * from_frame, int is_ip4)
2469 {
2470  u32 n_left_from, next_index, *from, *to_next, n_fins = 0;
2471  u32 my_thread_index = vm->thread_index, errors = 0;
2472 
2473  from = vlib_frame_vector_args (from_frame);
2474  n_left_from = from_frame->n_vectors;
2475  next_index = node->cached_next_index;
2476 
2477  while (n_left_from > 0)
2478  {
2479  u32 n_left_to_next;
2480 
2481  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2482 
2483  while (n_left_from > 0 && n_left_to_next > 0)
2484  {
2485  u32 bi0;
2486  vlib_buffer_t *b0;
2487  tcp_header_t *tcp0 = 0;
2488  tcp_connection_t *tc0;
2489  u32 next0 = tcp_next_drop (is_ip4), error0 = TCP_ERROR_NONE;
2490  u8 is_fin0;
2491 
2492  bi0 = from[0];
2493  to_next[0] = bi0;
2494  from += 1;
2495  to_next += 1;
2496  n_left_from -= 1;
2497  n_left_to_next -= 1;
2498 
2499  b0 = vlib_get_buffer (vm, bi0);
2500  tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
2501  my_thread_index);
2502  if (PREDICT_FALSE (tc0 == 0))
2503  {
2504  error0 = TCP_ERROR_INVALID_CONNECTION;
2505  goto drop;
2506  }
2507 
2508  tcp0 = tcp_buffer_hdr (b0);
2509  is_fin0 = tcp_is_fin (tcp0);
2510 
2511  /* SYNs, FINs and data consume sequence numbers */
2512  vnet_buffer (b0)->tcp.seq_end = vnet_buffer (b0)->tcp.seq_number
2513  + tcp_is_syn (tcp0) + is_fin0 + vnet_buffer (b0)->tcp.data_len;
2514 
2515  if (CLIB_DEBUG)
2516  {
2517  tcp_connection_t *tmp;
2518  tmp = tcp_lookup_connection (tc0->c_fib_index, b0,
2519  my_thread_index, is_ip4);
2520  if (tmp->state != tc0->state)
2521  {
2522  clib_warning ("state changed");
2523  goto drop;
2524  }
2525  }
2526 
2527  /*
2528  * Special treatment for CLOSED
2529  */
2530  if (PREDICT_FALSE (tc0->state == TCP_STATE_CLOSED))
2531  {
2532  error0 = TCP_ERROR_CONNECTION_CLOSED;
2533  goto drop;
2534  }
2535 
2536  /*
2537  * For all other states (except LISTEN)
2538  */
2539 
2540  /* 1-4: check SEQ, RST, SYN */
2541  if (PREDICT_FALSE (tcp_segment_validate (vm, tc0, b0, tcp0,
2542  &next0, &error0)))
2543  {
2544  tcp_maybe_inc_counter (rcv_process, error0, 1);
2545  goto drop;
2546  }
2547 
2548  /* 5: check the ACK field */
2549  switch (tc0->state)
2550  {
2551  case TCP_STATE_SYN_RCVD:
2552  /*
2553  * If the segment acknowledgment is not acceptable, form a
2554  * reset segment,
2555  * <SEQ=SEG.ACK><CTL=RST>
2556  * and send it.
2557  */
2558  if (!tcp_rcv_ack_is_acceptable (tc0, b0))
2559  {
2560  TCP_DBG ("connection not accepted");
2561  tcp_send_reset_w_pkt (tc0, b0, is_ip4);
2562  error0 = TCP_ERROR_ACK_INVALID;
2563  goto drop;
2564  }
2565 
2566  /* Update rtt and rto */
2567  tcp_update_rtt (tc0, vnet_buffer (b0)->tcp.ack_number);
2568 
2569  /* Switch state to ESTABLISHED */
2570  tc0->state = TCP_STATE_ESTABLISHED;
2571  TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc0);
2572 
2573  /* Initialize session variables */
2574  tc0->snd_una = vnet_buffer (b0)->tcp.ack_number;
2575  tc0->snd_wnd = clib_net_to_host_u16 (tcp0->window)
2576  << tc0->rcv_opts.wscale;
2577  tc0->snd_wl1 = vnet_buffer (b0)->tcp.seq_number;
2578  tc0->snd_wl2 = vnet_buffer (b0)->tcp.ack_number;
2579 
2580  /* Reset SYN-ACK retransmit and SYN_RCV establish timers */
2582  tcp_timer_reset (tc0, TCP_TIMER_ESTABLISH);
2583  stream_session_accept_notify (&tc0->connection);
2584  error0 = TCP_ERROR_ACK_OK;
2585  break;
2586  case TCP_STATE_ESTABLISHED:
2587  /* We can get packets in established state here because they
2588  * were enqueued before state change */
2589  if (tcp_rcv_ack (tc0, b0, tcp0, &next0, &error0))
2590  {
2591  tcp_maybe_inc_counter (rcv_process, error0, 1);
2592  goto drop;
2593  }
2594 
2595  break;
2596  case TCP_STATE_FIN_WAIT_1:
2597  /* In addition to the processing for the ESTABLISHED state, if
2598  * our FIN is now acknowledged then enter FIN-WAIT-2 and
2599  * continue processing in that state. */
2600  if (tcp_rcv_ack (tc0, b0, tcp0, &next0, &error0))
2601  {
2602  tcp_maybe_inc_counter (rcv_process, error0, 1);
2603  goto drop;
2604  }
2605 
2606  /* Still have to send the FIN */
2607  if (tc0->flags & TCP_CONN_FINPNDG)
2608  {
2609  /* TX fifo finally drained */
2610  if (!session_tx_fifo_max_dequeue (&tc0->connection))
2611  tcp_send_fin (tc0);
2612  }
2613  /* If FIN is ACKed */
2614  else if (tc0->snd_una == tc0->snd_una_max)
2615  {
2616  tc0->state = TCP_STATE_FIN_WAIT_2;
2617  TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc0);
2618 
2619  /* Stop all retransmit timers because we have nothing more
2620  * to send. Enable waitclose though because we're willing to
2621  * wait for peer's FIN but not indefinitely. */
2623  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME);
2624  }
2625  break;
2626  case TCP_STATE_FIN_WAIT_2:
2627  /* In addition to the processing for the ESTABLISHED state, if
2628  * the retransmission queue is empty, the user's CLOSE can be
2629  * acknowledged ("ok") but do not delete the TCB. */
2630  if (tcp_rcv_ack (tc0, b0, tcp0, &next0, &error0))
2631  {
2632  tcp_maybe_inc_counter (rcv_process, error0, 1);
2633  goto drop;
2634  }
2635  break;
2636  case TCP_STATE_CLOSE_WAIT:
2637  /* Do the same processing as for the ESTABLISHED state. */
2638  if (tcp_rcv_ack (tc0, b0, tcp0, &next0, &error0))
2639  {
2640  tcp_maybe_inc_counter (rcv_process, error0, 1);
2641  goto drop;
2642  }
2643  if (tc0->flags & TCP_CONN_FINPNDG)
2644  {
2645  /* TX fifo finally drained */
2646  if (!session_tx_fifo_max_dequeue (&tc0->connection))
2647  {
2648  tcp_send_fin (tc0);
2650  tc0->state = TCP_STATE_LAST_ACK;
2651  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE,
2652  TCP_2MSL_TIME);
2653  }
2654  }
2655  break;
2656  case TCP_STATE_CLOSING:
2657  /* In addition to the processing for the ESTABLISHED state, if
2658  * the ACK acknowledges our FIN then enter the TIME-WAIT state,
2659  * otherwise ignore the segment. */
2660  if (tcp_rcv_ack (tc0, b0, tcp0, &next0, &error0))
2661  {
2662  tcp_maybe_inc_counter (rcv_process, error0, 1);
2663  goto drop;
2664  }
2665 
2666  tc0->state = TCP_STATE_TIME_WAIT;
2667  TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc0);
2668  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_TIMEWAIT_TIME);
2669  goto drop;
2670 
2671  break;
2672  case TCP_STATE_LAST_ACK:
2673  /* The only thing that [should] arrive in this state is an
2674  * acknowledgment of our FIN. If our FIN is now acknowledged,
2675  * delete the TCB, enter the CLOSED state, and return. */
2676 
2677  if (!tcp_rcv_ack_is_acceptable (tc0, b0))
2678  {
2679  error0 = TCP_ERROR_ACK_INVALID;
2680  goto drop;
2681  }
2682  error0 = TCP_ERROR_ACK_OK;
2683  tc0->snd_una = vnet_buffer (b0)->tcp.ack_number;
2684  /* Apparently our ACK for the peer's FIN was lost */
2685  if (is_fin0 && tc0->snd_una != tc0->snd_una_max)
2686  {
2687  tcp_send_fin (tc0);
2688  goto drop;
2689  }
2690 
2691  tc0->state = TCP_STATE_CLOSED;
2692  TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc0);
2693 
2694  /* Don't free the connection from the data path since
2695  * we can't ensure that we have no packets already enqueued
2696  * to output. Rely instead on the waitclose timer */
2698  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, 1);
2699 
2700  goto drop;
2701 
2702  break;
2703  case TCP_STATE_TIME_WAIT:
2704  /* The only thing that can arrive in this state is a
2705  * retransmission of the remote FIN. Acknowledge it, and restart
2706  * the 2 MSL timeout. */
2707 
2708  if (tcp_rcv_ack (tc0, b0, tcp0, &next0, &error0))
2709  {
2710  tcp_maybe_inc_counter (rcv_process, error0, 1);
2711  goto drop;
2712  }
2713 
2714  tcp_make_ack (tc0, b0);
2715  next0 = tcp_next_output (is_ip4);
2716  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_TIMEWAIT_TIME);
2717  goto drop;
2718 
2719  break;
2720  default:
2721  ASSERT (0);
2722  }
2723 
2724  /* 6: check the URG bit TODO */
2725 
2726  /* 7: process the segment text */
2727  switch (tc0->state)
2728  {
2729  case TCP_STATE_ESTABLISHED:
2730  case TCP_STATE_FIN_WAIT_1:
2731  case TCP_STATE_FIN_WAIT_2:
2732  if (vnet_buffer (b0)->tcp.data_len)
2733  {
2734  error0 = tcp_segment_rcv (tc0, b0, &next0);
2735  tcp_maybe_inc_counter (rcv_process, error0, 1);
2736  }
2737  else if (is_fin0)
2738  tc0->rcv_nxt += 1;
2739  break;
2740  case TCP_STATE_CLOSE_WAIT:
2741  case TCP_STATE_CLOSING:
2742  case TCP_STATE_LAST_ACK:
2743  case TCP_STATE_TIME_WAIT:
2744  /* This should not occur, since a FIN has been received from the
2745  * remote side. Ignore the segment text. */
2746  break;
2747  }
2748 
2749  /* 8: check the FIN bit */
2750  if (!is_fin0)
2751  goto drop;
2752 
2753  switch (tc0->state)
2754  {
2755  case TCP_STATE_ESTABLISHED:
2756  case TCP_STATE_SYN_RCVD:
2757  /* Send FIN-ACK notify app and enter CLOSE-WAIT */
2759  tcp_make_fin (tc0, b0);
2760  tc0->snd_nxt += 1;
2761  tc0->snd_una_max = tc0->snd_nxt;
2763  next0 = tcp_next_output (tc0->c_is_ip4);
2764  stream_session_disconnect_notify (&tc0->connection);
2765  tc0->state = TCP_STATE_CLOSE_WAIT;
2766  TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc0);
2767  break;
2768  case TCP_STATE_CLOSE_WAIT:
2769  case TCP_STATE_CLOSING:
2770  case TCP_STATE_LAST_ACK:
2771  /* move along .. */
2772  break;
2773  case TCP_STATE_FIN_WAIT_1:
2774  tc0->state = TCP_STATE_CLOSING;
2775  tcp_make_ack (tc0, b0);
2776  next0 = tcp_next_output (is_ip4);
2777  TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc0);
2778  /* Wait for ACK but not forever */
2779  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME);
2780  break;
2781  case TCP_STATE_FIN_WAIT_2:
2782  /* Got FIN, send ACK! Be more aggressive with resource cleanup */
2783  tc0->state = TCP_STATE_TIME_WAIT;
2785  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_TIMEWAIT_TIME);
2786  tcp_make_ack (tc0, b0);
2787  next0 = tcp_next_output (is_ip4);
2788  TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc0);
2789  break;
2790  case TCP_STATE_TIME_WAIT:
2791  /* Remain in the TIME-WAIT state. Restart the time-wait
2792  * timeout.
2793  */
2794  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_TIMEWAIT_TIME);
2795  break;
2796  }
2797  TCP_EVT_DBG (TCP_EVT_FIN_RCVD, tc0);
2798  n_fins += 1;
2799 
2800  drop:
2801  b0->error = error0 ? node->errors[error0] : 0;
2802 
2803  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2804  {
2805  tcp_rx_trace_t *t0 =
2806  vlib_add_trace (vm, node, b0, sizeof (*t0));
2807  tcp_set_rx_trace_data (t0, tc0, tcp0, b0, is_ip4);
2808  }
2809 
2810  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2811  n_left_to_next, bi0, next0);
2812  }
2813 
2814  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2815  }
2816 
2818  my_thread_index);
2819  tcp_inc_counter (rcv_process, TCP_ERROR_EVENT_FIFO_FULL, errors);
2820  tcp_inc_counter (rcv_process, TCP_ERROR_FIN_RCVD, n_fins);
2821  return from_frame->n_vectors;
2822 }
2823 
2824 static uword
2826  vlib_frame_t * from_frame)
2827 {
2828  return tcp46_rcv_process_inline (vm, node, from_frame, 1 /* is_ip4 */ );
2829 }
2830 
2831 static uword
2833  vlib_frame_t * from_frame)
2834 {
2835  return tcp46_rcv_process_inline (vm, node, from_frame, 0 /* is_ip4 */ );
2836 }
2837 
2838 /* *INDENT-OFF* */
2840 {
2841  .function = tcp4_rcv_process,
2842  .name = "tcp4-rcv-process",
2843  /* Takes a vector of packets. */
2844  .vector_size = sizeof (u32),
2845  .n_errors = TCP_N_ERROR,
2846  .error_strings = tcp_error_strings,
2847  .n_next_nodes = TCP_RCV_PROCESS_N_NEXT,
2848  .next_nodes =
2849  {
2850 #define _(s,n) [TCP_RCV_PROCESS_NEXT_##s] = n,
2852 #undef _
2853  },
2854  .format_trace = format_tcp_rx_trace_short,
2855 };
2856 /* *INDENT-ON* */
2857 
2859 
2860 /* *INDENT-OFF* */
2862 {
2863  .function = tcp6_rcv_process,
2864  .name = "tcp6-rcv-process",
2865  /* Takes a vector of packets. */
2866  .vector_size = sizeof (u32),
2867  .n_errors = TCP_N_ERROR,
2868  .error_strings = tcp_error_strings,
2869  .n_next_nodes = TCP_RCV_PROCESS_N_NEXT,
2870  .next_nodes =
2871  {
2872 #define _(s,n) [TCP_RCV_PROCESS_NEXT_##s] = n,
2874 #undef _
2875  },
2876  .format_trace = format_tcp_rx_trace_short,
2877 };
2878 /* *INDENT-ON* */
2879 
2881 
2884 
2885 /**
2886  * LISTEN state processing as per RFC 793 p. 65
2887  */
2890  vlib_frame_t * from_frame, int is_ip4)
2891 {
2892  u32 n_left_from, next_index, *from, *to_next, n_syns = 0;
2893  u32 my_thread_index = vm->thread_index;
2894 
2895  from = vlib_frame_vector_args (from_frame);
2896  n_left_from = from_frame->n_vectors;
2897 
2898  next_index = node->cached_next_index;
2899 
2900  while (n_left_from > 0)
2901  {
2902  u32 n_left_to_next;
2903 
2904  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2905 
2906  while (n_left_from > 0 && n_left_to_next > 0)
2907  {
2908  u32 bi0;
2909  vlib_buffer_t *b0;
2910  tcp_rx_trace_t *t0;
2911  tcp_header_t *th0 = 0;
2912  tcp_connection_t *lc0;
2913  ip4_header_t *ip40;
2914  ip6_header_t *ip60;
2915  tcp_connection_t *child0;
2916  u32 error0 = TCP_ERROR_NONE, next0 = tcp_next_drop (is_ip4);
2917 
2918  bi0 = from[0];
2919  to_next[0] = bi0;
2920  from += 1;
2921  to_next += 1;
2922  n_left_from -= 1;
2923  n_left_to_next -= 1;
2924 
2925  b0 = vlib_get_buffer (vm, bi0);
2926  lc0 = tcp_listener_get (vnet_buffer (b0)->tcp.connection_index);
2927 
2928  if (is_ip4)
2929  {
2930  ip40 = vlib_buffer_get_current (b0);
2931  th0 = ip4_next_header (ip40);
2932  }
2933  else
2934  {
2935  ip60 = vlib_buffer_get_current (b0);
2936  th0 = ip6_next_header (ip60);
2937  }
2938 
2939  /* Create child session. For syn-flood protection use filter */
2940 
2941  /* 1. first check for an RST: handled in dispatch */
2942  /* if (tcp_rst (th0))
2943  goto drop; */
2944 
2945  /* 2. second check for an ACK: handled in dispatch */
2946  /* if (tcp_ack (th0))
2947  {
2948  tcp_send_reset (b0, is_ip4);
2949  goto drop;
2950  } */
2951 
2952  /* 3. check for a SYN (did that already) */
2953 
2954  /* Make sure connection wasn't just created */
2955  child0 = tcp_lookup_connection (lc0->c_fib_index, b0,
2956  my_thread_index, is_ip4);
2957  if (PREDICT_FALSE (child0->state != TCP_STATE_LISTEN))
2958  {
2959  error0 = TCP_ERROR_CREATE_EXISTS;
2960  goto drop;
2961  }
2962 
2963  /* Create child session and send SYN-ACK */
2964  child0 = tcp_connection_new (my_thread_index);
2965  child0->c_lcl_port = th0->dst_port;
2966  child0->c_rmt_port = th0->src_port;
2967  child0->c_is_ip4 = is_ip4;
2968  child0->state = TCP_STATE_SYN_RCVD;
2969  child0->c_fib_index = lc0->c_fib_index;
2970 
2971  if (is_ip4)
2972  {
2973  child0->c_lcl_ip4.as_u32 = ip40->dst_address.as_u32;
2974  child0->c_rmt_ip4.as_u32 = ip40->src_address.as_u32;
2975  }
2976  else
2977  {
2978  clib_memcpy (&child0->c_lcl_ip6, &ip60->dst_address,
2979  sizeof (ip6_address_t));
2980  clib_memcpy (&child0->c_rmt_ip6, &ip60->src_address,
2981  sizeof (ip6_address_t));
2982  }
2983 
2984  if (tcp_options_parse (th0, &child0->rcv_opts))
2985  {
2986  clib_warning ("options parse fail");
2987  goto drop;
2988  }
2989 
2990  child0->irs = vnet_buffer (b0)->tcp.seq_number;
2991  child0->rcv_nxt = vnet_buffer (b0)->tcp.seq_number + 1;
2992  child0->rcv_las = child0->rcv_nxt;
2993  child0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2994 
2995  /* RFC1323: TSval timestamps sent on {SYN} and {SYN,ACK}
2996  * segments are used to initialize PAWS. */
2997  if (tcp_opts_tstamp (&child0->rcv_opts))
2998  {
2999  child0->tsval_recent = child0->rcv_opts.tsval;
3000  child0->tsval_recent_age = tcp_time_now ();
3001  }
3002 
3003  if (tcp_opts_wscale (&child0->rcv_opts))
3004  child0->snd_wscale = child0->rcv_opts.wscale;
3005 
3006  child0->snd_wnd = clib_net_to_host_u16 (th0->window)
3007  << child0->snd_wscale;
3008  child0->snd_wl1 = vnet_buffer (b0)->tcp.seq_number;
3009  child0->snd_wl2 = vnet_buffer (b0)->tcp.ack_number;
3010 
3011  tcp_connection_init_vars (child0);
3012  TCP_EVT_DBG (TCP_EVT_SYN_RCVD, child0, 1);
3013 
3014  if (stream_session_accept (&child0->connection, lc0->c_s_index,
3015  0 /* notify */ ))
3016  {
3017  clib_warning ("session accept fail");
3018  tcp_connection_cleanup (child0);
3019  error0 = TCP_ERROR_CREATE_SESSION_FAIL;
3020  goto drop;
3021  }
3022 
3023  /* Reuse buffer to make syn-ack and send */
3024  tcp_make_synack (child0, b0);
3025  next0 = tcp_next_output (is_ip4);
3026  tcp_timer_set (child0, TCP_TIMER_ESTABLISH, TCP_SYN_RCVD_TIME);
3027 
3028  drop:
3029  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
3030  {
3031  t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
3032  clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header));
3033  clib_memcpy (&t0->tcp_connection, lc0,
3034  sizeof (t0->tcp_connection));
3035  }
3036 
3037  n_syns += (error0 == TCP_ERROR_NONE);
3038  b0->error = node->errors[error0];
3039 
3040  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
3041  n_left_to_next, bi0, next0);
3042  }
3043 
3044  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3045  }
3046 
3047  tcp_inc_counter (listen, TCP_ERROR_SYNS_RCVD, n_syns);
3048  return from_frame->n_vectors;
3049 }
3050 
3051 static uword
3053  vlib_frame_t * from_frame)
3054 {
3055  return tcp46_listen_inline (vm, node, from_frame, 1 /* is_ip4 */ );
3056 }
3057 
3058 static uword
3060  vlib_frame_t * from_frame)
3061 {
3062  return tcp46_listen_inline (vm, node, from_frame, 0 /* is_ip4 */ );
3063 }
3064 
3065 /* *INDENT-OFF* */
3067 {
3068  .function = tcp4_listen,
3069  .name = "tcp4-listen",
3070  /* Takes a vector of packets. */
3071  .vector_size = sizeof (u32),
3072  .n_errors = TCP_N_ERROR,
3073  .error_strings = tcp_error_strings,
3074  .n_next_nodes = TCP_LISTEN_N_NEXT,
3075  .next_nodes =
3076  {
3077 #define _(s,n) [TCP_LISTEN_NEXT_##s] = n,
3079 #undef _
3080  },
3081  .format_trace = format_tcp_rx_trace_short,
3082 };
3083 /* *INDENT-ON* */
3084 
3086 
3087 /* *INDENT-OFF* */
3089 {
3090  .function = tcp6_listen,
3091  .name = "tcp6-listen",
3092  /* Takes a vector of packets. */
3093  .vector_size = sizeof (u32),
3094  .n_errors = TCP_N_ERROR,
3095  .error_strings = tcp_error_strings,
3096  .n_next_nodes = TCP_LISTEN_N_NEXT,
3097  .next_nodes =
3098  {
3099 #define _(s,n) [TCP_LISTEN_NEXT_##s] = n,
3101 #undef _
3102  },
3103  .format_trace = format_tcp_rx_trace_short,
3104 };
3105 /* *INDENT-ON* */
3106 
3108 
3111 
3112 typedef enum _tcp_input_next
3113 {
3123 
3124 #define foreach_tcp4_input_next \
3125  _ (DROP, "ip4-drop") \
3126  _ (LISTEN, "tcp4-listen") \
3127  _ (RCV_PROCESS, "tcp4-rcv-process") \
3128  _ (SYN_SENT, "tcp4-syn-sent") \
3129  _ (ESTABLISHED, "tcp4-established") \
3130  _ (RESET, "tcp4-reset") \
3131  _ (PUNT, "ip4-punt")
3132 
3133 #define foreach_tcp6_input_next \
3134  _ (DROP, "ip6-drop") \
3135  _ (LISTEN, "tcp6-listen") \
3136  _ (RCV_PROCESS, "tcp6-rcv-process") \
3137  _ (SYN_SENT, "tcp6-syn-sent") \
3138  _ (ESTABLISHED, "tcp6-established") \
3139  _ (RESET, "tcp6-reset") \
3140  _ (PUNT, "ip6-punt")
3141 
3142 #define filter_flags (TCP_FLAG_SYN|TCP_FLAG_ACK|TCP_FLAG_RST|TCP_FLAG_FIN)
3143 
3144 static void
3146  vlib_buffer_t ** bs, u32 n_bufs, u8 is_ip4)
3147 {
3148  tcp_connection_t *tc;
3149  tcp_header_t *tcp;
3150  tcp_rx_trace_t *t;
3151  int i;
3152 
3153  for (i = 0; i < n_bufs; i++)
3154  {
3155  if (bs[i]->flags & VLIB_BUFFER_IS_TRACED)
3156  {
3157  t = vlib_add_trace (vm, node, bs[i], sizeof (*t));
3158  tc = tcp_connection_get (vnet_buffer (bs[i])->tcp.connection_index,
3159  vm->thread_index);
3160  tcp = vlib_buffer_get_current (bs[i]);
3161  tcp_set_rx_trace_data (t, tc, tcp, bs[i], is_ip4);
3162  }
3163  }
3164 }
3165 
3166 static void
3167 tcp_input_set_error_next (tcp_main_t * tm, u16 * next, u32 * error, u8 is_ip4)
3168 {
3169  if (*error == TCP_ERROR_FILTERED)
3170  {
3171  *next = TCP_INPUT_NEXT_DROP;
3172  }
3173  else if ((is_ip4 && tm->punt_unknown4) || (!is_ip4 && tm->punt_unknown6))
3174  {
3175  *next = TCP_INPUT_NEXT_PUNT;
3176  *error = TCP_ERROR_PUNT;
3177  }
3178  else
3179  {
3180  *next = TCP_INPUT_NEXT_RESET;
3181  *error = TCP_ERROR_NO_LISTENER;
3182  }
3183 }
3184 
3185 static inline tcp_connection_t *
3186 tcp_input_lookup_buffer (vlib_buffer_t * b, u8 thread_index, u32 * error,
3187  u8 is_ip4)
3188 {
3189  u32 fib_index = vnet_buffer (b)->ip.fib_index;
3190  int n_advance_bytes, n_data_bytes;
3192  tcp_header_t *tcp;
3193  u8 is_filtered = 0;
3194 
3195  if (is_ip4)
3196  {
3198  tcp = ip4_next_header (ip4);
3199  vnet_buffer (b)->tcp.hdr_offset = (u8 *) tcp - (u8 *) ip4;
3200  n_advance_bytes = (ip4_header_bytes (ip4) + tcp_header_bytes (tcp));
3201  n_data_bytes = clib_net_to_host_u16 (ip4->length) - n_advance_bytes;
3202 
3203  /* Length check. Checksum computed by ipx_local no need to compute again */
3204  if (PREDICT_FALSE (n_advance_bytes < 0))
3205  {
3206  *error = TCP_ERROR_LENGTH;
3207  return 0;
3208  }
3209 
3210  tc = session_lookup_connection_wt4 (fib_index, &ip4->dst_address,
3211  &ip4->src_address, tcp->dst_port,
3212  tcp->src_port, TRANSPORT_PROTO_TCP,
3213  thread_index, &is_filtered);
3214  }
3215  else
3216  {
3218  tcp = ip6_next_header (ip6);
3219  vnet_buffer (b)->tcp.hdr_offset = (u8 *) tcp - (u8 *) ip6;
3220  n_advance_bytes = tcp_header_bytes (tcp);
3221  n_data_bytes = clib_net_to_host_u16 (ip6->payload_length)
3222  - n_advance_bytes;
3223  n_advance_bytes += sizeof (ip6[0]);
3224 
3225  if (PREDICT_FALSE (n_advance_bytes < 0))
3226  {
3227  *error = TCP_ERROR_LENGTH;
3228  return 0;
3229  }
3230 
3231  tc = session_lookup_connection_wt6 (fib_index, &ip6->dst_address,
3232  &ip6->src_address, tcp->dst_port,
3233  tcp->src_port, TRANSPORT_PROTO_TCP,
3234  thread_index, &is_filtered);
3235  }
3236 
3237  vnet_buffer (b)->tcp.seq_number = clib_net_to_host_u32 (tcp->seq_number);
3238  vnet_buffer (b)->tcp.ack_number = clib_net_to_host_u32 (tcp->ack_number);
3239  vnet_buffer (b)->tcp.data_offset = n_advance_bytes;
3240  vnet_buffer (b)->tcp.data_len = n_data_bytes;
3241  vnet_buffer (b)->tcp.flags = 0;
3242 
3243  *error = is_filtered ? TCP_ERROR_FILTERED : *error;
3244 
3246 }
3247 
3248 static inline void
3250  vlib_buffer_t * b, u16 * next, u32 * error)
3251 {
3252  tcp_header_t *tcp;
3253  u8 flags;
3254 
3255  tcp = tcp_buffer_hdr (b);
3256  flags = tcp->flags & filter_flags;
3257  *next = tm->dispatch_table[tc->state][flags].next;
3258  *error = tm->dispatch_table[tc->state][flags].error;
3259 
3260  if (PREDICT_FALSE (*error == TCP_ERROR_DISPATCH
3261  || *next == TCP_INPUT_NEXT_RESET))
3262  {
3263  /* Overload tcp flags to store state */
3264  tcp_state_t state = tc->state;
3265  vnet_buffer (b)->tcp.flags = tc->state;
3266 
3267  if (*error == TCP_ERROR_DISPATCH)
3268  clib_warning ("disp error state %U flags %U", format_tcp_state,
3269  state, format_tcp_flags, (int) flags);
3270  }
3271 }
3272 
3275  vlib_frame_t * frame, int is_ip4)
3276 {
3277  u32 n_left_from, *from, thread_index = vm->thread_index;
3278  tcp_main_t *tm = vnet_get_tcp_main ();
3279  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
3280  u16 nexts[VLIB_FRAME_SIZE], *next;
3281 
3282  tcp_set_time_now (thread_index);
3283 
3284  from = vlib_frame_vector_args (frame);
3285  n_left_from = frame->n_vectors;
3286  vlib_get_buffers (vm, from, bufs, n_left_from);
3287 
3288  b = bufs;
3289  next = nexts;
3290 
3291  while (n_left_from >= 4)
3292  {
3293  u32 error0 = TCP_ERROR_NO_LISTENER, error1 = TCP_ERROR_NO_LISTENER;
3294  tcp_connection_t *tc0, *tc1;
3295 
3296  {
3297  vlib_prefetch_buffer_header (b[2], STORE);
3298  CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
3299 
3300  vlib_prefetch_buffer_header (b[3], STORE);
3301  CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
3302  }
3303 
3304  next[0] = next[1] = TCP_INPUT_NEXT_DROP;
3305 
3306  tc0 = tcp_input_lookup_buffer (b[0], thread_index, &error0, is_ip4);
3307  tc1 = tcp_input_lookup_buffer (b[1], thread_index, &error1, is_ip4);
3308 
3309  if (PREDICT_TRUE (!tc0 + !tc1 == 0))
3310  {
3311  ASSERT (tcp_lookup_is_valid (tc0, tcp_buffer_hdr (b[0])));
3312  ASSERT (tcp_lookup_is_valid (tc1, tcp_buffer_hdr (b[1])));
3313 
3314  vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
3315  vnet_buffer (b[1])->tcp.connection_index = tc1->c_c_index;
3316 
3317  tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], &error0);
3318  tcp_input_dispatch_buffer (tm, tc1, b[1], &next[1], &error1);
3319  }
3320  else
3321  {
3322  if (PREDICT_TRUE (tc0 != 0))
3323  {
3324  ASSERT (tcp_lookup_is_valid (tc0, tcp_buffer_hdr (b[0])));
3325  vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
3326  tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], &error0);
3327  }
3328  else
3329  tcp_input_set_error_next (tm, &next[0], &error0, is_ip4);
3330 
3331  if (PREDICT_TRUE (tc1 != 0))
3332  {
3333  ASSERT (tcp_lookup_is_valid (tc1, tcp_buffer_hdr (b[1])));
3334  vnet_buffer (b[1])->tcp.connection_index = tc1->c_c_index;
3335  tcp_input_dispatch_buffer (tm, tc1, b[1], &next[1], &error1);
3336  }
3337  else
3338  tcp_input_set_error_next (tm, &next[1], &error1, is_ip4);
3339  }
3340 
3341  b += 2;
3342  next += 2;
3343  n_left_from -= 2;
3344  }
3345  while (n_left_from > 0)
3346  {
3347  tcp_connection_t *tc0;
3348  u32 error0 = TCP_ERROR_NO_LISTENER;
3349 
3350  if (n_left_from > 1)
3351  {
3352  vlib_prefetch_buffer_header (b[1], STORE);
3353  CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
3354  }
3355 
3356  next[0] = TCP_INPUT_NEXT_DROP;
3357  tc0 = tcp_input_lookup_buffer (b[0], thread_index, &error0, is_ip4);
3358  if (PREDICT_TRUE (tc0 != 0))
3359  {
3360  ASSERT (tcp_lookup_is_valid (tc0, tcp_buffer_hdr (b[0])));
3361  vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
3362  tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], &error0);
3363  }
3364  else
3365  tcp_input_set_error_next (tm, &next[0], &error0, is_ip4);
3366 
3367  b += 1;
3368  next += 1;
3369  n_left_from -= 1;
3370  }
3371 
3373  tcp_input_trace_frame (vm, node, bufs, frame->n_vectors, is_ip4);
3374 
3375  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
3376  return frame->n_vectors;
3377 }
3378 
3379 static uword
3381  vlib_frame_t * from_frame)
3382 {
3383  return tcp46_input_inline (vm, node, from_frame, 1 /* is_ip4 */ );
3384 }
3385 
3386 static uword
3388  vlib_frame_t * from_frame)
3389 {
3390  return tcp46_input_inline (vm, node, from_frame, 0 /* is_ip4 */ );
3391 }
3392 
3393 /* *INDENT-OFF* */
3395 {
3396  .function = tcp4_input,
3397  .name = "tcp4-input",
3398  /* Takes a vector of packets. */
3399  .vector_size = sizeof (u32),
3400  .n_errors = TCP_N_ERROR,
3401  .error_strings = tcp_error_strings,
3402  .n_next_nodes = TCP_INPUT_N_NEXT,
3403  .next_nodes =
3404  {
3405 #define _(s,n) [TCP_INPUT_NEXT_##s] = n,
3407 #undef _
3408  },
3409  .format_buffer = format_tcp_header,
3410  .format_trace = format_tcp_rx_trace,
3411 };
3412 /* *INDENT-ON* */
3413 
3415 
3416 /* *INDENT-OFF* */
3418 {
3419  .function = tcp6_input,
3420  .name = "tcp6-input",
3421  /* Takes a vector of packets. */
3422  .vector_size = sizeof (u32),
3423  .n_errors = TCP_N_ERROR,
3424  .error_strings = tcp_error_strings,
3425  .n_next_nodes = TCP_INPUT_N_NEXT,
3426  .next_nodes =
3427  {
3428 #define _(s,n) [TCP_INPUT_NEXT_##s] = n,
3430 #undef _
3431  },
3432  .format_buffer = format_tcp_header,
3433  .format_trace = format_tcp_rx_trace,
3434 };
3435 /* *INDENT-ON* */
3436 
3438 
3439 static void
3441 {
3442  int i, j;
3443  for (i = 0; i < ARRAY_LEN (tm->dispatch_table); i++)
3444  for (j = 0; j < ARRAY_LEN (tm->dispatch_table[i]); j++)
3445  {
3446  tm->dispatch_table[i][j].next = TCP_INPUT_NEXT_DROP;
3447  tm->dispatch_table[i][j].error = TCP_ERROR_DISPATCH;
3448  }
3449 
3450 #define _(t,f,n,e) \
3451 do { \
3452  tm->dispatch_table[TCP_STATE_##t][f].next = (n); \
3453  tm->dispatch_table[TCP_STATE_##t][f].error = (e); \
3454 } while (0)
3455 
3456  /* SYNs for new connections -> tcp-listen. */
3457  _(LISTEN, TCP_FLAG_SYN, TCP_INPUT_NEXT_LISTEN, TCP_ERROR_NONE);
3458  _(LISTEN, TCP_FLAG_ACK, TCP_INPUT_NEXT_RESET, TCP_ERROR_NONE);
3459  _(LISTEN, TCP_FLAG_RST, TCP_INPUT_NEXT_DROP, TCP_ERROR_RST_RCVD);
3461  TCP_ERROR_NONE);
3462  /* ACK for for a SYN-ACK -> tcp-rcv-process. */
3463  _(SYN_RCVD, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3464  _(SYN_RCVD, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3466  TCP_ERROR_NONE);
3467  _(SYN_RCVD, TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3469  TCP_ERROR_NONE);
3470  /* SYN-ACK for a SYN */
3472  TCP_ERROR_NONE);
3473  _(SYN_SENT, TCP_FLAG_ACK, TCP_INPUT_NEXT_SYN_SENT, TCP_ERROR_NONE);
3474  _(SYN_SENT, TCP_FLAG_RST, TCP_INPUT_NEXT_SYN_SENT, TCP_ERROR_NONE);
3476  TCP_ERROR_NONE);
3477  /* ACK for for established connection -> tcp-established. */
3478  _(ESTABLISHED, TCP_FLAG_ACK, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
3479  /* FIN for for established connection -> tcp-established. */
3480  _(ESTABLISHED, TCP_FLAG_FIN, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
3482  TCP_ERROR_NONE);
3483  _(ESTABLISHED, TCP_FLAG_RST, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
3485  TCP_ERROR_NONE);
3486  _(ESTABLISHED, TCP_FLAG_SYN, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
3488  TCP_ERROR_NONE);
3489  /* ACK or FIN-ACK to our FIN */
3490  _(FIN_WAIT_1, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3492  TCP_ERROR_NONE);
3493  /* FIN in reply to our FIN from the other side */
3494  _(FIN_WAIT_1, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3495  _(FIN_WAIT_1, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3496  _(CLOSING, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3497  /* FIN confirming that the peer (app) has closed */
3498  _(FIN_WAIT_2, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3499  _(FIN_WAIT_2, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3501  TCP_ERROR_NONE);
3502  _(CLOSE_WAIT, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3504  TCP_ERROR_NONE);
3505  _(LAST_ACK, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3506  _(LAST_ACK, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3508  TCP_ERROR_NONE);
3509  _(LAST_ACK, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3510  _(LAST_ACK, TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3511  _(TIME_WAIT, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3513  TCP_ERROR_NONE);
3514  _(TIME_WAIT, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3515  _(TIME_WAIT, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3516  _(CLOSED, TCP_FLAG_ACK, TCP_INPUT_NEXT_DROP, TCP_ERROR_CONNECTION_CLOSED);
3517  _(CLOSED, TCP_FLAG_RST, TCP_INPUT_NEXT_DROP, TCP_ERROR_CONNECTION_CLOSED);
3519  TCP_ERROR_CONNECTION_CLOSED);
3520 #undef _
3521 }
3522 
3523 static clib_error_t *
3525 {
3526  clib_error_t *error = 0;
3527  tcp_main_t *tm = vnet_get_tcp_main ();
3528 
3529  if ((error = vlib_call_init_function (vm, tcp_init)))
3530  return error;
3531 
3532  /* Initialize dispatch table. */
3534 
3535  return error;
3536 }
3537 
3539 
3540 /*
3541  * fd.io coding-style-patch-verification: ON
3542  *
3543  * Local Variables:
3544  * eval: (c-set-style "gnu")
3545  * End:
3546  */
#define tcp_in_cong_recovery(tc)
Definition: tcp.h:350
static int tcp_session_enqueue_ooo(tcp_connection_t *tc, vlib_buffer_t *b, u16 data_len)
Enqueue out-of-order data.
Definition: tcp_input.c:1576
static void tcp_update_timestamp(tcp_connection_t *tc, u32 seq, u32 seq_end)
Update tsval recent.
Definition: tcp_input.c:247
static sack_scoreboard_hole_t * scoreboard_insert_hole(sack_scoreboard_t *sb, u32 prev_index, u32 start, u32 end)
Definition: tcp_input.c:631
static u8 tcp_scoreboard_is_sane_post_recovery(tcp_connection_t *tc)
Test that scoreboard is sane after recovery.
Definition: tcp_input.c:820
void scoreboard_clear(sack_scoreboard_t *sb)
Definition: tcp_input.c:794
static u8 tcp_should_fastrecover(tcp_connection_t *tc)
Definition: tcp_input.c:1155
#define TCP_2MSL_TIME
Definition: tcp.h:101
End of options.
Definition: tcp_packet.h:104
#define tcp_next_drop(is_ip4)
Definition: tcp_input.c:79
#define tcp_fastrecovery_1_smss_off(tc)
Definition: tcp.h:348
#define clib_min(x, y)
Definition: clib.h:291
#define CLIB_UNUSED(x)
Definition: clib.h:81
static void tcp_cc_update(tcp_connection_t *tc, vlib_buffer_t *b)
Definition: tcp_input.c:1127
vlib_node_registration_t tcp6_rcv_process_node
(constructor) VLIB_REGISTER_NODE (tcp6_rcv_process_node)
Definition: tcp_input.c:2460
#define tcp_in_recovery(tc)
Definition: tcp.h:344
static void tcp_retransmit_timer_set(tcp_connection_t *tc)
Definition: tcp.h:745
#define TCP_OPTION_LEN_SACK_PERMITTED
Definition: tcp_packet.h:167
static int tcp_rcv_ack_is_acceptable(tcp_connection_t *tc0, vlib_buffer_t *tb0)
Definition: tcp_input.c:392
#define seq_leq(_s1, _s2)
Definition: tcp.h:555
void tcp_make_fin(tcp_connection_t *tc, vlib_buffer_t *b)
Convert buffer to FIN-ACK.
Definition: tcp_output.c:574
struct _sack_block sack_block_t
void tcp_rcv_sacks(tcp_connection_t *tc, u32 ack)
Definition: tcp_input.c:829
#define timestamp_leq(_t1, _t2)
Definition: tcp.h:562
ip4_address_t src_address
Definition: ip4_packet.h:169
static u8 tcp_cc_is_spurious_retransmit(tcp_connection_t *tc)
Definition: tcp_input.c:1097
enum _tcp_state_next tcp_state_next_t
struct _transport_connection transport_connection_t
#define tcp_rst(_th)
Definition: tcp_packet.h:81
#define TCP_TIMEWAIT_TIME
Definition: tcp.h:103
static uword tcp6_input(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_input.c:3387
Selective Ack permitted.
Definition: tcp_packet.h:108
#define TCP_FLAG_SYN
Definition: fa_node.h:13
#define tcp_opts_tstamp(_to)
Definition: tcp_packet.h:157
#define PREDICT_TRUE(x)
Definition: clib.h:108
void tcp_fast_retransmit(tcp_connection_t *tc)
Do fast retransmit.
Definition: tcp_output.c:1827
tcp_connection_t * tcp_connection_new(u8 thread_index)
Definition: tcp.c:249
#define tcp_inc_err_counter(cnts, err, val)
Definition: tcp_input.c:1858
unsigned long u64
Definition: types.h:89
#define tcp_store_err_counters(node_id, cnts)
Definition: tcp_input.c:1862
static void tcp_dispatch_table_init(tcp_main_t *tm)
Definition: tcp_input.c:3440
static u8 * format_tcp_rx_trace_short(u8 *s, va_list *args)
Definition: tcp_input.c:1770
struct _sack_scoreboard sack_scoreboard_t
static uword tcp46_established_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_ip4)
Definition: tcp_input.c:1872
static tcp_connection_t * tcp_half_open_connection_get(u32 conn_index)
Definition: tcp.h:523
void tcp_update_rto(tcp_connection_t *tc)
Definition: tcp_input.c:432
static int tcp_segment_rcv(tcp_connection_t *tc, vlib_buffer_t *b, u32 *next0)
Receive buffer for connection and handle acks.
Definition: tcp_input.c:1679
#define tcp_doff(_th)
Definition: tcp_packet.h:78
struct _tcp_main tcp_main_t
u32 thread_index
Definition: main.h:179
void tcp_connection_timers_reset(tcp_connection_t *tc)
Stop all connection timers.
Definition: tcp.c:404
static int tcp_segment_validate(vlib_main_t *vm, tcp_connection_t *tc0, vlib_buffer_t *b0, tcp_header_t *th0, u32 *next0, u32 *error0)
Validate incoming segment as per RFC793 p.
Definition: tcp_input.c:275
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
static int tcp_options_parse(tcp_header_t *th, tcp_options_t *to)
Parse TCP header options.
Definition: tcp_input.c:127
#define tcp_recovery_off(tc)
Definition: tcp.h:342
#define clib_abs(x)
Definition: clib.h:298
static int tcp_update_rtt(tcp_connection_t *tc, u32 ack)
Update RTT estimate and RTO timer.
Definition: tcp_input.c:450
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
vlib_node_registration_t tcp4_rcv_process_node
(constructor) VLIB_REGISTER_NODE (tcp4_rcv_process_node)
Definition: tcp_input.c:2459
struct _tcp_connection tcp_connection_t
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
static u32 tcp_available_cc_snd_space(const tcp_connection_t *tc)
Estimate of how many bytes we can still push into the network.
Definition: tcp.h:643
#define tcp_opts_sack(_to)
Definition: tcp_packet.h:159
void tcp_fast_retransmit_sack(tcp_connection_t *tc)
Do fast retransmit with SACKs.
Definition: tcp_output.c:1707
tcp_connection_t tcp_connection
Definition: tcp_input.c:1750
static u8 tcp_sack_vector_is_sane(sack_block_t *sacks)
Definition: tcp_input.c:1434
static tcp_connection_t * tcp_get_connection_from_transport(transport_connection_t *tconn)
Definition: tcp.h:496
static void tcp_cc_congestion_undo(tcp_connection_t *tc)
Definition: tcp_input.c:1083
int session_enqueue_stream_connection(transport_connection_t *tc, vlib_buffer_t *b, u32 offset, u8 queue_event, u8 is_in_order)
Definition: session.c:355
u64 session_lookup_half_open_handle(transport_connection_t *tc)
void tcp_send_reset_w_pkt(tcp_connection_t *tc, vlib_buffer_t *pkt, u8 is_ip4)
Send reset without reusing existing buffer.
Definition: tcp_output.c:831
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:472
No operation.
Definition: tcp_packet.h:105
format_function_t format_tcp_flags
Definition: tcp.h:64
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:228
u8 n_sack_blocks
Number of SACKs blocks.
Definition: tcp_packet.h:152
struct _tcp_header tcp_header_t
int tcp_half_open_connection_cleanup(tcp_connection_t *tc)
Try to cleanup half-open connection.
Definition: tcp.c:170
static uword tcp6_listen(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_input.c:3059
ip6_address_t src_address
Definition: ip6_packet.h:378
unsigned char u8
Definition: types.h:56
#define tcp_inc_counter(node_id, err, count)
Definition: tcp_input.c:1850
vlib_node_registration_t tcp6_syn_sent_node
(constructor) VLIB_REGISTER_NODE (tcp6_syn_sent_node)
Definition: tcp_input.c:2056
struct _sack_scoreboard_hole sack_scoreboard_hole_t
u8 wscale
Window scale advertised.
Definition: tcp_packet.h:148
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
static tcp_connection_t * tcp_lookup_connection(u32 fib_index, vlib_buffer_t *b, u8 thread_index, u8 is_ip4)
Lookup transport connection.
Definition: tcp_input.c:2097
static void tcp_dequeue_acked(tcp_connection_t *tc, u32 ack)
Dequeue bytes that have been acked and while at it update RTT estimates.
Definition: tcp_input.c:498
#define tcp_fastrecovery_on(tc)
Definition: tcp.h:339
void tcp_flush_frame_to_output(vlib_main_t *vm, u8 thread_index, u8 is_ip4)
Flush tx frame populated by retransmits and timer pops.
Definition: tcp_output.c:1023
Limit MSS.
Definition: tcp_packet.h:106
static uword tcp4_listen(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_input.c:3052
static void scoreboard_init_high_rxt(sack_scoreboard_t *sb, u32 seq)
Definition: tcp_input.c:773
sack_scoreboard_hole_t * scoreboard_get_hole(sack_scoreboard_t *sb, u32 index)
Definition: tcp_input.c:556
memset(h->entries, 0, sizeof(h->entries[0])*entries)
#define tcp_is_fin(_th)
Definition: tcp_packet.h:90
static uword tcp6_rcv_process(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_input.c:2832
static uword tcp4_syn_sent(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_input.c:2402
#define seq_gt(_s1, _s2)
Definition: tcp.h:556
static u8 * format_tcp_rx_trace(u8 *s, va_list *args)
Definition: tcp_input.c:1754
void tcp_init_snd_vars(tcp_connection_t *tc)
Initialize connection send variables.
Definition: tcp.c:539
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
vlib_node_registration_t tcp4_established_node
(constructor) VLIB_REGISTER_NODE (tcp4_established_node)
Definition: tcp_input.c:82
#define TCP_CLOSEWAIT_TIME
Definition: tcp.h:102
void stream_session_accept_notify(transport_connection_t *tc)
Definition: session.c:734
#define always_inline
Definition: clib.h:94
#define TCP_OPTION_LEN_SACK_BLOCK
Definition: tcp_packet.h:169
ip4_address_t dst_address
Definition: ip4_packet.h:169
#define TCP_FLAG_ACK
Definition: fa_node.h:16
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
#define TCP_DELACK_TIME
Definition: tcp.h:98
static tcp_header_t * tcp_buffer_hdr(vlib_buffer_t *b)
Definition: tcp.h:456
static void tcp_cc_recovery_exit(tcp_connection_t *tc)
Definition: tcp_input.c:1060
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:187
enum _tcp_state tcp_state_t
#define TCP_ALWAYS_ACK
On/off delayed acks.
Definition: tcp.h:38
vlib_node_registration_t tcp6_input_node
(constructor) VLIB_REGISTER_NODE (tcp6_input_node)
Definition: tcp_input.c:3110
static u8 tcp_ack_is_dupack(tcp_connection_t *tc, vlib_buffer_t *b, u32 prev_snd_wnd, u32 prev_snd_una)
Check if duplicate ack as per RFC5681 Sec.
Definition: tcp_input.c:518
vhost_vring_state_t state
Definition: vhost_user.h:120
#define TCP_RTO_MAX
Definition: tcp.h:107
static u32 ooo_segment_length(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:294
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:240
static u32 tcp_time_now(void)
Definition: tcp.h:677
sack_block_t * sacks
SACK blocks.
Definition: tcp_packet.h:151
unsigned int u32
Definition: types.h:88
#define vec_end(v)
End (last data address) of vector.
struct _stream_session_t stream_session_t
#define vlib_call_init_function(vm, x)
Definition: init.h:260
static void tcp_node_inc_counter_i(vlib_main_t *vm, u32 tcp4_node, u32 tcp6_node, u8 is_ip4, u32 evt, u32 val)
Definition: tcp_input.c:1834
#define TCP_MAX_SACK_BLOCKS
Max number of SACK blocks stored.
Definition: tcp.h:162
#define VLIB_FRAME_SIZE
Definition: node.h:382
#define tcp_validate_txf_size(_tc, _a)
Definition: tcp.h:808
#define TCP_EVT_DBG(_evt, _args...)
Definition: tcp_debug.h:238
#define timestamp_lt(_t1, _t2)
Definition: tcp.h:561
static void tcp_timer_set(tcp_connection_t *tc, u8 timer_id, u32 interval)
Definition: tcp.h:706
#define TCP_OPTION_LEN_WINDOW_SCALE
Definition: tcp_packet.h:166
static void svm_fifo_newest_ooo_segment_reset(svm_fifo_t *f)
Definition: svm_fifo.h:217
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
void scoreboard_init(sack_scoreboard_t *sb)
Definition: tcp_input.c:786
u32 stream_session_dequeue_drop(transport_connection_t *tc, u32 max_bytes)
Definition: session.c:483
#define TCP_INVALID_SACK_HOLE_INDEX
Definition: tcp.h:163
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
void tcp_cc_fastrecovery_exit(tcp_connection_t *tc)
Definition: tcp_input.c:1071
static uword tcp46_listen_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, int is_ip4)
LISTEN state processing as per RFC 793 p.
Definition: tcp_input.c:2889
#define tcp_in_fastrecovery(tc)
Definition: tcp.h:343
static void tcp_input_set_error_next(tcp_main_t *tm, u16 *next, u32 *error, u8 is_ip4)
Definition: tcp_input.c:3167
void tcp_retransmit_first_unacked(tcp_connection_t *tc)
Retransmit first unacked segment.
Definition: tcp_output.c:1684
unsigned short u16
Definition: types.h:57
#define foreach_tcp4_input_next
Definition: tcp_input.c:3124
static u32 ooo_segment_offset(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:282
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:205
#define filter_flags
Definition: tcp_input.c:3142
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:274
static int tcp_buffer_discard_bytes(vlib_buffer_t *b, u32 n_bytes_to_drop)
Definition: tcp_input.c:1646
#define foreach_tcp6_input_next
Definition: tcp_input.c:3133
#define TCP_TIMER_HANDLE_INVALID
Definition: tcp.h:93
void tcp_fast_retransmit_no_sack(tcp_connection_t *tc)
Fast retransmit without SACK info.
Definition: tcp_output.c:1790
static void tcp_input_trace_frame(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t **bs, u32 n_bufs, u8 is_ip4)
Definition: tcp_input.c:3145
#define PREDICT_FALSE(x)
Definition: clib.h:107
#define vec_del1(v, i)
Delete the element at index I.
Definition: vec.h:806
#define TCP_FLAG_FIN
Definition: fa_node.h:12
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
int stream_session_accept(transport_connection_t *tc, u32 listener_index, u8 notify)
Accept a stream session.
Definition: session.c:855
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:364
vlib_node_registration_t tcp4_listen_node
(constructor) VLIB_REGISTER_NODE (tcp4_listen_node)
Definition: tcp_input.c:2882
#define TCP_OPTION_LEN_TIMESTAMP
Definition: tcp_packet.h:168
static u8 tcp_lookup_is_valid(tcp_connection_t *tc, tcp_header_t *hdr)
Definition: tcp_input.c:2059
static ooo_segment_t * svm_fifo_newest_ooo_segment(svm_fifo_t *f)
Definition: svm_fifo.h:209
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:138
u32 tcp_sack_list_bytes(tcp_connection_t *tc)
Definition: tcp_input.c:1511
Selective Ack block.
Definition: tcp_packet.h:109
vlib_node_registration_t tcp6_established_node
(constructor) VLIB_REGISTER_NODE (tcp6_established_node)
Definition: tcp_input.c:83
sack_scoreboard_hole_t * scoreboard_first_hole(sack_scoreboard_t *sb)
Definition: tcp_input.c:580
static int tcp_can_delack(tcp_connection_t *tc)
Check if ACK could be delayed.
Definition: tcp_input.c:1630
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1176
static int tcp_cc_recover(tcp_connection_t *tc)
Definition: tcp_input.c:1106
#define TCP_FLAG_RST
Definition: fa_node.h:14
static stream_session_t * session_get(u32 si, u32 thread_index)
Definition: session.h:314
#define TCP_DBG(_fmt, _args...)
Definition: tcp_debug.h:89
u32 flags
Definition: vhost_user.h:115
#define TCP_MAX_WND_SCALE
Definition: tcp_packet.h:173
static void tcp_timer_reset(tcp_connection_t *tc, u8 timer_id)
Definition: tcp.h:717
static uword tcp6_syn_sent_rcv(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_input.c:2409
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:155
vlib_node_registration_t tcp4_syn_sent_node
(constructor) VLIB_REGISTER_NODE (tcp4_syn_sent_node)
Definition: tcp_input.c:2055
u16 n_vectors
Definition: node.h:401
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:79
vlib_main_t * vm
Definition: buffer.c:294
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_node.h:332
static void tcp_set_rx_trace_data(tcp_rx_trace_t *t0, tcp_connection_t *tc0, tcp_header_t *th0, vlib_buffer_t *b0, u8 is_ip4)
Definition: tcp_input.c:1785
void tcp_send_reset(tcp_connection_t *tc)
Build and set reset packet for connection.
Definition: tcp_output.c:910
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
#define TCP_DUPACK_THRESHOLD
Definition: tcp.h:34
format_function_t format_tcp_state
Definition: tcp.h:63
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:75
static int tcp_rcv_ack(tcp_connection_t *tc, vlib_buffer_t *b, tcp_header_t *th, u32 *next, u32 *error)
Process incoming ACK.
Definition: tcp_input.c:1335
tcp_header_t tcp_header
Definition: tcp_input.c:1749
format_function_t format_tcp_header
Definition: format.h:101
void tcp_make_synack(tcp_connection_t *ts, vlib_buffer_t *b)
Convert buffer to SYN-ACK.
Definition: tcp_output.c:617
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
Definition: pool.h:271
#define ARRAY_LEN(x)
Definition: clib.h:61
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:455
#define TCP_RTT_MAX
Definition: tcp.h:109
sack_scoreboard_hole_t * scoreboard_next_rxt_hole(sack_scoreboard_t *sb, sack_scoreboard_hole_t *start, u8 have_sent_1_smss, u8 *can_rescue, u8 *snd_limited)
Figure out the next hole to retransmit.
Definition: tcp_input.c:718
u16 mss
Option flags, see above.
Definition: tcp_packet.h:147
static void * ip6_next_header(ip6_header_t *i)
Definition: ip6_packet.h:405
void tcp_make_ack(tcp_connection_t *ts, vlib_buffer_t *b)
Convert buffer to ACK.
Definition: tcp_output.c:559
void stream_session_disconnect_notify(transport_connection_t *tc)
Notification from transport that connection is being closed.
Definition: session.c:756
transport_connection_t * session_lookup_connection_wt4(u32 fib_index, ip4_address_t *lcl, ip4_address_t *rmt, u16 lcl_port, u16 rmt_port, u8 proto, u32 thread_index, u8 *is_filtered)
Lookup connection with ip4 and transport layer information.
static void tcp_timer_update(tcp_connection_t *tc, u8 timer_id, u32 interval)
Definition: tcp.h:730
#define TCP_PAWS_IDLE
24 days
Definition: tcp.h:30
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:513
vslo right
#define ASSERT(truth)
#define tcp_syn(_th)
Definition: tcp_packet.h:80
static clib_error_t * tcp_input_init(vlib_main_t *vm)
Definition: tcp_input.c:3524
u32 session_tx_fifo_max_dequeue(transport_connection_t *tc)
Definition: session.c:466
static void tcp_estimate_rtt(tcp_connection_t *tc, u32 mrtt)
Compute smoothed RTT as per VJ&#39;s &#39;88 SIGCOMM and RFC6298.
Definition: tcp_input.c:409
enum _tcp_rcv_process_next tcp_rcv_process_next_t
static uword tcp4_established(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_input.c:1997
#define seq_geq(_s1, _s2)
Definition: tcp.h:557
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:129
static void transport_add_tx_event(transport_connection_t *tc)
Definition: session.h:579
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:218
static int tcp_segment_check_paws(tcp_connection_t *tc)
RFC1323: Check against wrapped sequence numbers (PAWS).
Definition: tcp_input.c:237
static void tcp_cc_handle_event(tcp_connection_t *tc, u32 is_dack)
One function to rule them all ...
Definition: tcp_input.c:1165
static void tcp_established_trace_frame(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, u8 is_ip4)
Definition: tcp_input.c:1800
enum _tcp_input_next tcp_input_next_t
transport_connection_t * session_lookup_connection_wt6(u32 fib_index, ip6_address_t *lcl, ip6_address_t *rmt, u16 lcl_port, u16 rmt_port, u8 proto, u32 thread_index, u8 *is_filtered)
Lookup connection with ip6 and transport layer information.
void tcp_update_sack_list(tcp_connection_t *tc, u32 start, u32 end)
Build SACK list as per RFC2018.
Definition: tcp_input.c:1457
Out-of-order segment.
Definition: svm_fifo.h:27
static u8 tcp_segment_in_rcv_wnd(tcp_connection_t *tc, u32 seq, u32 end_seq)
Validate segment sequence number.
Definition: tcp_input.c:113
#define clib_max(x, y)
Definition: clib.h:284
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static clib_error_t * tcp_init(vlib_main_t *vm)
Definition: tcp.c:1372
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:57
VLIB_NODE_FUNCTION_MULTIARCH(tcp4_established_node, tcp4_established)
static uword tcp6_established(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_input.c:2004
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:155
#define seq_lt(_s1, _s2)
Definition: tcp.h:554
struct _vlib_node_registration vlib_node_registration_t
#define tcp_is_syn(_th)
Definition: tcp_packet.h:89
#define tcp_opts_wscale(_to)
Definition: tcp_packet.h:158
enum _tcp_syn_sent_next tcp_syn_sent_next_t
static void tcp_update_snd_wnd(tcp_connection_t *tc, u32 seq, u32 ack, u32 snd_wnd)
Try to update snd_wnd based on feedback received from peer.
Definition: tcp_input.c:1012
void tcp_connection_reset(tcp_connection_t *tc)
Notify session that connection has been reset.
Definition: tcp.c:266
u32 tsval
Timestamp value.
Definition: tcp_packet.h:149
enum _tcp_established_next tcp_established_next_t
u16 payload_length
Definition: ip6_packet.h:369
u32 tsecr
Echoed/reflected time stamp.
Definition: tcp_packet.h:150
vlib_node_registration_t tcp4_input_node
(constructor) VLIB_REGISTER_NODE (tcp4_input_node)
Definition: tcp_input.c:3109
void tcp_send_fin(tcp_connection_t *tc)
Send FIN.
Definition: tcp_output.c:1071
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
enum _tcp_listen_next tcp_listen_next_t
#define foreach_tcp_state_next
Definition: tcp_input.c:29
static u32 tcp_set_time_now(u32 thread_index)
Definition: tcp.h:683
static u8 tcp_is_lost_fin(tcp_connection_t *tc)
Definition: tcp.h:655
static uword tcp4_rcv_process(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_input.c:2825
static u32 scoreboard_hole_bytes(sack_scoreboard_hole_t *hole)
Definition: tcp_input.c:550
static void tcp_retransmit_timer_update(tcp_connection_t *tc)
Definition: tcp.h:789
static int tcp_session_enqueue_data(tcp_connection_t *tc, vlib_buffer_t *b, u16 data_len)
Enqueue data for delivery to application.
Definition: tcp_input.c:1521
static u8 tcp_should_fastrecover_sack(tcp_connection_t *tc)
Definition: tcp_input.c:1149
u64 uword
Definition: types.h:112
#define seq_max(_s1, _s2)
Definition: tcp.h:558
sack_scoreboard_hole_t * scoreboard_next_hole(sack_scoreboard_t *sb, sack_scoreboard_hole_t *hole)
Definition: tcp_input.c:564
sack_scoreboard_hole_t * scoreboard_prev_hole(sack_scoreboard_t *sb, sack_scoreboard_hole_t *hole)
Definition: tcp_input.c:572
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:267
void tcp_connection_init_vars(tcp_connection_t *tc)
Initialize tcp connection variables.
Definition: tcp.c:563
static void scoreboard_remove_hole(sack_scoreboard_t *sb, sack_scoreboard_hole_t *hole)
Definition: tcp_input.c:596
#define tcp_maybe_inc_counter(node_id, err, count)
Definition: tcp_input.c:1843
#define TCP_OPTION_LEN_MSS
Definition: tcp_packet.h:165
sack_scoreboard_hole_t * scoreboard_last_hole(sack_scoreboard_t *sb)
Definition: tcp_input.c:588
static void scoreboard_update_bytes(tcp_connection_t *tc, sack_scoreboard_t *sb)
Definition: tcp_input.c:668
left
#define tcp_next_output(is_ip4)
Definition: tcp_input.c:76
#define TCP_RTO_MIN
Definition: tcp.h:108
struct clib_bihash_value offset
template key/value backing page structure
static void tcp_retransmit_timer_force_update(tcp_connection_t *tc)
Definition: tcp.h:759
#define tcp_scoreboard_trace_add(_tc, _ack)
Definition: tcp.h:226
u8 * format_tcp_connection(u8 *s, va_list *args)
Definition: tcp.c:826
#define vnet_buffer(b)
Definition: buffer.h:344
int session_manager_flush_enqueue_events(u8 transport_proto, u32 thread_index)
Flushes queue of sessions that are to be notified of new data enqueued events.
Definition: session.c:540
static tcp_connection_t * tcp_connection_get(u32 conn_index, u32 thread_index)
Definition: tcp.h:477
#define tcp_maybe_inc_err_counter(cnts, err)
Definition: tcp_input.c:1854
static u32 scoreboard_hole_index(sack_scoreboard_t *sb, sack_scoreboard_hole_t *hole)
Definition: tcp_input.c:543
static int tcp_header_bytes(tcp_header_t *t)
Definition: tcp_packet.h:93
int session_stream_connect_notify(transport_connection_t *tc, u8 is_fail)
Definition: session.c:594
void tcp_connection_cleanup(tcp_connection_t *tc)
Cleans up connection state.
Definition: tcp.c:199
static uword tcp4_input(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: tcp_input.c:3380
u8 data[0]
Packet data.
Definition: buffer.h:175
static tcp_connection_t * tcp_input_lookup_buffer(vlib_buffer_t *b, u8 thread_index, u32 *error, u8 is_ip4)
Definition: tcp_input.c:3186
u16 flags
Copy of main node flags.
Definition: node.h:507
Window scale.
Definition: tcp_packet.h:107
vlib_node_registration_t tcp6_listen_node
(constructor) VLIB_REGISTER_NODE (tcp6_listen_node)
Definition: tcp_input.c:2883
#define tcp_opts_sack_permitted(_to)
Definition: tcp_packet.h:160
static int ip4_header_bytes(const ip4_header_t *i)
Definition: ip4_packet.h:234
Timestamps.
Definition: tcp_packet.h:110
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:141
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:310
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define TCP_SYN_RCVD_TIME
Definition: tcp.h:100
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:116
static uword tcp46_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_ip4)
Definition: tcp_input.c:3274
static void tcp_persist_timer_set(tcp_connection_t *tc)
Definition: tcp.h:766
static tcp_main_t * vnet_get_tcp_main()
Definition: tcp.h:450
static uword tcp46_syn_sent_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, int is_ip4)
Definition: tcp_input.c:2138
#define tcp_fastrecovery_off(tc)
Definition: tcp.h:340
static uword tcp46_rcv_process_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, int is_ip4)
Handles reception for all states except LISTEN, SYN-SENT and ESTABLISHED as per RFC793 p...
Definition: tcp_input.c:2467
static void tcp_retransmit_timer_reset(tcp_connection_t *tc)
Definition: tcp.h:753
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:58
static void tcp_input_dispatch_buffer(tcp_main_t *tm, tcp_connection_t *tc, vlib_buffer_t *b, u16 *next, u32 *error)
Definition: tcp_input.c:3249
#define tcp_ack(_th)
Definition: tcp_packet.h:83
static u8 tcp_timer_is_active(tcp_connection_t *tc, tcp_timers_e timer)
Definition: tcp.h:803
transport_connection_t * session_lookup_half_open_connection(u64 handle, u8 proto, u8 is_ip4)
Definition: defs.h:46
static tcp_connection_t * tcp_listener_get(u32 tli)
Definition: tcp.h:517
ip6_address_t dst_address
Definition: ip6_packet.h:378
static u8 tcp_ack_is_cc_event(tcp_connection_t *tc, vlib_buffer_t *b, u32 prev_snd_wnd, u32 prev_snd_una, u8 *is_dack)
Checks if ack is a congestion control event.
Definition: tcp_input.c:531
void tcp_cc_init_congestion(tcp_connection_t *tc)
Init loss recovery/fast recovery.
Definition: tcp_input.c:1050
static void tcp_persist_timer_reset(tcp_connection_t *tc)
Definition: tcp.h:783
static char * tcp_error_strings[]
Definition: tcp_input.c:22
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128