FD.io VPP  v18.04-17-g3a0d853
Vector Packet Processing
session_node.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <math.h>
17 #include <vlib/vlib.h>
18 #include <vnet/vnet.h>
19 #include <vppinfra/elog.h>
20 #include <vnet/session/transport.h>
23 #include <svm/queue.h>
24 
26 
27 typedef struct
28 {
32 
33 /* packet trace format function */
34 static u8 *
35 format_session_queue_trace (u8 * s, va_list * args)
36 {
37  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
38  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
39  session_queue_trace_t *t = va_arg (*args, session_queue_trace_t *);
40 
41  s = format (s, "SESSION_QUEUE: session index %d, server thread index %d",
43  return s;
44 }
45 
47 
48 #define foreach_session_queue_error \
49 _(TX, "Packets transmitted") \
50 _(TIMER, "Timer events") \
51 _(NO_BUFFER, "Out of buffers")
52 
53 typedef enum
54 {
55 #define _(sym,str) SESSION_QUEUE_ERROR_##sym,
57 #undef _
60 
61 static char *session_queue_error_strings[] = {
62 #define _(sym,string) string,
64 #undef _
65 };
66 
67 always_inline void
69  u8 thread_index, svm_fifo_t * fifo,
70  vlib_buffer_t * b0, u32 bi0, u8 n_bufs_per_seg,
71  u32 left_from_seg, u32 * left_to_snd0,
72  u16 * n_bufs, u32 * tx_offset, u16 deq_per_buf,
73  u8 peek_data)
74 {
75  vlib_buffer_t *chain_b0, *prev_b0;
76  u32 chain_bi0, to_deq;
77  u16 len_to_deq0, n_bytes_read;
78  u8 *data0, j;
79 
80  b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
82 
83  chain_bi0 = bi0;
84  chain_b0 = b0;
85  to_deq = left_from_seg;
86  for (j = 1; j < n_bufs_per_seg; j++)
87  {
88  prev_b0 = chain_b0;
89  len_to_deq0 = clib_min (to_deq, deq_per_buf);
90 
91  *n_bufs -= 1;
92  chain_bi0 = smm->tx_buffers[thread_index][*n_bufs];
93  _vec_len (smm->tx_buffers[thread_index]) = *n_bufs;
94 
95  chain_b0 = vlib_get_buffer (vm, chain_bi0);
96  chain_b0->current_data = 0;
97  data0 = vlib_buffer_get_current (chain_b0);
98  if (peek_data)
99  {
100  n_bytes_read = svm_fifo_peek (fifo, *tx_offset, len_to_deq0, data0);
101  *tx_offset += n_bytes_read;
102  }
103  else
104  {
105  n_bytes_read = svm_fifo_dequeue_nowait (fifo, len_to_deq0, data0);
106  }
107  ASSERT (n_bytes_read == len_to_deq0);
108  chain_b0->current_length = n_bytes_read;
110 
111  /* update previous buffer */
112  prev_b0->next_buffer = chain_bi0;
113  prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
114 
115  /* update current buffer */
116  chain_b0->next_buffer = 0;
117 
118  to_deq -= n_bytes_read;
119  if (to_deq == 0)
120  break;
121  }
122  ASSERT (to_deq == 0
123  && b0->total_length_not_including_first_buffer == left_from_seg);
124  *left_to_snd0 -= left_from_seg;
125 }
126 
127 always_inline int
130  session_fifo_event_t * e0,
131  stream_session_t * s0, u32 thread_index,
132  int *n_tx_packets, u8 peek_data)
133 {
134  u32 n_trace = vlib_get_trace_count (vm, node);
135  u32 left_to_snd0, max_len_to_snd0, len_to_deq0, snd_space0;
136  u32 n_bufs_per_evt, n_frames_per_evt, n_bufs_per_frame;
138  transport_proto_vft_t *transport_vft;
140  u32 next_index, next0, *to_next, n_left_to_next, bi0;
141  vlib_buffer_t *b0;
142  u32 tx_offset = 0, max_dequeue0, n_bytes_per_seg, left_for_seg;
143  u16 snd_mss0, n_bufs_per_seg, n_bufs;
144  u8 *data0;
145  int i, n_bytes_read;
146  u32 n_bytes_per_buf, deq_per_buf, deq_per_first_buf;
147  u32 bufs_alloc, bufs_now;
148 
149  next_index = next0 = smm->session_type_to_next[s0->session_type];
150 
151  tp = session_get_transport_proto (s0);
152  transport_vft = transport_protocol_get_vft (tp);
153  tc0 = transport_vft->get_connection (s0->connection_index, thread_index);
154 
155  /* Make sure we have space to send and there's something to dequeue */
156  snd_mss0 = transport_vft->send_mss (tc0);
157  snd_space0 = transport_vft->send_space (tc0);
158 
159  /* Can't make any progress */
160  if (snd_space0 == 0 || snd_mss0 == 0)
161  {
162  vec_add1 (smm->pending_event_vector[thread_index], *e0);
163  return 0;
164  }
165 
166  /* Allow enqueuing of a new event */
167  svm_fifo_unset_event (s0->server_tx_fifo);
168 
169  /* Check how much we can pull. */
170  max_dequeue0 = svm_fifo_max_dequeue (s0->server_tx_fifo);
171 
172  if (peek_data)
173  {
174  /* Offset in rx fifo from where to peek data */
175  tx_offset = transport_vft->tx_fifo_offset (tc0);
176  if (PREDICT_FALSE (tx_offset >= max_dequeue0))
177  max_dequeue0 = 0;
178  else
179  max_dequeue0 -= tx_offset;
180  }
181 
182  /* Nothing to read return */
183  if (max_dequeue0 == 0)
184  return 0;
185 
186  /* Ensure we're not writing more than transport window allows */
187  if (max_dequeue0 < snd_space0)
188  {
189  /* Constrained by tx queue. Try to send only fully formed segments */
190  max_len_to_snd0 = (max_dequeue0 > snd_mss0) ?
191  max_dequeue0 - max_dequeue0 % snd_mss0 : max_dequeue0;
192  /* TODO Nagle ? */
193  }
194  else
195  {
196  /* Expectation is that snd_space0 is already a multiple of snd_mss */
197  max_len_to_snd0 = snd_space0;
198  }
199 
200  n_bytes_per_buf = vlib_buffer_free_list_buffer_size
202  ASSERT (n_bytes_per_buf > MAX_HDRS_LEN);
203  n_bytes_per_seg = MAX_HDRS_LEN + snd_mss0;
204  n_bufs_per_seg = ceil ((double) n_bytes_per_seg / n_bytes_per_buf);
205  n_bufs_per_evt = ceil ((double) max_len_to_snd0 / n_bytes_per_seg);
206  n_frames_per_evt = ceil ((double) n_bufs_per_evt / VLIB_FRAME_SIZE);
207  n_bufs_per_frame = n_bufs_per_seg * VLIB_FRAME_SIZE;
208 
209  deq_per_buf = clib_min (snd_mss0, n_bytes_per_buf);
210  deq_per_first_buf = clib_min (snd_mss0, n_bytes_per_buf - MAX_HDRS_LEN);
211 
212  n_bufs = vec_len (smm->tx_buffers[thread_index]);
213  left_to_snd0 = max_len_to_snd0;
214  for (i = 0; i < n_frames_per_evt; i++)
215  {
216  /* Make sure we have at least one full frame of buffers ready */
217  if (PREDICT_FALSE (n_bufs < n_bufs_per_frame))
218  {
219  vec_validate (smm->tx_buffers[thread_index],
220  n_bufs + n_bufs_per_frame - 1);
221  bufs_alloc = 0;
222  do
223  {
224  bufs_now =
225  vlib_buffer_alloc (vm,
226  &smm->tx_buffers[thread_index][n_bufs +
227  bufs_alloc],
228  n_bufs_per_frame - bufs_alloc);
229  bufs_alloc += bufs_now;
230  }
231  while (bufs_now > 0 && ((bufs_alloc + n_bufs < n_bufs_per_frame)));
232 
233  n_bufs += bufs_alloc;
234  _vec_len (smm->tx_buffers[thread_index]) = n_bufs;
235 
236  if (PREDICT_FALSE (n_bufs < n_bufs_per_frame))
237  {
238  vec_add1 (smm->pending_event_vector[thread_index], *e0);
239  return -1;
240  }
241  ASSERT (n_bufs >= n_bufs_per_frame);
242  }
243 
244  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
245  while (left_to_snd0 && n_left_to_next)
246  {
247  /*
248  * Handle first buffer in chain separately
249  */
250 
251  len_to_deq0 = clib_min (left_to_snd0, deq_per_first_buf);
252  if (left_to_snd0 > len_to_deq0 && n_left_to_next > 1)
253  {
254  vlib_buffer_t *pb;
255  u32 pbi = smm->tx_buffers[thread_index][n_bufs - 2];
256  pb = vlib_get_buffer (vm, pbi);
257  vlib_prefetch_buffer_header (pb, LOAD);
258  }
259 
260  /* Get free buffer */
261  ASSERT (n_bufs >= 1);
262  bi0 = smm->tx_buffers[thread_index][--n_bufs];
263  _vec_len (smm->tx_buffers[thread_index]) = n_bufs;
264 
265  /* usual speculation, or the enqueue_x1 macro will barf */
266  to_next[0] = bi0;
267  to_next += 1;
268  n_left_to_next -= 1;
269 
270  b0 = vlib_get_buffer (vm, bi0);
271  b0->error = 0;
272  b0->flags = VNET_BUFFER_F_LOCALLY_ORIGINATED;
273  b0->current_data = 0;
275 
277  if (peek_data)
278  {
279  n_bytes_read = svm_fifo_peek (s0->server_tx_fifo, tx_offset,
280  len_to_deq0, data0);
281  if (n_bytes_read <= 0)
282  goto dequeue_fail;
283  /* Keep track of progress locally, transport is also supposed to
284  * increment it independently when pushing the header */
285  tx_offset += n_bytes_read;
286  }
287  else
288  {
289  n_bytes_read = svm_fifo_dequeue_nowait (s0->server_tx_fifo,
290  len_to_deq0, data0);
291  if (n_bytes_read <= 0)
292  goto dequeue_fail;
293  }
294 
295  b0->current_length = n_bytes_read;
296 
297  left_to_snd0 -= n_bytes_read;
298  *n_tx_packets = *n_tx_packets + 1;
299 
300  /*
301  * Fill in the remaining buffers in the chain, if any
302  */
303  if (PREDICT_FALSE (n_bufs_per_seg > 1 && left_to_snd0))
304  {
305  left_for_seg = clib_min (snd_mss0 - n_bytes_read, left_to_snd0);
306  session_tx_fifo_chain_tail (smm, vm, thread_index,
307  s0->server_tx_fifo, b0, bi0,
308  n_bufs_per_seg, left_for_seg,
309  &left_to_snd0, &n_bufs, &tx_offset,
310  deq_per_buf, peek_data);
311  }
312 
313  /* Ask transport to push header after current_length and
314  * total_length_not_including_first_buffer are updated */
315  transport_vft->push_header (tc0, b0);
316 
317  /* *INDENT-OFF* */
318  SESSION_EVT_DBG(SESSION_EVT_DEQ, s0, ({
319  ed->data[0] = e0->event_type;
320  ed->data[1] = max_dequeue0;
321  ed->data[2] = len_to_deq0;
322  ed->data[3] = left_to_snd0;
323  }));
324  /* *INDENT-ON* */
325 
327 
328  if (PREDICT_FALSE (n_trace > 0))
329  {
331  vlib_trace_buffer (vm, node, next_index, b0,
332  1 /* follow_chain */ );
333  vlib_set_trace_count (vm, node, --n_trace);
334  t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
335  t0->session_index = s0->session_index;
336  t0->server_thread_index = s0->thread_index;
337  }
338 
339  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
340  to_next, n_left_to_next,
341  bi0, next0);
342  }
343  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
344  }
345 
346  /* If we couldn't dequeue all bytes mark as partially read */
347  if (max_len_to_snd0 < max_dequeue0)
348  {
349  /* If we don't already have new event */
350  if (svm_fifo_set_event (s0->server_tx_fifo))
351  {
352  vec_add1 (smm->pending_event_vector[thread_index], *e0);
353  }
354  }
355  return 0;
356 
357 dequeue_fail:
358  /*
359  * Can't read from fifo. If we don't already have an event, save as partially
360  * read, return buff to free list and return
361  */
362  clib_warning ("dequeue fail");
363 
364  if (svm_fifo_set_event (s0->server_tx_fifo))
365  {
366  vec_add1 (smm->pending_event_vector[thread_index], *e0);
367  }
368  vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1);
369  _vec_len (smm->tx_buffers[thread_index]) += 1;
370 
371  return 0;
372 }
373 
374 int
377  session_fifo_event_t * e0,
378  stream_session_t * s0, u32 thread_index,
379  int *n_tx_pkts)
380 {
381  return session_tx_fifo_read_and_snd_i (vm, node, smm, e0, s0, thread_index,
382  n_tx_pkts, 1);
383 }
384 
385 int
388  session_fifo_event_t * e0,
389  stream_session_t * s0, u32 thread_index,
390  int *n_tx_pkts)
391 {
392  return session_tx_fifo_read_and_snd_i (vm, node, smm, e0, s0, thread_index,
393  n_tx_pkts, 0);
394 }
395 
396 int
398  vlib_node_runtime_t * node,
400  session_fifo_event_t * e0,
401  stream_session_t * s0, u32 thread_index,
402  int *n_tx_pkts)
403 {
404  application_t *app;
405  app = application_get (s0->opaque);
406  svm_fifo_unset_event (s0->server_tx_fifo);
407  return app->cb_fns.builtin_app_tx_callback (s0);
408 }
409 
411 session_event_get_session (session_fifo_event_t * e, u8 thread_index)
412 {
413  return session_get_if_valid (e->fifo->master_session_index, thread_index);
414 }
415 
416 void
418 {
421  u32 my_thread_index = vm->thread_index;
422  session_fifo_event_t _e, *e = &_e;
423  stream_session_t *s0;
424  int i, index;
425  i8 *headp;
426 
427  svm_queue_t *q;
428  q = smm->vpp_event_queues[my_thread_index];
429 
430  index = q->head;
431 
432  for (i = 0; i < q->cursize; i++)
433  {
434  headp = (i8 *) (&q->data[0] + q->elsize * index);
435  clib_memcpy (e, headp, q->elsize);
436 
437  switch (e->event_type)
438  {
439  case FIFO_EVENT_APP_TX:
440  s0 = session_event_get_session (e, my_thread_index);
441  fformat (stdout, "[%04d] TX session %d\n", i, s0->session_index);
442  break;
443 
445  s0 = session_get_from_handle (e->session_handle);
446  fformat (stdout, "[%04d] disconnect session %d\n", i,
447  s0->session_index);
448  break;
449 
451  s0 = session_event_get_session (e, my_thread_index);
452  fformat (stdout, "[%04d] builtin_rx %d\n", i, s0->session_index);
453  break;
454 
455  case FIFO_EVENT_RPC:
456  fformat (stdout, "[%04d] RPC call %llx with %llx\n",
457  i, (u64) (e->rpc_args.fp), (u64) (e->rpc_args.arg));
458  break;
459 
460  default:
461  fformat (stdout, "[%04d] unhandled event type %d\n",
462  i, e->event_type);
463  break;
464  }
465 
466  index++;
467 
468  if (index == q->maxsize)
469  index = 0;
470  }
471 }
472 
473 static u8
474 session_node_cmp_event (session_fifo_event_t * e, svm_fifo_t * f)
475 {
476  stream_session_t *s;
477  switch (e->event_type)
478  {
479  case FIFO_EVENT_APP_RX:
480  case FIFO_EVENT_APP_TX:
482  if (e->fifo == f)
483  return 1;
484  break;
486  break;
487  case FIFO_EVENT_RPC:
488  s = session_get_from_handle (e->session_handle);
489  if (!s)
490  {
491  clib_warning ("session has event but doesn't exist!");
492  break;
493  }
494  if (s->server_rx_fifo == f || s->server_tx_fifo == f)
495  return 1;
496  break;
497  default:
498  break;
499  }
500  return 0;
501 }
502 
503 u8
504 session_node_lookup_fifo_event (svm_fifo_t * f, session_fifo_event_t * e)
505 {
507  svm_queue_t *q;
508  session_fifo_event_t *pending_event_vector, *evt;
509  int i, index, found = 0;
510  i8 *headp;
511  u8 thread_index;
512 
513  ASSERT (e);
514  thread_index = f->master_thread_index;
515  /*
516  * Search evt queue
517  */
518  q = smm->vpp_event_queues[thread_index];
519  index = q->head;
520  for (i = 0; i < q->cursize; i++)
521  {
522  headp = (i8 *) (&q->data[0] + q->elsize * index);
523  clib_memcpy (e, headp, q->elsize);
524  found = session_node_cmp_event (e, f);
525  if (found)
526  return 1;
527  if (++index == q->maxsize)
528  index = 0;
529  }
530  /*
531  * Search pending events vector
532  */
533  pending_event_vector = smm->pending_event_vector[thread_index];
534  vec_foreach (evt, pending_event_vector)
535  {
536  found = session_node_cmp_event (evt, f);
537  if (found)
538  {
539  clib_memcpy (e, evt, sizeof (*evt));
540  break;
541  }
542  }
543  return found;
544 }
545 
546 static uword
548  vlib_frame_t * frame)
549 {
551  session_fifo_event_t *my_pending_event_vector, *pending_disconnects, *e;
552  session_fifo_event_t *my_fifo_events;
553  u32 n_to_dequeue, n_events;
554  svm_queue_t *q;
555  application_t *app;
556  int n_tx_packets = 0;
557  u32 my_thread_index = vm->thread_index;
558  int i, rv;
559  f64 now = vlib_time_now (vm);
560  void (*fp) (void *);
561 
562  SESSION_EVT_DBG (SESSION_EVT_POLL_GAP_TRACK, smm, my_thread_index);
563 
564  /*
565  * Update transport time
566  */
567  transport_update_time (now, my_thread_index);
568 
569  /*
570  * Get vpp queue events
571  */
572  q = smm->vpp_event_queues[my_thread_index];
573  if (PREDICT_FALSE (q == 0))
574  return 0;
575 
576  my_fifo_events = smm->free_event_vector[my_thread_index];
577 
578  /* min number of events we can dequeue without blocking */
579  n_to_dequeue = q->cursize;
580  my_pending_event_vector = smm->pending_event_vector[my_thread_index];
581  pending_disconnects = smm->pending_disconnects[my_thread_index];
582 
583  if (!n_to_dequeue && !vec_len (my_pending_event_vector)
584  && !vec_len (pending_disconnects))
585  return 0;
586 
587  SESSION_EVT_DBG (SESSION_EVT_DEQ_NODE, 0);
588 
589  /*
590  * If we didn't manage to process previous events try going
591  * over them again without dequeuing new ones.
592  */
593  /* XXX: Block senders to sessions that can't keep up */
594  if (0 && vec_len (my_pending_event_vector) >= 100)
595  {
596  clib_warning ("too many fifo events unsolved");
597  goto skip_dequeue;
598  }
599 
600  /* See you in the next life, don't be late */
601  if (pthread_mutex_trylock (&q->mutex))
602  return 0;
603 
604  for (i = 0; i < n_to_dequeue; i++)
605  {
606  vec_add2 (my_fifo_events, e, 1);
607  svm_queue_sub_raw (q, (u8 *) e);
608  }
609 
610  /* The other side of the connection is not polling */
611  if (q->cursize < (q->maxsize / 8))
612  (void) pthread_cond_broadcast (&q->condvar);
613  pthread_mutex_unlock (&q->mutex);
614 
615  vec_append (my_fifo_events, my_pending_event_vector);
616  vec_append (my_fifo_events, smm->pending_disconnects[my_thread_index]);
617 
618  _vec_len (my_pending_event_vector) = 0;
619  smm->pending_event_vector[my_thread_index] = my_pending_event_vector;
620  _vec_len (smm->pending_disconnects[my_thread_index]) = 0;
621 
622 skip_dequeue:
623  n_events = vec_len (my_fifo_events);
624  for (i = 0; i < n_events; i++)
625  {
626  stream_session_t *s0; /* $$$ prefetch 1 ahead maybe */
627  session_fifo_event_t *e0;
628 
629  e0 = &my_fifo_events[i];
630 
631  switch (e0->event_type)
632  {
633  case FIFO_EVENT_APP_TX:
634  s0 = session_event_get_session (e0, my_thread_index);
635 
636  if (PREDICT_FALSE (!s0))
637  {
638  clib_warning ("It's dead, Jim!");
639  continue;
640  }
641  /* Can retransmit for closed sessions but can't do anything if
642  * session is not ready or closed */
643  if (PREDICT_FALSE (s0->session_state < SESSION_STATE_READY))
644  {
645  vec_add1 (smm->pending_event_vector[my_thread_index], *e0);
646  continue;
647  }
648  /* Spray packets in per session type frames, since they go to
649  * different nodes */
650  rv = (smm->session_tx_fns[s0->session_type]) (vm, node, smm, e0, s0,
651  my_thread_index,
652  &n_tx_packets);
653  /* Out of buffers */
654  if (PREDICT_FALSE (rv < 0))
655  {
657  SESSION_QUEUE_ERROR_NO_BUFFER, 1);
658  continue;
659  }
660  break;
662  /* Make sure disconnects run after the pending list is drained */
663  if (!e0->postponed)
664  {
665  e0->postponed = 1;
666  vec_add1 (smm->pending_disconnects[my_thread_index], *e0);
667  continue;
668  }
669  s0 = session_get_from_handle (e0->session_handle);
671  break;
673  s0 = session_event_get_session (e0, my_thread_index);
674  if (PREDICT_FALSE (!s0))
675  continue;
676  svm_fifo_unset_event (s0->server_rx_fifo);
677  app = application_get (s0->app_index);
678  app->cb_fns.builtin_app_rx_callback (s0);
679  break;
680  case FIFO_EVENT_RPC:
681  fp = e0->rpc_args.fp;
682  (*fp) (e0->rpc_args.arg);
683  break;
684 
685  default:
686  clib_warning ("unhandled event type %d", e0->event_type);
687  }
688  }
689 
690  _vec_len (my_fifo_events) = 0;
691  smm->free_event_vector[my_thread_index] = my_fifo_events;
692 
694  SESSION_QUEUE_ERROR_TX, n_tx_packets);
695 
696  SESSION_EVT_DBG (SESSION_EVT_DEQ_NODE, 1);
697 
698  return n_tx_packets;
699 }
700 
701 /* *INDENT-OFF* */
703 {
704  .function = session_queue_node_fn,
705  .name = "session-queue",
706  .format_trace = format_session_queue_trace,
707  .type = VLIB_NODE_TYPE_INPUT,
709  .error_strings = session_queue_error_strings,
710  .state = VLIB_NODE_STATE_DISABLED,
711 };
712 /* *INDENT-ON* */
713 
714 static clib_error_t *
716 {
717  if (vec_len (vlib_mains) < 2)
718  return 0;
719 
720  /*
721  * Shut off (especially) worker-thread session nodes.
722  * Otherwise, vpp can crash as the main thread unmaps the
723  * API segment.
724  */
726  session_node_enable_disable (0 /* is_enable */ );
728  return 0;
729 }
730 
732 
733 /*
734  * fd.io coding-style-patch-verification: ON
735  *
736  * Local Variables:
737  * eval: (c-set-style "gnu")
738  * End:
739  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:434
vlib_main_t vlib_global_main
Definition: main.c:1642
#define clib_min(x, y)
Definition: clib.h:340
#define CLIB_UNUSED(x)
Definition: clib.h:79
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:143
struct _transport_connection transport_connection_t
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:227
u32 thread_index
Definition: main.h:176
struct _transport_proto_vft transport_proto_vft_t
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:520
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:559
int i
static clib_error_t * session_queue_exit(vlib_main_t *vm)
Definition: session_node.c:715
static uword session_queue_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: session_node.c:547
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
static stream_session_t * session_get_from_handle(session_handle_t handle)
Definition: session.h:287
void session_node_enable_disable(u8 is_en)
Definition: session.c:1300
vlib_main_t ** vlib_mains
Definition: buffer.c:303
void transport_update_time(f64 time_now, u8 thread_index)
Definition: transport.c:352
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:212
struct _svm_fifo svm_fifo_t
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:104
static int session_tx_fifo_read_and_snd_i(vlib_main_t *vm, vlib_node_runtime_t *node, session_manager_main_t *smm, session_fifo_event_t *e0, stream_session_t *s0, u32 thread_index, int *n_tx_packets, u8 peek_data)
Definition: session_node.c:128
void dump_thread_0_event_queue(void)
Definition: session_node.c:417
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:104
#define always_inline
Definition: clib.h:92
static u32 svm_fifo_max_dequeue(svm_fifo_t *f)
Definition: svm_fifo.h:100
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:191
char i8
Definition: types.h:45
unsigned long u64
Definition: types.h:89
struct _stream_session_t stream_session_t
#define VLIB_FRAME_SIZE
Definition: node.h:328
static char * session_queue_error_strings[]
Definition: session_node.c:61
transport_proto_vft_t * transport_protocol_get_vft(transport_proto_t transport_proto)
Get transport virtual function table.
Definition: transport.c:181
static transport_proto_t session_get_transport_proto(stream_session_t *s)
Definition: session.h:324
static session_manager_main_t * vnet_get_session_manager_main()
Definition: session.h:217
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:108
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
Definition: buffer.h:321
#define MAX_HDRS_LEN
Definition: session.h:31
session_queue_error_t
Definition: session_node.c:53
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:209
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:446
int session_tx_fifo_peek_and_snd(vlib_main_t *vm, vlib_node_runtime_t *node, session_manager_main_t *smm, session_fifo_event_t *e0, stream_session_t *s0, u32 thread_index, int *n_tx_pkts)
Definition: session_node.c:375
#define PREDICT_FALSE(x)
Definition: clib.h:105
static void svm_fifo_unset_event(svm_fifo_t *f)
Unsets fifo event flag.
Definition: svm_fifo.h:133
static stream_session_t * session_get_if_valid(u64 si, u32 thread_index)
Definition: session.h:248
u32 node_index
Node index.
Definition: node.h:437
struct _session_manager_main session_manager_main_t
Definition: session.h:107
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:364
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:130
word fformat(FILE *f, char *fmt,...)
Definition: format.c:453
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1166
static u8 svm_fifo_set_event(svm_fifo_t *f)
Sets fifo event flag.
Definition: svm_fifo.h:123
The fine-grained event logger allows lightweight, thread-safe event logging at minimum cost...
#define SESSION_EVT_DBG(_evt, _args...)
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
vlib_main_t * vm
Definition: buffer.c:294
#define VLIB_MAIN_LOOP_EXIT_FUNCTION(x)
Definition: init.h:116
#define clib_warning(format, args...)
Definition: error.h:59
#define clib_memcpy(a, b, c)
Definition: string.h:75
struct _application application_t
#define ARRAY_LEN(x)
Definition: clib.h:59
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:454
#define foreach_session_queue_error
Definition: session_node.c:48
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
static u8 session_node_cmp_event(session_fifo_event_t *e, svm_fifo_t *f)
Definition: session_node.c:474
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:126
#define vec_append(v1, v2)
Append v2 after v1.
Definition: vec.h:817
int svm_queue_sub_raw(svm_queue_t *q, u8 *elem)
Definition: queue.c:404
static stream_session_t * session_event_get_session(session_fifo_event_t *e, u8 thread_index)
Definition: session_node.c:411
static u8 * format_session_queue_trace(u8 *s, va_list *args)
Definition: session_node.c:35
u64 uword
Definition: types.h:112
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:159
struct _vlib_node_registration vlib_node_registration_t
int session_tx_fifo_dequeue_and_snd(vlib_main_t *vm, vlib_node_runtime_t *node, session_manager_main_t *smm, session_fifo_event_t *e0, stream_session_t *s0, u32 thread_index, int *n_tx_pkts)
Definition: session_node.c:386
unsigned short u16
Definition: types.h:57
enum _transport_proto transport_proto_t
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
unsigned char u8
Definition: types.h:56
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:553
application_t * application_get(u32 index)
Definition: application.c:386
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:533
vlib_node_registration_t session_queue_node
(constructor) VLIB_REGISTER_NODE (session_queue_node)
Definition: session_node.c:25
struct _svm_queue svm_queue_t
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1497
int session_tx_fifo_dequeue_internal(vlib_main_t *vm, vlib_node_runtime_t *node, session_manager_main_t *smm, session_fifo_event_t *e0, stream_session_t *s0, u32 thread_index, int *n_tx_pkts)
Definition: session_node.c:397
#define vec_foreach(var, vec)
Vector iterator.
static void session_tx_fifo_chain_tail(session_manager_main_t *smm, vlib_main_t *vm, u8 thread_index, svm_fifo_t *fifo, vlib_buffer_t *b0, u32 bi0, u8 n_bufs_per_seg, u32 left_from_seg, u32 *left_to_snd0, u16 *n_bufs, u32 *tx_offset, u16 deq_per_buf, u8 peek_data)
Definition: session_node.c:68
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:159
u8 session_node_lookup_fifo_event(svm_fifo_t *f, session_fifo_event_t *e)
Definition: session_node.c:504
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:347
int svm_fifo_peek(svm_fifo_t *f, u32 relative_offset, u32 max_bytes, u8 *copy_here)
Definition: svm_fifo.c:758
int svm_fifo_dequeue_nowait(svm_fifo_t *f, u32 max_bytes, u8 *copy_here)
Definition: svm_fifo.c:692
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
void stream_session_disconnect_transport(stream_session_t *s)
Notify transport the session can be disconnected.
Definition: session.c:1035