FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
svm_fifo.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <svm/svm_fifo.h>
17 #include <vppinfra/cpu.h>
18 
19 static inline u8
21 {
22  return (ooo_segment_distance_from_tail (f, a)
24 }
25 
26 static inline u8
28 {
29  return (ooo_segment_distance_from_tail (f, a)
31 }
32 
33 static inline u8
35 {
36  return (ooo_segment_distance_from_tail (f, a)
38 }
39 
40 static inline u32
41 position_diff (svm_fifo_t * f, u32 posa, u32 posb)
42 {
43  return ooo_segment_distance_from_tail (f, posa)
45 }
46 
47 static inline u32
49 {
50  return (s->start + s->length) % f->nitems;
51 }
52 
53 #ifndef CLIB_MARCH_VARIANT
54 
55 u8 *
56 format_ooo_segment (u8 * s, va_list * args)
57 {
58  svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
59  ooo_segment_t *seg = va_arg (*args, ooo_segment_t *);
60  u32 normalized_start = (seg->start + f->nitems - f->tail) % f->nitems;
61  s = format (s, "[%u, %u], len %u, next %d, prev %d", normalized_start,
62  (normalized_start + seg->length) % f->nitems, seg->length,
63  seg->next, seg->prev);
64  return s;
65 }
66 
67 u8 *
69 {
70 #if SVM_FIFO_TRACE
71  svm_fifo_trace_elem_t *seg = 0;
72  int i = 0;
73 
74  if (f->trace)
75  {
76  vec_foreach (seg, f->trace)
77  {
78  s = format (s, "{%u, %u, %u}, ", seg->offset, seg->len, seg->action);
79  i++;
80  if (i % 5 == 0)
81  s = format (s, "\n");
82  }
83  s = format (s, "\n");
84  }
85  return s;
86 #else
87  return 0;
88 #endif
89 }
90 
91 u8 *
92 svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
93 {
94  int i, trace_len;
95  u8 *data = 0;
97  u32 offset;
98  svm_fifo_t *dummy_fifo;
99 
100  if (!f)
101  return s;
102 
103 #if SVM_FIFO_TRACE
104  trace = f->trace;
105  trace_len = vec_len (trace);
106 #else
107  trace = 0;
108  trace_len = 0;
109 #endif
110 
111  dummy_fifo = svm_fifo_create (f->nitems);
112  clib_memset (f->data, 0xFF, f->nitems);
113 
114  vec_validate (data, f->nitems);
115  for (i = 0; i < vec_len (data); i++)
116  data[i] = i;
117 
118  for (i = 0; i < trace_len; i++)
119  {
120  offset = trace[i].offset;
121  if (trace[i].action == 1)
122  {
123  if (verbose)
124  s = format (s, "adding [%u, %u]:", trace[i].offset,
125  (trace[i].offset +
126  trace[i].len) % dummy_fifo->nitems);
127  svm_fifo_enqueue_with_offset (dummy_fifo, trace[i].offset,
128  trace[i].len, &data[offset]);
129  }
130  else if (trace[i].action == 2)
131  {
132  if (verbose)
133  s = format (s, "adding [%u, %u]:", 0, trace[i].len);
134  svm_fifo_enqueue_nowait (dummy_fifo, trace[i].len, &data[offset]);
135  }
136  else if (!no_read)
137  {
138  if (verbose)
139  s = format (s, "read: %u", trace[i].len);
140  svm_fifo_dequeue_drop (dummy_fifo, trace[i].len);
141  }
142  if (verbose)
143  s = format (s, "%U", format_svm_fifo, dummy_fifo, 1);
144  }
145 
146  s = format (s, "result: %U", format_svm_fifo, dummy_fifo, 1);
147 
148  return s;
149 }
150 
151 u8 *
152 format_ooo_list (u8 * s, va_list * args)
153 {
154  svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
155  u32 indent = va_arg (*args, u32);
156  u32 ooo_segment_index = f->ooos_list_head;
157  ooo_segment_t *seg;
158 
159  while (ooo_segment_index != OOO_SEGMENT_INVALID_INDEX)
160  {
161  seg = pool_elt_at_index (f->ooo_segments, ooo_segment_index);
162  s = format (s, "%U%U\n", format_white_space, indent, format_ooo_segment,
163  f, seg);
164  ooo_segment_index = seg->next;
165  }
166 
167  return s;
168 }
169 
170 u8 *
171 format_svm_fifo (u8 * s, va_list * args)
172 {
173  svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
174  int verbose = va_arg (*args, int);
175  u32 indent;
176 
177  if (!s)
178  return s;
179 
180  indent = format_get_indent (s);
181  s = format (s, "cursize %u nitems %u has_event %d\n",
182  f->cursize, f->nitems, f->has_event);
183  s = format (s, "%Uhead %d tail %d segment manager %u\n", format_white_space,
184  indent, f->head, f->tail, f->segment_manager);
185 
186  if (verbose > 1)
187  s = format (s, "%Uvpp session %d thread %d app session %d thread %d\n",
188  format_white_space, indent, f->master_session_index,
189  f->master_thread_index, f->client_session_index,
190  f->client_thread_index);
191 
192  if (verbose)
193  {
194  s = format (s, "%Uooo pool %d active elts newest %u\n",
195  format_white_space, indent, pool_elts (f->ooo_segments),
196  f->ooos_newest);
197  if (svm_fifo_has_ooo_data (f))
198  s = format (s, " %U", format_ooo_list, f, indent, verbose);
199  }
200  return s;
201 }
202 
203 /** create an svm fifo, in the current heap. Fails vs blow up the process */
204 svm_fifo_t *
205 svm_fifo_create (u32 data_size_in_bytes)
206 {
207  svm_fifo_t *f;
208  u32 rounded_data_size;
209 
210  /* always round fifo data size to the next highest power-of-two */
211  rounded_data_size = (1 << (max_log2 (data_size_in_bytes)));
212  f = clib_mem_alloc_aligned_or_null (sizeof (*f) + rounded_data_size,
214  if (f == 0)
215  return 0;
216 
217  clib_memset (f, 0, sizeof (*f));
218  f->nitems = data_size_in_bytes;
219  f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
220  f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
221  f->segment_index = SVM_FIFO_INVALID_INDEX;
222  f->refcnt = 1;
223  return (f);
224 }
225 
226 void
228 {
229  ASSERT (f->refcnt > 0);
230 
231  if (--f->refcnt == 0)
232  {
233  pool_free (f->ooo_segments);
234  clib_mem_free (f);
235  }
236 }
237 #endif
238 
240 ooo_segment_new (svm_fifo_t * f, u32 start, u32 length)
241 {
242  ooo_segment_t *s;
243 
244  pool_get (f->ooo_segments, s);
245 
246  s->start = start;
247  s->length = length;
248 
250 
251  return s;
252 }
253 
254 always_inline void
256 {
257  ooo_segment_t *cur, *prev = 0, *next = 0;
258  cur = pool_elt_at_index (f->ooo_segments, index);
259 
260  if (cur->next != OOO_SEGMENT_INVALID_INDEX)
261  {
262  next = pool_elt_at_index (f->ooo_segments, cur->next);
263  next->prev = cur->prev;
264  }
265 
266  if (cur->prev != OOO_SEGMENT_INVALID_INDEX)
267  {
268  prev = pool_elt_at_index (f->ooo_segments, cur->prev);
269  prev->next = cur->next;
270  }
271  else
272  {
273  f->ooos_list_head = cur->next;
274  }
275 
276  pool_put (f->ooo_segments, cur);
277 }
278 
279 /**
280  * Add segment to fifo's out-of-order segment list. Takes care of merging
281  * adjacent segments and removing overlapping ones.
282  */
283 static void
285 {
286  ooo_segment_t *s, *new_s, *prev, *next, *it;
287  u32 new_index, s_end_pos, s_index;
288  u32 normalized_position, normalized_end_position;
289 
290  ASSERT (offset + length <= ooo_segment_distance_from_tail (f, f->head));
291  normalized_position = (f->tail + offset) % f->nitems;
292  normalized_end_position = (f->tail + offset + length) % f->nitems;
293 
294  f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
295 
296  if (f->ooos_list_head == OOO_SEGMENT_INVALID_INDEX)
297  {
298  s = ooo_segment_new (f, normalized_position, length);
299  f->ooos_list_head = s - f->ooo_segments;
300  f->ooos_newest = f->ooos_list_head;
301  return;
302  }
303 
304  /* Find first segment that starts after new segment */
305  s = pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
306  while (s->next != OOO_SEGMENT_INVALID_INDEX
307  && position_lt (f, s->start, normalized_position))
308  s = pool_elt_at_index (f->ooo_segments, s->next);
309 
310  /* If we have a previous and we overlap it, use it as starting point */
311  prev = ooo_segment_get_prev (f, s);
312  if (prev
313  && position_leq (f, normalized_position, ooo_segment_end_pos (f, prev)))
314  {
315  s = prev;
316  s_end_pos = ooo_segment_end_pos (f, s);
317 
318  /* Since we have previous, normalized start position cannot be smaller
319  * than prev->start. Check tail */
320  ASSERT (position_lt (f, s->start, normalized_position));
321  goto check_tail;
322  }
323 
324  s_index = s - f->ooo_segments;
325  s_end_pos = ooo_segment_end_pos (f, s);
326 
327  /* No overlap, add before current segment */
328  if (position_lt (f, normalized_end_position, s->start))
329  {
330  new_s = ooo_segment_new (f, normalized_position, length);
331  new_index = new_s - f->ooo_segments;
332 
333  /* Pool might've moved, get segment again */
334  s = pool_elt_at_index (f->ooo_segments, s_index);
336  {
337  new_s->prev = s->prev;
338  prev = pool_elt_at_index (f->ooo_segments, new_s->prev);
339  prev->next = new_index;
340  }
341  else
342  {
343  /* New head */
344  f->ooos_list_head = new_index;
345  }
346 
347  new_s->next = s_index;
348  s->prev = new_index;
349  f->ooos_newest = new_index;
350  return;
351  }
352  /* No overlap, add after current segment */
353  else if (position_gt (f, normalized_position, s_end_pos))
354  {
355  new_s = ooo_segment_new (f, normalized_position, length);
356  new_index = new_s - f->ooo_segments;
357 
358  /* Pool might've moved, get segment again */
359  s = pool_elt_at_index (f->ooo_segments, s_index);
360 
361  /* Needs to be last */
363 
364  new_s->prev = s_index;
365  s->next = new_index;
366  f->ooos_newest = new_index;
367 
368  return;
369  }
370 
371  /*
372  * Merge needed
373  */
374 
375  /* Merge at head */
376  if (position_lt (f, normalized_position, s->start))
377  {
378  s->start = normalized_position;
379  s->length = position_diff (f, s_end_pos, s->start);
380  f->ooos_newest = s - f->ooo_segments;
381  }
382 
383 check_tail:
384 
385  /* Overlapping tail */
386  if (position_gt (f, normalized_end_position, s_end_pos))
387  {
388  s->length = position_diff (f, normalized_end_position, s->start);
389 
390  /* Remove the completely overlapped segments in the tail */
391  it = ooo_segment_next (f, s);
392  while (it && position_leq (f, ooo_segment_end_pos (f, it),
393  normalized_end_position))
394  {
395  next = ooo_segment_next (f, it);
396  ooo_segment_del (f, it - f->ooo_segments);
397  it = next;
398  }
399 
400  /* If partial overlap with last, merge */
401  if (it && position_leq (f, it->start, normalized_end_position))
402  {
403  s->length = position_diff (f, ooo_segment_end_pos (f, it),
404  s->start);
405  ooo_segment_del (f, it - f->ooo_segments);
406  }
407  f->ooos_newest = s - f->ooo_segments;
408  }
409 }
410 
411 /**
412  * Removes segments that can now be enqueued because the fifo's tail has
413  * advanced. Returns the number of bytes added to tail.
414  */
415 static int
416 ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued)
417 {
418  ooo_segment_t *s;
419  u32 index, bytes = 0;
420  i32 diff;
421 
422  s = pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
423  diff = ooo_segment_distance_to_tail (f, s->start);
424 
425  ASSERT (diff != n_bytes_enqueued);
426 
427  if (diff > n_bytes_enqueued)
428  return 0;
429 
430  /* If last tail update overlaps one/multiple ooo segments, remove them */
431  while (0 <= diff && diff < n_bytes_enqueued)
432  {
433  index = s - f->ooo_segments;
434 
435  /* Segment end is beyond the tail. Advance tail and remove segment */
436  if (s->length > diff)
437  {
438  bytes = s->length - diff;
439  f->tail += bytes;
440  f->tail %= f->nitems;
441  ooo_segment_del (f, index);
442  break;
443  }
444 
445  /* If we have next go on */
447  {
448  s = pool_elt_at_index (f->ooo_segments, s->next);
449  diff = ooo_segment_distance_to_tail (f, s->start);
450  ooo_segment_del (f, index);
451  }
452  /* End of search */
453  else
454  {
455  ooo_segment_del (f, index);
456  break;
457  }
458  }
459 
460  ASSERT (bytes <= f->nitems);
461  return bytes;
462 }
463 
465  const u8 * copy_from_here)
466 {
467  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
468  u32 cursize, nitems;
469 
470  /* read cursize, which can only increase while we're working */
471  cursize = svm_fifo_max_dequeue (f);
472  f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
473 
474  if (PREDICT_FALSE (cursize == f->nitems))
475  return SVM_FIFO_FULL;
476 
477  nitems = f->nitems;
478 
479  /* Number of bytes we're going to copy */
480  total_copy_bytes = (nitems - cursize) < max_bytes ?
481  (nitems - cursize) : max_bytes;
482 
483  if (PREDICT_TRUE (copy_from_here != 0))
484  {
485  /* Number of bytes in first copy segment */
486  first_copy_bytes = ((nitems - f->tail) < total_copy_bytes)
487  ? (nitems - f->tail) : total_copy_bytes;
488 
489  clib_memcpy_fast (&f->data[f->tail], copy_from_here, first_copy_bytes);
490  f->tail += first_copy_bytes;
491  f->tail = (f->tail == nitems) ? 0 : f->tail;
492 
493  /* Number of bytes in second copy segment, if any */
494  second_copy_bytes = total_copy_bytes - first_copy_bytes;
495  if (second_copy_bytes)
496  {
497  clib_memcpy_fast (&f->data[f->tail],
498  copy_from_here + first_copy_bytes,
499  second_copy_bytes);
500  f->tail += second_copy_bytes;
501  f->tail = (f->tail == nitems) ? 0 : f->tail;
502  }
503  }
504  else
505  {
506  ASSERT (0);
507 
508  /* Account for a zero-copy enqueue done elsewhere */
509  ASSERT (max_bytes <= (nitems - cursize));
510  f->tail += max_bytes;
511  f->tail = f->tail % nitems;
512  total_copy_bytes = max_bytes;
513  }
514 
515  svm_fifo_trace_add (f, f->head, total_copy_bytes, 2);
516 
517  /* Any out-of-order segments to collect? */
518  if (PREDICT_FALSE (f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX))
519  total_copy_bytes += ooo_segment_try_collect (f, total_copy_bytes);
520 
521  /* Atomically increase the queue length */
522  ASSERT (cursize + total_copy_bytes <= nitems);
523  clib_atomic_fetch_add_rel (&f->cursize, total_copy_bytes);
524 
525  return (total_copy_bytes);
526 }
527 
528 #ifndef CLIB_MARCH_VARIANT
529 int
531  const u8 * copy_from_here)
532 {
533  return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_nowait) (f, max_bytes,
534  copy_from_here);
535 }
536 #endif
537 
538 /**
539  * Enqueue a future segment.
540  *
541  * Two choices: either copies the entire segment, or copies nothing
542  * Returns 0 of the entire segment was copied
543  * Returns -1 if none of the segment was copied due to lack of space
544  */
546  u32 offset, u32 required_bytes, u8 * copy_from_here)
547 {
548  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
549  u32 cursize, nitems, normalized_offset;
550 
551  f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
552 
553  /* read cursize, which can only increase while we're working */
554  cursize = svm_fifo_max_dequeue (f);
555  nitems = f->nitems;
556 
557  ASSERT (required_bytes < nitems);
558 
559  normalized_offset = (f->tail + offset) % nitems;
560 
561  /* Will this request fit? */
562  if ((required_bytes + offset) > (nitems - cursize))
563  return -1;
564 
565  svm_fifo_trace_add (f, offset, required_bytes, 1);
566 
567  ooo_segment_add (f, offset, required_bytes);
568 
569  /* Number of bytes we're going to copy */
570  total_copy_bytes = required_bytes;
571 
572  /* Number of bytes in first copy segment */
573  first_copy_bytes = ((nitems - normalized_offset) < total_copy_bytes)
574  ? (nitems - normalized_offset) : total_copy_bytes;
575 
576  clib_memcpy_fast (&f->data[normalized_offset], copy_from_here,
577  first_copy_bytes);
578 
579  /* Number of bytes in second copy segment, if any */
580  second_copy_bytes = total_copy_bytes - first_copy_bytes;
581  if (second_copy_bytes)
582  {
583  normalized_offset += first_copy_bytes;
584  normalized_offset %= nitems;
585 
586  ASSERT (normalized_offset == 0);
587 
588  clib_memcpy_fast (&f->data[normalized_offset],
589  copy_from_here + first_copy_bytes, second_copy_bytes);
590  }
591 
592  return (0);
593 }
594 
595 #ifndef CLIB_MARCH_VARIANT
596 
597 int
599  u8 * copy_from_here)
600 {
602  required_bytes,
603  copy_from_here);
604 }
605 
606 void
608 {
609  u32 first_chunk;
610  first_chunk = f->nitems - f->head;
611  ASSERT (len <= f->nitems);
612  if (len <= first_chunk)
613  clib_memcpy_fast (&f->data[f->head], data, len);
614  else
615  {
616  clib_memcpy_fast (&f->data[f->head], data, first_chunk);
617  clib_memcpy_fast (&f->data[0], data + first_chunk, len - first_chunk);
618  }
619 }
620 #endif
621 
623  u8 * copy_here)
624 {
625  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
626  u32 cursize, nitems;
627 
628  /* read cursize, which can only increase while we're working */
629  cursize = svm_fifo_max_dequeue (f);
630  if (PREDICT_FALSE (cursize == 0))
631  return -2; /* nothing in the fifo */
632 
633  nitems = f->nitems;
634 
635  /* Number of bytes we're going to copy */
636  total_copy_bytes = (cursize < max_bytes) ? cursize : max_bytes;
637 
638  if (PREDICT_TRUE (copy_here != 0))
639  {
640  /* Number of bytes in first copy segment */
641  first_copy_bytes = ((nitems - f->head) < total_copy_bytes)
642  ? (nitems - f->head) : total_copy_bytes;
643  clib_memcpy_fast (copy_here, &f->data[f->head], first_copy_bytes);
644  f->head += first_copy_bytes;
645  f->head = (f->head == nitems) ? 0 : f->head;
646 
647  /* Number of bytes in second copy segment, if any */
648  second_copy_bytes = total_copy_bytes - first_copy_bytes;
649  if (second_copy_bytes)
650  {
651  clib_memcpy_fast (copy_here + first_copy_bytes,
652  &f->data[f->head], second_copy_bytes);
653  f->head += second_copy_bytes;
654  f->head = (f->head == nitems) ? 0 : f->head;
655  }
656  }
657  else
658  {
659  ASSERT (0);
660  /* Account for a zero-copy dequeue done elsewhere */
661  ASSERT (max_bytes <= cursize);
662  f->head += max_bytes;
663  f->head = f->head % nitems;
664  cursize -= max_bytes;
665  total_copy_bytes = max_bytes;
666  }
667 
668  ASSERT (f->head <= nitems);
669  ASSERT (cursize >= total_copy_bytes);
670  clib_atomic_fetch_sub_rel (&f->cursize, total_copy_bytes);
671 
672  return (total_copy_bytes);
673 }
674 
675 #ifndef CLIB_MARCH_VARIANT
676 
677 int
678 svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
679 {
680  return CLIB_MARCH_FN_SELECT (svm_fifo_dequeue_nowait) (f, max_bytes,
681  copy_here);
682 }
683 #endif
684 
685 CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
686  u32 max_bytes, u8 * copy_here)
687 {
688  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
689  u32 cursize, nitems, real_head;
690 
691  /* read cursize, which can only increase while we're working */
692  cursize = svm_fifo_max_dequeue (f);
693  if (PREDICT_FALSE (cursize < relative_offset))
694  return -2; /* nothing in the fifo */
695 
696  nitems = f->nitems;
697  real_head = f->head + relative_offset;
698  real_head = real_head >= nitems ? real_head - nitems : real_head;
699 
700  /* Number of bytes we're going to copy */
701  total_copy_bytes = (cursize - relative_offset < max_bytes) ?
702  cursize - relative_offset : max_bytes;
703 
704  if (PREDICT_TRUE (copy_here != 0))
705  {
706  /* Number of bytes in first copy segment */
707  first_copy_bytes =
708  ((nitems - real_head) < total_copy_bytes) ?
709  (nitems - real_head) : total_copy_bytes;
710  clib_memcpy_fast (copy_here, &f->data[real_head], first_copy_bytes);
711 
712  /* Number of bytes in second copy segment, if any */
713  second_copy_bytes = total_copy_bytes - first_copy_bytes;
714  if (second_copy_bytes)
715  {
716  clib_memcpy_fast (copy_here + first_copy_bytes, &f->data[0],
717  second_copy_bytes);
718  }
719  }
720  return total_copy_bytes;
721 }
722 
723 #ifndef CLIB_MARCH_VARIANT
724 
725 int
726 svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes,
727  u8 * copy_here)
728 {
729  return CLIB_MARCH_FN_SELECT (svm_fifo_peek) (f, relative_offset, max_bytes,
730  copy_here);
731 }
732 
733 int
735 {
736  u32 total_drop_bytes, first_drop_bytes, second_drop_bytes;
737  u32 cursize, nitems;
738 
739  /* read cursize, which can only increase while we're working */
740  cursize = svm_fifo_max_dequeue (f);
741  if (PREDICT_FALSE (cursize == 0))
742  return -2; /* nothing in the fifo */
743 
744  nitems = f->nitems;
745 
746  /* Number of bytes we're going to drop */
747  total_drop_bytes = (cursize < max_bytes) ? cursize : max_bytes;
748 
749  svm_fifo_trace_add (f, f->tail, total_drop_bytes, 3);
750 
751  /* Number of bytes in first copy segment */
752  first_drop_bytes =
753  ((nitems - f->head) < total_drop_bytes) ?
754  (nitems - f->head) : total_drop_bytes;
755  f->head += first_drop_bytes;
756  f->head = (f->head == nitems) ? 0 : f->head;
757 
758  /* Number of bytes in second drop segment, if any */
759  second_drop_bytes = total_drop_bytes - first_drop_bytes;
760  if (second_drop_bytes)
761  {
762  f->head += second_drop_bytes;
763  f->head = (f->head == nitems) ? 0 : f->head;
764  }
765 
766  ASSERT (f->head <= nitems);
767  ASSERT (cursize >= total_drop_bytes);
768  clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes);
769 
770  return total_drop_bytes;
771 }
772 
773 void
775 {
776  f->head = f->tail;
777  clib_atomic_fetch_sub_rel (&f->cursize, f->cursize);
778 }
779 
780 int
782 {
783  u32 cursize, nitems;
784 
785  /* read cursize, which can only increase while we're working */
786  cursize = svm_fifo_max_dequeue (f);
787  if (PREDICT_FALSE (cursize == 0))
788  return -2;
789 
790  nitems = f->nitems;
791 
792  fs[0].len = ((nitems - f->head) < cursize) ? (nitems - f->head) : cursize;
793  fs[0].data = f->data + f->head;
794 
795  if (fs[0].len < cursize)
796  {
797  fs[1].len = cursize - fs[0].len;
798  fs[1].data = f->data;
799  }
800  else
801  {
802  fs[1].len = 0;
803  fs[1].data = 0;
804  }
805  return cursize;
806 }
807 
808 void
810 {
811  u32 total_drop_bytes;
812 
813  ASSERT (fs[0].data == f->data + f->head);
814  if (fs[1].len)
815  {
816  f->head = fs[1].len;
817  total_drop_bytes = fs[0].len + fs[1].len;
818  }
819  else
820  {
821  f->head = (f->head + fs[0].len) % f->nitems;
822  total_drop_bytes = fs[0].len;
823  }
824  clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes);
825 }
826 
827 u32
829 {
830  return pool_elts (f->ooo_segments);
831 }
832 
835 {
836  return pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
837 }
838 
839 /**
840  * Set fifo pointers to requested offset
841  */
842 void
844 {
845  f->head = f->tail = pointer % f->nitems;
846 }
847 
848 #endif
849 /*
850  * fd.io coding-style-patch-verification: ON
851  *
852  * Local Variables:
853  * eval: (c-set-style "gnu")
854  * End:
855  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:862
void svm_fifo_init_pointers(svm_fifo_t *f, u32 pointer)
Set fifo pointers to requested offset.
Definition: svm_fifo.c:843
static u32 position_diff(svm_fifo_t *f, u32 posa, u32 posb)
Definition: svm_fifo.c:41
a
Definition: bitmap.h:538
int svm_fifo_segments(svm_fifo_t *f, svm_fifo_segment_t *fs)
Definition: svm_fifo.c:781
static u8 svm_fifo_has_ooo_data(svm_fifo_t *f)
Definition: svm_fifo.h:154
#define PREDICT_TRUE(x)
Definition: clib.h:112
void svm_fifo_free(svm_fifo_t *f)
Definition: svm_fifo.c:227
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
u32 prev
Previous linked-list element pool index.
Definition: svm_fifo.h:30
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
#define CLIB_MARCH_FN_SELECT(fn)
Definition: cpu.h:310
void svm_fifo_overwrite_head(svm_fifo_t *f, u8 *data, u32 len)
Definition: svm_fifo.c:607
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
static int ooo_segment_try_collect(svm_fifo_t *f, u32 n_bytes_enqueued)
Removes segments that can now be enqueued because the fifo&#39;s tail has advanced.
Definition: svm_fifo.c:416
ooo_segment_t * svm_fifo_first_ooo_segment(svm_fifo_t *f)
Definition: svm_fifo.c:834
void svm_fifo_dequeue_drop_all(svm_fifo_t *f)
Definition: svm_fifo.c:774
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define SVM_FIFO_INVALID_INDEX
Definition: svm_fifo.h:42
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:236
unsigned char u8
Definition: types.h:56
static u32 ooo_segment_distance_from_tail(svm_fifo_t *f, u32 pos)
Definition: svm_fifo.h:318
struct _svm_fifo svm_fifo_t
static void ooo_segment_add(svm_fifo_t *f, u32 offset, u32 length)
Add segment to fifo&#39;s out-of-order segment list.
Definition: svm_fifo.c:284
#define always_inline
Definition: clib.h:98
static u32 svm_fifo_max_dequeue(svm_fifo_t *f)
Definition: svm_fifo.h:124
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
u8 * format_ooo_list(u8 *s, va_list *args)
Definition: svm_fifo.c:152
int svm_fifo_enqueue_nowait(svm_fifo_t *f, u32 max_bytes, const u8 *copy_from_here)
Definition: svm_fifo.c:530
unsigned int u32
Definition: types.h:88
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:511
static u8 position_gt(svm_fifo_t *f, u32 a, u32 b)
Definition: svm_fifo.c:34
u32 svm_fifo_number_ooo_segments(svm_fifo_t *f)
Definition: svm_fifo.c:828
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:286
#define PREDICT_FALSE(x)
Definition: clib.h:111
static ooo_segment_t * ooo_segment_next(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:360
#define svm_fifo_trace_add(_f, _s, _l, _t)
Definition: svm_fifo.h:117
#define clib_atomic_fetch_add_rel(a, b)
Definition: atomics.h:47
u8 * svm_fifo_replay(u8 *s, svm_fifo_t *f, u8 no_read, u8 verbose)
Definition: svm_fifo.c:92
u8 len
Definition: ip_types.api:49
#define pool_free(p)
Free a pool.
Definition: pool.h:404
static u8 position_leq(svm_fifo_t *f, u32 a, u32 b)
Definition: svm_fifo.c:27
static ooo_segment_t * ooo_segment_get_prev(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.h:352
static void ooo_segment_del(svm_fifo_t *f, u32 index)
Definition: svm_fifo.c:255
CLIB_MARCH_FN(svm_fifo_enqueue_nowait, int, svm_fifo_t *f, u32 max_bytes, const u8 *copy_from_here)
Definition: svm_fifo.c:464
#define OOO_SEGMENT_INVALID_INDEX
Definition: svm_fifo.h:40
u8 * format_ooo_segment(u8 *s, va_list *args)
Definition: svm_fifo.c:56
static void * clib_mem_alloc_aligned_or_null(uword size, uword align)
Definition: mem.h:156
signed int i32
Definition: types.h:77
u8 * format_svm_fifo(u8 *s, va_list *args)
Definition: svm_fifo.c:171
#define ASSERT(truth)
static u8 position_lt(svm_fifo_t *f, u32 a, u32 b)
Definition: svm_fifo.c:20
static void clib_mem_free(void *p)
Definition: mem.h:205
int svm_fifo_enqueue_with_offset(svm_fifo_t *f, u32 offset, u32 required_bytes, u8 *copy_from_here)
Definition: svm_fifo.c:598
Out-of-order segment.
Definition: svm_fifo.h:27
u32 length
Length of segment.
Definition: svm_fifo.h:33
#define clib_atomic_fetch_sub_rel(a, b)
Definition: atomics.h:48
u8 * svm_fifo_dump_trace(u8 *s, svm_fifo_t *f)
Definition: svm_fifo.c:68
u32 next
Next linked-list element pool index.
Definition: svm_fifo.h:29
template key/value backing page structure
Definition: bihash_doc.h:44
void svm_fifo_segments_free(svm_fifo_t *f, svm_fifo_segment_t *fs)
Definition: svm_fifo.c:809
int svm_fifo_dequeue_drop(svm_fifo_t *f, u32 max_bytes)
Definition: svm_fifo.c:734
static u32 ooo_segment_end_pos(svm_fifo_t *f, ooo_segment_t *s)
Definition: svm_fifo.c:48
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static ooo_segment_t * ooo_segment_new(svm_fifo_t *f, u32 start, u32 length)
Definition: svm_fifo.c:240
static uword max_log2(uword x)
Definition: clib.h:191
struct clib_bihash_value offset
template key/value backing page structure
#define vec_foreach(var, vec)
Vector iterator.
#define SVM_FIFO_INVALID_SESSION_INDEX
Definition: svm_fifo.h:41
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
int svm_fifo_peek(svm_fifo_t *f, u32 relative_offset, u32 max_bytes, u8 *copy_here)
Definition: svm_fifo.c:726
int svm_fifo_dequeue_nowait(svm_fifo_t *f, u32 max_bytes, u8 *copy_here)
Definition: svm_fifo.c:678
u32 start
Start of segment, normalized.
Definition: svm_fifo.h:32
static u32 ooo_segment_distance_to_tail(svm_fifo_t *f, u32 pos)
Definition: svm_fifo.h:328
svm_fifo_t * svm_fifo_create(u32 data_size_in_bytes)
create an svm fifo, in the current heap.
Definition: svm_fifo.c:205
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128