FD.io VPP  v18.04-17-g3a0d853
Vector Packet Processing
serialize.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  Copyright (c) 2005 Eliot Dresselhaus
17 
18  Permission is hereby granted, free of charge, to any person obtaining
19  a copy of this software and associated documentation files (the
20  "Software"), to deal in the Software without restriction, including
21  without limitation the rights to use, copy, modify, merge, publish,
22  distribute, sublicense, and/or sell copies of the Software, and to
23  permit persons to whom the Software is furnished to do so, subject to
24  the following conditions:
25 
26  The above copyright notice and this permission notice shall be
27  included in all copies or substantial portions of the Software.
28 
29  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
33  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
34  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
35  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
36 */
37 
38 /* Turn data structures into byte streams for saving or transport. */
39 
40 #include <vppinfra/heap.h>
41 #include <vppinfra/pool.h>
42 #include <vppinfra/serialize.h>
43 
44 void
45 serialize_64 (serialize_main_t * m, va_list * va)
46 {
47  u64 x = va_arg (*va, u64);
48  u32 lo, hi;
49  lo = x;
50  hi = x >> 32;
51  serialize_integer (m, lo, sizeof (lo));
52  serialize_integer (m, hi, sizeof (hi));
53 }
54 
55 void
56 serialize_32 (serialize_main_t * m, va_list * va)
57 {
58  u32 x = va_arg (*va, u32);
59  serialize_integer (m, x, sizeof (x));
60 }
61 
62 void
63 serialize_16 (serialize_main_t * m, va_list * va)
64 {
65  u32 x = va_arg (*va, u32);
66  serialize_integer (m, x, sizeof (u16));
67 }
68 
69 void
70 serialize_8 (serialize_main_t * m, va_list * va)
71 {
72  u32 x = va_arg (*va, u32);
73  serialize_integer (m, x, sizeof (u8));
74 }
75 
76 void
77 unserialize_64 (serialize_main_t * m, va_list * va)
78 {
79  u64 *x = va_arg (*va, u64 *);
80  u32 lo, hi;
81  unserialize_integer (m, &lo, sizeof (lo));
82  unserialize_integer (m, &hi, sizeof (hi));
83  *x = ((u64) hi << 32) | (u64) lo;
84 }
85 
86 void
87 unserialize_32 (serialize_main_t * m, va_list * va)
88 {
89  u32 *x = va_arg (*va, u32 *);
90  unserialize_integer (m, x, sizeof (x[0]));
91 }
92 
93 void
94 unserialize_16 (serialize_main_t * m, va_list * va)
95 {
96  u16 *x = va_arg (*va, u16 *);
97  u32 t;
98  unserialize_integer (m, &t, sizeof (x[0]));
99  x[0] = t;
100 }
101 
102 void
103 unserialize_8 (serialize_main_t * m, va_list * va)
104 {
105  u8 *x = va_arg (*va, u8 *);
106  u32 t;
107  unserialize_integer (m, &t, sizeof (x[0]));
108  x[0] = t;
109 }
110 
111 void
112 serialize_f64 (serialize_main_t * m, va_list * va)
113 {
114  f64 x = va_arg (*va, f64);
115  union
116  {
117  f64 f;
118  u64 i;
119  } y;
120  y.f = x;
121  serialize (m, serialize_64, y.i);
122 }
123 
124 void
125 serialize_f32 (serialize_main_t * m, va_list * va)
126 {
127  f32 x = va_arg (*va, f64);
128  union
129  {
130  f32 f;
131  u32 i;
132  } y;
133  y.f = x;
134  serialize_integer (m, y.i, sizeof (y.i));
135 }
136 
137 void
139 {
140  f64 *x = va_arg (*va, f64 *);
141  union
142  {
143  f64 f;
144  u64 i;
145  } y;
146  unserialize (m, unserialize_64, &y.i);
147  *x = y.f;
148 }
149 
150 void
152 {
153  f32 *x = va_arg (*va, f32 *);
154  union
155  {
156  f32 f;
157  u32 i;
158  } y;
159  unserialize_integer (m, &y.i, sizeof (y.i));
160  *x = y.f;
161 }
162 
163 void
165 {
166  u32 len = s ? strlen (s) : 0;
167  void *p;
168 
170  if (len > 0)
171  {
172  p = serialize_get (m, len);
173  clib_memcpy (p, s, len);
174  }
175 }
176 
177 void
179 {
180  char *p, *r = 0;
181  u32 len;
182 
184 
185  /*
186  * Given broken enough data, we could get len = 0xFFFFFFFF.
187  * Add one, it overflows, we call vec_new (char, 0), then
188  * memcpy until we bus error.
189  */
190  if (len > 0 && len != 0xFFFFFFFF)
191  {
192  r = vec_new (char, len + 1);
193  p = unserialize_get (m, len);
194  clib_memcpy (r, p, len);
195 
196  /* Null terminate. */
197  r[len] = 0;
198  }
199  *s = r;
200 }
201 
202 /* vec_serialize/vec_unserialize helper functions for basic vector types. */
203 void
205 {
206  u8 *s = va_arg (*va, u8 *);
207  u32 n = va_arg (*va, u32);
208  u8 *p = serialize_get (m, n * sizeof (u8));
209  clib_memcpy (p, s, n * sizeof (u8));
210 }
211 
212 void
214 {
215  u8 *s = va_arg (*va, u8 *);
216  u32 n = va_arg (*va, u32);
217  u8 *p = unserialize_get (m, n);
218  clib_memcpy (s, p, n);
219 }
220 
221 #define _(n_bits) \
222  void serialize_vec_##n_bits (serialize_main_t * m, va_list * va) \
223  { \
224  u##n_bits * s = va_arg (*va, u##n_bits *); \
225  u32 n = va_arg (*va, u32); \
226  u##n_bits * p = serialize_get (m, n * sizeof (s[0])); \
227  \
228  while (n >= 4) \
229  { \
230  p[0] = clib_host_to_net_u##n_bits (s[0]); \
231  p[1] = clib_host_to_net_u##n_bits (s[1]); \
232  p[2] = clib_host_to_net_u##n_bits (s[2]); \
233  p[3] = clib_host_to_net_u##n_bits (s[3]); \
234  s += 4; \
235  p += 4; \
236  n -= 4; \
237  } \
238  \
239  while (n >= 1) \
240  { \
241  p[0] = clib_host_to_net_u##n_bits (s[0]); \
242  s += 1; \
243  p += 1; \
244  n -= 1; \
245  } \
246  } \
247  \
248  void unserialize_vec_##n_bits (serialize_main_t * m, va_list * va) \
249  { \
250  u##n_bits * s = va_arg (*va, u##n_bits *); \
251  u32 n = va_arg (*va, u32); \
252  u##n_bits * p = unserialize_get (m, n * sizeof (s[0])); \
253  \
254  while (n >= 4) \
255  { \
256  s[0] = clib_net_to_host_mem_u##n_bits (&p[0]); \
257  s[1] = clib_net_to_host_mem_u##n_bits (&p[1]); \
258  s[2] = clib_net_to_host_mem_u##n_bits (&p[2]); \
259  s[3] = clib_net_to_host_mem_u##n_bits (&p[3]); \
260  s += 4; \
261  p += 4; \
262  n -= 4; \
263  } \
264  \
265  while (n >= 1) \
266  { \
267  s[0] = clib_net_to_host_mem_u##n_bits (&p[0]); \
268  s += 1; \
269  p += 1; \
270  n -= 1; \
271  } \
272  }
273 
274 _(16);
275 _(32);
276 _(64);
277 
278 #undef _
279 
280 #define SERIALIZE_VECTOR_CHUNK_SIZE 64
281 
282 void
284 {
285  void *vec = va_arg (*va, void *);
286  u32 elt_bytes = va_arg (*va, u32);
287  serialize_function_t *f = va_arg (*va, serialize_function_t *);
288  u32 l = vec_len (vec);
289  void *p = vec;
290 
291  serialize_integer (m, l, sizeof (l));
292 
293  /* Serialize vector in chunks for cache locality. */
294  while (l != 0)
295  {
297  serialize (m, f, p, n);
298  l -= n;
299  p += SERIALIZE_VECTOR_CHUNK_SIZE * elt_bytes;
300  }
301 }
302 
303 void *
305  u32 elt_bytes,
306  u32 header_bytes,
307  u32 align, u32 max_length, serialize_function_t * f)
308 {
309  void *v, *p;
310  u32 l;
311 
312  unserialize_integer (m, &l, sizeof (l));
313  if (l > max_length)
314  serialize_error (&m->header,
315  clib_error_create ("bad vector length %d", l));
316  p = v = _vec_resize (0, l, (uword) l * elt_bytes, header_bytes,
317  /* align */ align);
318 
319  while (l != 0)
320  {
322  unserialize (m, f, p, n);
323  l -= n;
324  p += SERIALIZE_VECTOR_CHUNK_SIZE * elt_bytes;
325  }
326  return v;
327 }
328 
329 void
331 {
332  void **vec = va_arg (*va, void **);
333  u32 elt_bytes = va_arg (*va, u32);
334  serialize_function_t *f = va_arg (*va, serialize_function_t *);
335  u32 align = va_arg (*va, u32);
336 
337  *vec = unserialize_vector_ha (m, elt_bytes,
338  /* header_bytes */ 0,
339  /* align */ align,
340  /* max_length */ ~0,
341  f);
342 }
343 
344 void
346 {
347  void **vec = va_arg (*va, void **);
348  u32 elt_bytes = va_arg (*va, u32);
349  serialize_function_t *f = va_arg (*va, serialize_function_t *);
350 
351  *vec = unserialize_vector_ha (m, elt_bytes,
352  /* header_bytes */ 0,
353  /* align */ 0,
354  /* max_length */ ~0,
355  f);
356 }
357 
358 void
360 {
361  u32 l, i, n_u32s;
362 
363  l = vec_len (b);
364  n_u32s = l * sizeof (b[0]) / sizeof (u32);
365  serialize_integer (m, n_u32s, sizeof (n_u32s));
366 
367  /* Send 32 bit words, low-order word first on 64 bit. */
368  for (i = 0; i < l; i++)
369  {
370  serialize_integer (m, b[i], sizeof (u32));
371  if (BITS (uword) == 64)
372  serialize_integer (m, (u64) b[i] >> (u64) 32, sizeof (u32));
373  }
374 }
375 
376 uword *
378 {
379  uword *b = 0;
380  u32 i, n_u32s;
381 
382  unserialize_integer (m, &n_u32s, sizeof (n_u32s));
383  if (n_u32s == 0)
384  return b;
385 
386  i = (n_u32s * sizeof (u32) + sizeof (b[0]) - 1) / sizeof (b[0]);
387  vec_resize (b, i);
388  for (i = 0; i < n_u32s; i++)
389  {
390  u32 data;
391  unserialize_integer (m, &data, sizeof (u32));
392 
393  /* Low-word is first on 64 bit. */
394  if (BITS (uword) == 64)
395  {
396  if ((i % 2) == 0)
397  b[i / 2] |= (u64) data << (u64) 0;
398  else
399  b[i / 2] |= (u64) data << (u64) 32;
400  }
401  else
402  {
403  b[i] = data;
404  }
405  }
406 
407  return b;
408 }
409 
410 void
411 serialize_pool (serialize_main_t * m, va_list * va)
412 {
413  void *pool = va_arg (*va, void *);
414  u32 elt_bytes = va_arg (*va, u32);
415  serialize_function_t *f = va_arg (*va, serialize_function_t *);
416  u32 l, lo, hi;
417  pool_header_t *p;
418 
419  l = vec_len (pool);
420  serialize_integer (m, l, sizeof (u32));
421  if (l == 0)
422  return;
423  p = pool_header (pool);
424 
425  /* No need to send free bitmap. Need to send index vector
426  to guarantee that unserialized pool will be identical. */
428 
429  pool_foreach_region (lo, hi, pool,
430  serialize (m, f, pool + lo * elt_bytes, hi - lo));
431 }
432 
433 static void *
435  u32 elt_bytes, u32 align, serialize_function_t * f)
436 {
437  void *v;
438  u32 i, l, lo, hi;
439  pool_header_t *p;
440 
441  unserialize_integer (m, &l, sizeof (l));
442  if (l == 0)
443  {
444  return 0;
445  }
446 
447  v = _vec_resize (0, l, (uword) l * elt_bytes, sizeof (p[0]), align);
448  p = pool_header (v);
449 
451 
452  /* Construct free bitmap. */
453  p->free_bitmap = 0;
454  for (i = 0; i < vec_len (p->free_indices); i++)
455  p->free_bitmap = clib_bitmap_ori (p->free_bitmap, p->free_indices[i]);
456 
457  pool_foreach_region (lo, hi, v,
458  unserialize (m, f, v + lo * elt_bytes, hi - lo));
459 
460  return v;
461 }
462 
463 void
465 {
466  void **result = va_arg (*va, void **);
467  u32 elt_bytes = va_arg (*va, u32);
468  serialize_function_t *f = va_arg (*va, serialize_function_t *);
469  *result = unserialize_pool_helper (m, elt_bytes, /* align */ 0, f);
470 }
471 
472 void
474 {
475  void **result = va_arg (*va, void **);
476  u32 elt_bytes = va_arg (*va, u32);
477  u32 align = va_arg (*va, u32);
478  serialize_function_t *f = va_arg (*va, serialize_function_t *);
479  *result = unserialize_pool_helper (m, elt_bytes, align, f);
480 }
481 
482 static void
484 {
485  heap_elt_t *e = va_arg (*va, heap_elt_t *);
486  u32 i, n = va_arg (*va, u32);
487  for (i = 0; i < n; i++)
488  {
489  serialize_integer (m, e[i].offset, sizeof (e[i].offset));
490  serialize_integer (m, e[i].next, sizeof (e[i].next));
491  serialize_integer (m, e[i].prev, sizeof (e[i].prev));
492  }
493 }
494 
495 static void
497 {
498  heap_elt_t *e = va_arg (*va, heap_elt_t *);
499  u32 i, n = va_arg (*va, u32);
500  for (i = 0; i < n; i++)
501  {
502  unserialize_integer (m, &e[i].offset, sizeof (e[i].offset));
503  unserialize_integer (m, &e[i].next, sizeof (e[i].next));
504  unserialize_integer (m, &e[i].prev, sizeof (e[i].prev));
505  }
506 }
507 
508 void
509 serialize_heap (serialize_main_t * m, va_list * va)
510 {
511  void *heap = va_arg (*va, void *);
512  serialize_function_t *f = va_arg (*va, serialize_function_t *);
513  u32 i, l;
514  heap_header_t *h;
515 
516  l = vec_len (heap);
517  serialize_integer (m, l, sizeof (u32));
518  if (l == 0)
519  return;
520 
521  h = heap_header (heap);
522 
523 #define foreach_serialize_heap_header_integer \
524  _ (head) _ (tail) _ (used_count) _ (max_len) _ (flags) _ (elt_bytes)
525 
526 #define _(f) serialize_integer (m, h->f, sizeof (h->f));
528 #undef _
529 
530  serialize_integer (m, vec_len (h->free_lists), sizeof (u32));
531  for (i = 0; i < vec_len (h->free_lists); i++)
533 
537 
538  /* Serialize data in heap. */
539  {
540  heap_elt_t *e, *end;
541  e = h->elts + h->head;
542  end = h->elts + h->tail;
543  while (1)
544  {
545  if (!heap_is_free (e))
546  {
547  void *v = heap + heap_offset (e) * h->elt_bytes;
548  u32 n = heap_elt_size (heap, e);
549  serialize (m, f, v, n);
550  }
551  if (e == end)
552  break;
553  e = heap_next (e);
554  }
555  }
556 }
557 
558 void
560 {
561  void **result = va_arg (*va, void **);
562  serialize_function_t *f = va_arg (*va, serialize_function_t *);
563  u32 i, vl, fl;
565  void *heap;
566 
567  unserialize_integer (m, &vl, sizeof (u32));
568  if (vl == 0)
569  {
570  *result = 0;
571  return;
572  }
573 
574  memset (&h, 0, sizeof (h));
575 #define _(f) unserialize_integer (m, &h.f, sizeof (h.f));
577 #undef _
578 
579  unserialize_integer (m, &fl, sizeof (u32));
580  vec_resize (h.free_lists, fl);
581 
582  for (i = 0; i < vec_len (h.free_lists); i++)
584 
588 
589  /* Re-construct used elt bitmap. */
590  if (CLIB_DEBUG > 0)
591  {
592  heap_elt_t *e;
593  vec_foreach (e, h.elts)
594  {
595  if (!heap_is_free (e))
596  h.used_elt_bitmap = clib_bitmap_ori (h.used_elt_bitmap, e - h.elts);
597  }
598  }
599 
600  heap = *result = _heap_new (vl, h.elt_bytes);
601  heap_header (heap)[0] = h;
602 
603  /* Unserialize data in heap. */
604  {
605  heap_elt_t *e, *end;
606  e = h.elts + h.head;
607  end = h.elts + h.tail;
608  while (1)
609  {
610  if (!heap_is_free (e))
611  {
612  void *v = heap + heap_offset (e) * h.elt_bytes;
613  u32 n = heap_elt_size (heap, e);
614  unserialize (m, f, v, n);
615  }
616  if (e == end)
617  break;
618  e = heap_next (e);
619  }
620  }
621 }
622 
623 void
624 serialize_magic (serialize_main_t * m, void *magic, u32 magic_bytes)
625 {
626  void *p;
627  serialize_integer (m, magic_bytes, sizeof (magic_bytes));
628  p = serialize_get (m, magic_bytes);
629  clib_memcpy (p, magic, magic_bytes);
630 }
631 
632 void
633 unserialize_check_magic (serialize_main_t * m, void *magic, u32 magic_bytes)
634 {
635  u32 l;
636  void *d;
637 
638  unserialize_integer (m, &l, sizeof (l));
639  if (l != magic_bytes)
640  {
641  bad:
642  serialize_error_return (m, "bad magic number");
643  }
644  d = serialize_get (m, magic_bytes);
645  if (memcmp (magic, d, magic_bytes))
646  goto bad;
647 }
648 
649 clib_error_t *
650 va_serialize (serialize_main_t * sm, va_list * va)
651 {
653  serialize_function_t *f = va_arg (*va, serialize_function_t *);
654  clib_error_t *error = 0;
655 
656  m->recursion_level += 1;
657  if (m->recursion_level == 1)
658  {
659  uword r = clib_setjmp (&m->error_longjmp, 0);
660  error = uword_to_pointer (r, clib_error_t *);
661  }
662 
663  if (!error)
664  f (sm, va);
665 
666  m->recursion_level -= 1;
667  return error;
668 }
669 
670 clib_error_t *
672 {
673  clib_error_t *error;
674  va_list va;
675 
676  va_start (va, m);
677  error = va_serialize (m, &va);
678  va_end (va);
679  return error;
680 }
681 
682 clib_error_t *
684 {
685  clib_error_t *error;
686  va_list va;
687 
688  va_start (va, m);
689  error = va_serialize (m, &va);
690  va_end (va);
691  return error;
692 }
693 
694 static void *
696  serialize_stream_t * s,
697  uword n_bytes_to_write, uword flags)
698 {
699  uword cur_bi, n_left_b, n_left_o;
700 
702  cur_bi = s->current_buffer_index;
703  n_left_b = s->n_buffer_bytes - cur_bi;
704  n_left_o = vec_len (s->overflow_buffer);
705 
706  /* Prepend overflow buffer if present. */
707  do
708  {
709  if (n_left_o > 0 && n_left_b > 0)
710  {
711  uword n = clib_min (n_left_b, n_left_o);
712  clib_memcpy (s->buffer + cur_bi, s->overflow_buffer, n);
713  cur_bi += n;
714  n_left_b -= n;
715  n_left_o -= n;
716  if (n_left_o == 0)
717  _vec_len (s->overflow_buffer) = 0;
718  else
719  vec_delete (s->overflow_buffer, n, 0);
720  }
721 
722  /* Call data function when buffer is complete. Data function should
723  dispatch with current buffer and give us a new one to write more
724  data into. */
725  if (n_left_b == 0)
726  {
727  s->current_buffer_index = cur_bi;
728  m->data_function (m, s);
729  cur_bi = s->current_buffer_index;
730  n_left_b = s->n_buffer_bytes - cur_bi;
731  }
732  }
733  while (n_left_o > 0);
734 
735  if (n_left_o > 0 || n_left_b < n_bytes_to_write)
736  {
737  u8 *r;
738  vec_add2 (s->overflow_buffer, r, n_bytes_to_write);
739  return r;
740  }
741  else
742  {
743  s->current_buffer_index = cur_bi + n_bytes_to_write;
744  return s->buffer + cur_bi;
745  }
746 }
747 
748 static void *
750  serialize_stream_t * s,
751  uword n_bytes_to_read, uword flags)
752 {
753  uword cur_bi, cur_oi, n_left_b, n_left_o, n_left_to_read;
754 
756 
757  cur_bi = s->current_buffer_index;
758  cur_oi = s->current_overflow_index;
759 
760  n_left_b = s->n_buffer_bytes - cur_bi;
761  n_left_o = vec_len (s->overflow_buffer) - cur_oi;
762 
763  /* Read from overflow? */
764  if (n_left_o >= n_bytes_to_read)
765  {
766  s->current_overflow_index = cur_oi + n_bytes_to_read;
767  return vec_elt_at_index (s->overflow_buffer, cur_oi);
768  }
769 
770  /* Reset overflow buffer. */
771  if (n_left_o == 0 && s->overflow_buffer)
772  {
773  s->current_overflow_index = 0;
774  _vec_len (s->overflow_buffer) = 0;
775  }
776 
777  n_left_to_read = n_bytes_to_read;
778  while (n_left_to_read > 0)
779  {
780  uword n;
781 
782  /* If we don't have enough data between overflow and normal buffer
783  call read function. */
784  if (n_left_o + n_left_b < n_bytes_to_read)
785  {
786  /* Save any left over buffer in overflow vector. */
787  if (n_left_b > 0)
788  {
789  vec_add (s->overflow_buffer, s->buffer + cur_bi, n_left_b);
790  n_left_o += n_left_b;
791  n_left_to_read -= n_left_b;
792  /* Advance buffer to end --- even if
793  SERIALIZE_FLAG_NO_ADVANCE_CURRENT_BUFFER_INDEX is set. */
794  cur_bi = s->n_buffer_bytes;
795  n_left_b = 0;
796  }
797 
798  if (m->data_function)
799  {
800  m->data_function (m, s);
801  cur_bi = s->current_buffer_index;
802  n_left_b = s->n_buffer_bytes - cur_bi;
803  }
804  }
805 
806  /* For first time through loop return if we have enough data
807  in normal buffer and overflow vector is empty. */
808  if (n_left_o == 0
809  && n_left_to_read == n_bytes_to_read && n_left_b >= n_left_to_read)
810  {
811  s->current_buffer_index = cur_bi + n_bytes_to_read;
812  return s->buffer + cur_bi;
813  }
814 
816  {
817  /* This can happen for a peek at end of file.
818  Pad overflow buffer with 0s. */
819  vec_resize (s->overflow_buffer, n_left_to_read);
820  n_left_o += n_left_to_read;
821  n_left_to_read = 0;
822  }
823  else
824  {
825  /* Copy from buffer to overflow vector. */
826  n = clib_min (n_left_to_read, n_left_b);
827  vec_add (s->overflow_buffer, s->buffer + cur_bi, n);
828  cur_bi += n;
829  n_left_b -= n;
830  n_left_o += n;
831  n_left_to_read -= n;
832  }
833  }
834 
835  s->current_buffer_index = cur_bi;
836  s->current_overflow_index = cur_oi + n_bytes_to_read;
837  return vec_elt_at_index (s->overflow_buffer, cur_oi);
838 }
839 
840 void *
842  serialize_stream_t * s,
843  uword n_bytes, uword flags)
844 {
846  serialize_write_not_inline) (m, s, n_bytes, flags));
847 }
848 
849 static void
852 {
854  return;
855 
856  if (flags & SERIALIZE_FLAG_IS_WRITE)
857  /* "Write" 0 bytes to flush overflow vector. */
858  serialize_write_not_inline (m, s, /* n bytes */ 0, flags);
859 
861 
862  /* Call it one last time to flush buffer and close. */
863  m->data_function (m, s);
864 
866 }
867 
868 void
870 {
873 }
874 
875 void
877 {
879 }
880 
881 void
882 serialize_open_data (serialize_main_t * m, u8 * data, uword n_data_bytes)
883 {
884  memset (m, 0, sizeof (m[0]));
885  m->stream.buffer = data;
886  m->stream.n_buffer_bytes = n_data_bytes;
887 }
888 
889 void
890 unserialize_open_data (serialize_main_t * m, u8 * data, uword n_data_bytes)
891 {
892  serialize_open_data (m, data, n_data_bytes);
893 }
894 
895 static void
897 {
899  {
900  /* Double buffer size. */
901  uword l = vec_len (s->buffer);
902  vec_resize (s->buffer, l > 0 ? l : 64);
903  s->n_buffer_bytes = vec_len (s->buffer);
904  }
905 }
906 
907 void
909 {
910  memset (m, 0, sizeof (m[0]));
912  m->stream.buffer = vector;
914  m->stream.n_buffer_bytes = vec_len (vector);
915 }
916 
917 void *
919 {
920  serialize_stream_t *s = &m->stream;
921  void *result;
922 
923  serialize_close (m); /* frees overflow buffer */
924 
925  if (s->buffer)
926  _vec_len (s->buffer) = s->current_buffer_index;
927  result = s->buffer;
928  memset (m, 0, sizeof (m[0]));
929  return result;
930 }
931 
932 void
934  void *data, uword data_stride, uword n_data)
935 {
936  u8 *d = data;
937  u8 *p;
938  uword n_left = n_data;
939 
940  while (n_left >= 4)
941  {
942  p = serialize_get (m, 4 * sizeof (d[0]));
943  p[0] = d[0 * data_stride];
944  p[1] = d[1 * data_stride];
945  p[2] = d[2 * data_stride];
946  p[3] = d[3 * data_stride];
947  n_left -= 4;
948  d += 4 * data_stride;
949  }
950 
951  if (n_left > 0)
952  {
953  p = serialize_get (m, n_left * sizeof (p[0]));
954  while (n_left > 0)
955  {
956  p[0] = d[0];
957  p += 1;
958  d += 1 * data_stride;
959  n_left -= 1;
960  }
961  }
962 }
963 
964 void
966  void *data, uword data_stride, uword n_data)
967 {
968  void *d = data;
969  u16 *p;
970  uword n_left = n_data;
971 
972  while (n_left >= 4)
973  {
974  p = serialize_get (m, 4 * sizeof (p[0]));
975  clib_mem_unaligned (p + 0, u16) =
976  clib_host_to_net_mem_u16 (d + 0 * data_stride);
977  clib_mem_unaligned (p + 1, u16) =
978  clib_host_to_net_mem_u16 (d + 1 * data_stride);
979  clib_mem_unaligned (p + 2, u16) =
980  clib_host_to_net_mem_u16 (d + 2 * data_stride);
981  clib_mem_unaligned (p + 3, u16) =
982  clib_host_to_net_mem_u16 (d + 3 * data_stride);
983  n_left -= 4;
984  d += 4 * data_stride;
985  }
986 
987  if (n_left > 0)
988  {
989  p = serialize_get (m, n_left * sizeof (p[0]));
990  while (n_left > 0)
991  {
992  clib_mem_unaligned (p + 0, u16) =
993  clib_host_to_net_mem_u16 (d + 0 * data_stride);
994  p += 1;
995  d += 1 * data_stride;
996  n_left -= 1;
997  }
998  }
999 }
1000 
1001 void
1003  void *data, uword data_stride, uword n_data)
1004 {
1005  void *d = data;
1006  u32 *p;
1007  uword n_left = n_data;
1008 
1009  while (n_left >= 4)
1010  {
1011  p = serialize_get (m, 4 * sizeof (p[0]));
1012  clib_mem_unaligned (p + 0, u32) =
1013  clib_host_to_net_mem_u32 (d + 0 * data_stride);
1014  clib_mem_unaligned (p + 1, u32) =
1015  clib_host_to_net_mem_u32 (d + 1 * data_stride);
1016  clib_mem_unaligned (p + 2, u32) =
1017  clib_host_to_net_mem_u32 (d + 2 * data_stride);
1018  clib_mem_unaligned (p + 3, u32) =
1019  clib_host_to_net_mem_u32 (d + 3 * data_stride);
1020  n_left -= 4;
1021  d += 4 * data_stride;
1022  }
1023 
1024  if (n_left > 0)
1025  {
1026  p = serialize_get (m, n_left * sizeof (p[0]));
1027  while (n_left > 0)
1028  {
1029  clib_mem_unaligned (p + 0, u32) =
1030  clib_host_to_net_mem_u32 (d + 0 * data_stride);
1031  p += 1;
1032  d += 1 * data_stride;
1033  n_left -= 1;
1034  }
1035  }
1036 }
1037 
1038 void
1040  void *data, uword data_stride, uword n_data)
1041 {
1042  u8 *d = data;
1043  u8 *p;
1044  uword n_left = n_data;
1045 
1046  while (n_left >= 4)
1047  {
1048  p = unserialize_get (m, 4 * sizeof (d[0]));
1049  d[0 * data_stride] = p[0];
1050  d[1 * data_stride] = p[1];
1051  d[2 * data_stride] = p[2];
1052  d[3 * data_stride] = p[3];
1053  n_left -= 4;
1054  d += 4 * data_stride;
1055  }
1056 
1057  if (n_left > 0)
1058  {
1059  p = unserialize_get (m, n_left * sizeof (p[0]));
1060  while (n_left > 0)
1061  {
1062  d[0] = p[0];
1063  p += 1;
1064  d += 1 * data_stride;
1065  n_left -= 1;
1066  }
1067  }
1068 }
1069 
1070 void
1072  void *data, uword data_stride, uword n_data)
1073 {
1074  void *d = data;
1075  u16 *p;
1076  uword n_left = n_data;
1077 
1078  while (n_left >= 4)
1079  {
1080  p = unserialize_get (m, 4 * sizeof (p[0]));
1081  clib_mem_unaligned (d + 0 * data_stride, u16) =
1082  clib_net_to_host_mem_u16 (p + 0);
1083  clib_mem_unaligned (d + 1 * data_stride, u16) =
1084  clib_net_to_host_mem_u16 (p + 1);
1085  clib_mem_unaligned (d + 2 * data_stride, u16) =
1086  clib_net_to_host_mem_u16 (p + 2);
1087  clib_mem_unaligned (d + 3 * data_stride, u16) =
1088  clib_net_to_host_mem_u16 (p + 3);
1089  n_left -= 4;
1090  d += 4 * data_stride;
1091  }
1092 
1093  if (n_left > 0)
1094  {
1095  p = unserialize_get (m, n_left * sizeof (p[0]));
1096  while (n_left > 0)
1097  {
1098  clib_mem_unaligned (d + 0 * data_stride, u16) =
1099  clib_net_to_host_mem_u16 (p + 0);
1100  p += 1;
1101  d += 1 * data_stride;
1102  n_left -= 1;
1103  }
1104  }
1105 }
1106 
1107 void
1109  void *data, uword data_stride, uword n_data)
1110 {
1111  void *d = data;
1112  u32 *p;
1113  uword n_left = n_data;
1114 
1115  while (n_left >= 4)
1116  {
1117  p = unserialize_get (m, 4 * sizeof (p[0]));
1118  clib_mem_unaligned (d + 0 * data_stride, u32) =
1119  clib_net_to_host_mem_u32 (p + 0);
1120  clib_mem_unaligned (d + 1 * data_stride, u32) =
1121  clib_net_to_host_mem_u32 (p + 1);
1122  clib_mem_unaligned (d + 2 * data_stride, u32) =
1123  clib_net_to_host_mem_u32 (p + 2);
1124  clib_mem_unaligned (d + 3 * data_stride, u32) =
1125  clib_net_to_host_mem_u32 (p + 3);
1126  n_left -= 4;
1127  d += 4 * data_stride;
1128  }
1129 
1130  if (n_left > 0)
1131  {
1132  p = unserialize_get (m, n_left * sizeof (p[0]));
1133  while (n_left > 0)
1134  {
1135  clib_mem_unaligned (d + 0 * data_stride, u32) =
1136  clib_net_to_host_mem_u32 (p + 0);
1137  p += 1;
1138  d += 1 * data_stride;
1139  n_left -= 1;
1140  }
1141  }
1142 }
1143 
1144 #ifdef CLIB_UNIX
1145 
1146 #include <unistd.h>
1147 #include <fcntl.h>
1148 
1149 static void
1151 {
1152  int fd, n;
1153 
1154  fd = s->data_function_opaque;
1155  n = write (fd, s->buffer, s->current_buffer_index);
1156  if (n < 0)
1157  {
1158  if (!unix_error_is_fatal (errno))
1159  n = 0;
1160  else
1161  serialize_error (m, clib_error_return_unix (0, "write"));
1162  }
1163  if (n == s->current_buffer_index)
1164  _vec_len (s->buffer) = 0;
1165  else
1166  vec_delete (s->buffer, n, 0);
1168 }
1169 
1170 static void
1172 {
1173  int fd, n;
1174 
1175  fd = s->data_function_opaque;
1176  n = read (fd, s->buffer, vec_len (s->buffer));
1177  if (n < 0)
1178  {
1179  if (!unix_error_is_fatal (errno))
1180  n = 0;
1181  else
1182  serialize_error (m, clib_error_return_unix (0, "read"));
1183  }
1184  else if (n == 0)
1186  s->current_buffer_index = 0;
1187  s->n_buffer_bytes = n;
1188 }
1189 
1190 static void
1192  uword is_read)
1193 {
1194  memset (m, 0, sizeof (m[0]));
1195  vec_resize (m->stream.buffer, 4096);
1196 
1197  if (!is_read)
1198  {
1200  _vec_len (m->stream.buffer) = 0;
1201  }
1202 
1204  m->stream.data_function_opaque = fd;
1205 }
1206 
1207 void
1209 {
1210  serialize_open_clib_file_descriptor_helper (m, fd, /* is_read */ 0);
1211 }
1212 
1213 void
1215 {
1216  serialize_open_clib_file_descriptor_helper (m, fd, /* is_read */ 1);
1217 }
1218 
1219 static clib_error_t *
1221  uword is_read)
1222 {
1223  int fd, mode;
1224 
1225  mode = is_read ? O_RDONLY : O_RDWR | O_CREAT | O_TRUNC;
1226  fd = open (file, mode, 0666);
1227  if (fd < 0)
1228  return clib_error_return_unix (0, "open `%s'", file);
1229 
1231  return 0;
1232 }
1233 
1234 clib_error_t *
1236 {
1237  return serialize_open_clib_file_helper (m, file, /* is_read */ 0);
1238 }
1239 
1240 clib_error_t *
1242 {
1243  return serialize_open_clib_file_helper (m, file, /* is_read */ 1);
1244 }
1245 
1246 #endif /* CLIB_UNIX */
1247 
1248 /*
1249  * fd.io coding-style-patch-verification: ON
1250  *
1251  * Local Variables:
1252  * eval: (c-set-style "gnu")
1253  * End:
1254  */
vmrglw vmrglh hi
u32 vl(void *p)
Definition: threads.c:30
static void serialize_vector_write(serialize_main_header_t *m, serialize_stream_t *s)
Definition: serialize.c:896
#define clib_min(x, y)
Definition: clib.h:340
void serialize_f32(serialize_main_t *m, va_list *va)
Definition: serialize.c:125
void unserialize_check_magic(serialize_main_t *m, void *magic, u32 magic_bytes)
Definition: serialize.c:633
u32 current_overflow_index
Definition: serialize.h:69
static u64 unserialize_likely_small_unsigned_integer(serialize_main_t *m)
Definition: serialize.h:254
adds_epu subs_epu i16x8 y
Definition: vector_sse42.h:227
#define serialize_error_return(m, args...)
Definition: serialize.h:112
clib_error_t * serialize_open_clib_file(serialize_main_t *m, char *file)
Definition: serialize.c:1235
void serialize_bitmap(serialize_main_t *m, uword *b)
Definition: serialize.c:359
void serialize_multiple_4(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:1002
#define vec_serialize(m, v, f)
Definition: serialize.h:371
Fixed length block allocator.
serialize_main_header_t header
Definition: serialize.h:146
static void serialize_error(serialize_main_header_t *m, clib_error_t *error)
Definition: serialize.h:107
#define vec_unserialize(m, v, f)
Definition: serialize.h:374
u32 * free_indices
Vector of free indices.
Definition: pool.h:58
u32 tail
Definition: heap.h:144
static void serialize_stream_set_end_of_stream(serialize_stream_t *s)
Definition: serialize.h:81
void unserialize_32(serialize_main_t *m, va_list *va)
Definition: serialize.c:87
void unserialize_multiple_2(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:1071
void serialize_f64(serialize_main_t *m, va_list *va)
Definition: serialize.c:112
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:559
int i
u32 head
Definition: heap.h:144
void serialize_open_clib_file_descriptor(serialize_main_t *m, int fd)
Definition: serialize.c:1208
uword clib_setjmp(clib_longjmp_t *save, uword return_value_not_taken)
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:597
float f32
Definition: types.h:143
u32 * free_elts
Definition: heap.h:133
#define SERIALIZE_VECTOR_CHUNK_SIZE
Definition: serialize.c:280
#define foreach_serialize_heap_header_integer
void * unserialize_vector_ha(serialize_main_t *m, u32 elt_bytes, u32 header_bytes, u32 align, u32 max_length, serialize_function_t *f)
Definition: serialize.c:304
void unserialize_open_data(serialize_main_t *m, u8 *data, uword n_data_bytes)
Definition: serialize.c:890
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
Definition: vec.h:306
void unserialize_heap(serialize_main_t *m, va_list *va)
Definition: serialize.c:559
static uword serialize_stream_is_end_of_stream(serialize_stream_t *s)
Definition: serialize.h:87
static void * serialize_get(serialize_main_t *m, uword n_bytes)
Definition: serialize.h:178
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
void unserialize_aligned_vector(serialize_main_t *m, va_list *va)
Definition: serialize.c:330
static uword heap_is_free(heap_elt_t *e)
Definition: heap.h:85
unsigned long u64
Definition: types.h:89
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:237
#define clib_error_create(args...)
Definition: error.h:96
static void * serialize_read_not_inline(serialize_main_header_t *m, serialize_stream_t *s, uword n_bytes_to_read, uword flags)
Definition: serialize.c:749
uword * used_elt_bitmap
Definition: heap.h:141
static heap_elt_t * heap_next(heap_elt_t *e)
Definition: heap.h:97
void serialize_16(serialize_main_t *m, va_list *va)
Definition: serialize.c:63
#define fl(x, y)
#define SERIALIZE_FLAG_IS_READ
Definition: serialize.h:119
void unserialize_vec_8(serialize_main_t *m, va_list *va)
Definition: serialize.c:213
serialize_function_t serialize_vec_32
Definition: serialize.h:364
static pool_header_t * pool_header(void *v)
Get pool header from user pool pointer.
Definition: pool.h:77
void serialize_open_data(serialize_main_t *m, u8 *data, uword n_data_bytes)
Definition: serialize.c:882
static void serialize_likely_small_unsigned_integer(serialize_main_t *m, u64 x)
Definition: serialize.h:218
void serialize_pool(serialize_main_t *m, va_list *va)
Definition: serialize.c:411
lo
void unserialize_multiple_4(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:1108
#define v
Definition: acl.c:495
#define clib_error_return_unix(e, args...)
Definition: error.h:102
heap_elt_t * elts
Definition: heap.h:126
void unserialize_8(serialize_main_t *m, va_list *va)
Definition: serialize.c:103
void serialize_8(serialize_main_t *m, va_list *va)
Definition: serialize.c:70
static void serialize_open_clib_file_descriptor_helper(serialize_main_t *m, int fd, uword is_read)
Definition: serialize.c:1191
static void * unserialize_get(serialize_main_t *m, uword n_bytes)
Definition: serialize.h:171
static void serialize_read_write_close(serialize_main_header_t *m, serialize_stream_t *s, uword flags)
Definition: serialize.c:850
void unserialize_cstring(serialize_main_t *m, char **s)
Definition: serialize.c:178
void * serialize_read_write_not_inline(serialize_main_header_t *m, serialize_stream_t *s, uword n_bytes, uword flags)
Definition: serialize.c:841
#define uword_to_pointer(u, type)
Definition: types.h:136
u32 elt_bytes
Definition: heap.h:149
serialize_stream_t stream
Definition: serialize.h:147
static void clib_file_read(serialize_main_header_t *m, serialize_stream_t *s)
Definition: serialize.c:1171
clib_error_t * serialize(serialize_main_t *m,...)
Definition: serialize.c:671
u32 current_buffer_index
Definition: serialize.h:62
void serialize_open_vector(serialize_main_t *m, u8 *vector)
Definition: serialize.c:908
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:336
clib_error_t * unserialize_open_clib_file(serialize_main_t *m, char *file)
Definition: serialize.c:1241
#define clib_memcpy(a, b, c)
Definition: string.h:75
static uword heap_offset(heap_elt_t *e)
Definition: heap.h:91
serialize_data_function_t * data_function
Definition: serialize.h:97
static void unserialize_integer(serialize_main_t *m, void *x, u32 n_bytes)
Definition: serialize.h:201
static void serialize_integer(serialize_main_t *m, u64 x, u32 n_bytes)
Definition: serialize.h:185
void unserialize_close(serialize_main_t *m)
Definition: serialize.c:876
static void serialize_vec_heap_elt(serialize_main_t *m, va_list *va)
Definition: serialize.c:483
uword data_function_opaque
Definition: serialize.h:74
u32 * small_free_elt_free_index
Definition: heap.h:130
void serialize_multiple_2(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:965
void serialize_magic(serialize_main_t *m, void *magic, u32 magic_bytes)
Definition: serialize.c:624
void unserialize_16(serialize_main_t *m, va_list *va)
Definition: serialize.c:94
void serialize_vec_8(serialize_main_t *m, va_list *va)
Definition: serialize.c:204
#define ASSERT(truth)
void unserialize_aligned_pool(serialize_main_t *m, va_list *va)
Definition: serialize.c:473
unsigned int u32
Definition: types.h:88
void unserialize_multiple_1(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:1039
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:783
static word unix_error_is_fatal(word error)
Definition: error.h:118
void serialize_close(serialize_main_t *m)
Definition: serialize.c:869
#define pool_foreach_region(LO, HI, POOL, BODY)
Optimized iteration through pool.
Definition: pool.h:366
clib_error_t * unserialize(serialize_main_t *m,...)
Definition: serialize.c:683
static uword heap_elt_size(void *v, heap_elt_t *e)
Definition: heap.h:109
static heap_header_t * heap_header(void *v)
Definition: heap.h:161
u64 uword
Definition: types.h:112
void serialize_cstring(serialize_main_t *m, char *s)
Definition: serialize.c:164
static void unserialize_vec_heap_elt(serialize_main_t *m, va_list *va)
Definition: serialize.c:496
uword * free_bitmap
Bitmap of indices of free objects.
Definition: pool.h:55
void serialize_32(serialize_main_t *m, va_list *va)
Definition: serialize.c:56
template key/value backing page structure
Definition: bihash_doc.h:44
static void * serialize_write_not_inline(serialize_main_header_t *m, serialize_stream_t *s, uword n_bytes_to_write, uword flags)
Definition: serialize.c:695
unsigned short u16
Definition: types.h:57
void unserialize_f64(serialize_main_t *m, va_list *va)
Definition: serialize.c:138
void unserialize_vector(serialize_main_t *m, va_list *va)
Definition: serialize.c:345
void unserialize_f32(serialize_main_t *m, va_list *va)
Definition: serialize.c:151
uword * unserialize_bitmap(serialize_main_t *m)
Definition: serialize.c:377
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
void( serialize_function_t)(serialize_main_t *m, va_list *va)
Definition: serialize.h:168
unsigned char u8
Definition: types.h:56
void serialize_heap(serialize_main_t *m, va_list *va)
Definition: serialize.c:509
void serialize_vector(serialize_main_t *m, va_list *va)
Definition: serialize.c:283
clib_error_t * va_serialize(serialize_main_t *sm, va_list *va)
Definition: serialize.c:650
void unserialize_pool(serialize_main_t *m, va_list *va)
Definition: serialize.c:464
static void * unserialize_pool_helper(serialize_main_t *m, u32 elt_bytes, u32 align, serialize_function_t *f)
Definition: serialize.c:434
static void clib_file_write(serialize_main_header_t *m, serialize_stream_t *s)
Definition: serialize.c:1150
#define SERIALIZE_FLAG_IS_WRITE
Definition: serialize.h:120
static clib_error_t * serialize_open_clib_file_helper(serialize_main_t *m, char *file, uword is_read)
Definition: serialize.c:1220
#define clib_mem_unaligned(pointer, type)
Definition: types.h:155
#define vec_foreach(var, vec)
Vector iterator.
void unserialize_open_clib_file_descriptor(serialize_main_t *m, int fd)
Definition: serialize.c:1214
void * serialize_close_vector(serialize_main_t *m)
Definition: serialize.c:918
void unserialize_64(serialize_main_t *m, va_list *va)
Definition: serialize.c:77
u32 flags
Definition: vhost-user.h:77
#define BITS(x)
Definition: clib.h:58
void serialize_64(serialize_main_t *m, va_list *va)
Definition: serialize.c:45
void serialize_multiple_1(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:933
clib_longjmp_t error_longjmp
Definition: serialize.h:103
serialize_function_t unserialize_vec_32
Definition: serialize.h:364
u32 ** free_lists
Definition: heap.h:136