FD.io VPP  v17.01-9-ge7dcee4
Vector Packet Processing
out2in.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
20 
21 #include <vnet/ip/ip.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/fib/ip4_fib.h>
24 #include <snat/snat.h>
25 
26 #include <vppinfra/hash.h>
27 #include <vppinfra/error.h>
28 #include <vppinfra/elog.h>
29 
30 typedef struct {
35 
36 typedef struct {
40 
41 /* packet trace format function */
42 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
43 {
44  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
45  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
46  snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
47 
48  s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
50  return s;
51 }
52 
53 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
54 {
55  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
56  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
57  snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
58 
59  s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
60  t->sw_if_index, t->next_index);
61  return s;
62 }
63 
64 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
65 {
66  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
67  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
69  va_arg (*args, snat_out2in_worker_handoff_trace_t *);
70  char * m;
71 
72  m = t->do_handoff ? "next worker" : "same worker";
73  s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
74 
75  return s;
76 }
77 
81 
82 #define foreach_snat_out2in_error \
83 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
84 _(OUT2IN_PACKETS, "Good out2in packets processed") \
85 _(BAD_ICMP_TYPE, "icmp type not echo-reply") \
86 _(NO_TRANSLATION, "No translation")
87 
88 typedef enum {
89 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
91 #undef _
94 
95 static char * snat_out2in_error_strings[] = {
96 #define _(sym,string) string,
98 #undef _
99 };
100 
101 typedef enum {
106 
107 /**
108  * @brief Create session for static mapping.
109  *
110  * Create NAT session initiated by host from external network with static
111  * mapping.
112  *
113  * @param sm SNAT main.
114  * @param b0 Vlib buffer.
115  * @param in2out In2out SNAT session key.
116  * @param out2in Out2in SNAT session key.
117  * @param node Vlib node.
118  *
119  * @returns SNAT session if successfully created otherwise 0.
120  */
121 static inline snat_session_t *
123  vlib_buffer_t *b0,
124  snat_session_key_t in2out,
125  snat_session_key_t out2in,
126  vlib_node_runtime_t * node,
127  u32 cpu_index)
128 {
129  snat_user_t *u;
130  snat_user_key_t user_key;
131  snat_session_t *s;
132  clib_bihash_kv_8_8_t kv0, value0;
133  dlist_elt_t * per_user_translation_list_elt;
134  dlist_elt_t * per_user_list_head_elt;
135 
136  user_key.addr = in2out.addr;
137  user_key.fib_index = in2out.fib_index;
138  kv0.key = user_key.as_u64;
139 
140  /* Ever heard of the "user" = inside ip4 address before? */
141  if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
142  {
143  /* no, make a new one */
144  pool_get (sm->per_thread_data[cpu_index].users, u);
145  memset (u, 0, sizeof (*u));
146  u->addr = in2out.addr;
147 
148  pool_get (sm->per_thread_data[cpu_index].list_pool,
149  per_user_list_head_elt);
150 
151  u->sessions_per_user_list_head_index = per_user_list_head_elt -
152  sm->per_thread_data[cpu_index].list_pool;
153 
154  clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
156 
157  kv0.value = u - sm->per_thread_data[cpu_index].users;
158 
159  /* add user */
160  clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
161 
162  /* add non-traslated packets worker lookup */
163  kv0.value = cpu_index;
164  clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
165  }
166  else
167  {
168  u = pool_elt_at_index (sm->per_thread_data[cpu_index].users,
169  value0.value);
170  }
171 
172  pool_get (sm->per_thread_data[cpu_index].sessions, s);
173  memset (s, 0, sizeof (*s));
174 
175  s->outside_address_index = ~0;
177  u->nstaticsessions++;
178 
179  /* Create list elts */
180  pool_get (sm->per_thread_data[cpu_index].list_pool,
181  per_user_translation_list_elt);
182  clib_dlist_init (sm->per_thread_data[cpu_index].list_pool,
183  per_user_translation_list_elt -
184  sm->per_thread_data[cpu_index].list_pool);
185 
186  per_user_translation_list_elt->value =
187  s - sm->per_thread_data[cpu_index].sessions;
188  s->per_user_index =
189  per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool;
190  s->per_user_list_head_index = u->sessions_per_user_list_head_index;
191 
193  s->per_user_list_head_index,
194  per_user_translation_list_elt -
195  sm->per_thread_data[cpu_index].list_pool);
196 
197  s->in2out = in2out;
198  s->out2in = out2in;
199  s->in2out.protocol = out2in.protocol;
200 
201  /* Add to translation hashes */
202  kv0.key = s->in2out.as_u64;
203  kv0.value = s - sm->per_thread_data[cpu_index].sessions;
204  if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
205  clib_warning ("in2out key add failed");
206 
207  kv0.key = s->out2in.as_u64;
208  kv0.value = s - sm->per_thread_data[cpu_index].sessions;
209 
210  if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
211  clib_warning ("out2in key add failed");
212 
213  return s;
214 }
215 
217  vlib_buffer_t * b0,
218  ip4_header_t * ip0,
219  icmp46_header_t * icmp0,
220  u32 sw_if_index0,
221  u32 rx_fib_index0,
222  vlib_node_runtime_t * node,
223  u32 next0, f64 now,
224  u32 cpu_index)
225 {
226  snat_session_key_t key0, sm0;
227  icmp_echo_header_t *echo0;
228  clib_bihash_kv_8_8_t kv0, value0;
229  snat_session_t * s0;
230  u32 new_addr0, old_addr0;
231  u16 old_id0, new_id0;
232  ip_csum_t sum0;
234 
235  echo0 = (icmp_echo_header_t *)(icmp0+1);
236 
237  key0.addr = ip0->dst_address;
238  key0.port = echo0->identifier;
240  key0.fib_index = rx_fib_index0;
241 
242  kv0.key = key0.as_u64;
243 
244  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
245  {
246  /* Try to match static mapping by external address and port,
247  destination address and port in packet */
248  if (snat_static_mapping_match(sm, key0, &sm0, 1))
249  {
250  ip4_address_t * first_int_addr;
251 
252  if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
253  {
254  first_int_addr =
255  ip4_interface_first_address (sm->ip4_main, sw_if_index0,
256  0 /* just want the address */);
257  rt->cached_sw_if_index = sw_if_index0;
258  rt->cached_ip4_address = first_int_addr->as_u32;
259  }
260 
261  /* Don't NAT packet aimed at the intfc address */
262  if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
263  rt->cached_ip4_address))
264  return next0;
265 
266  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
267  return SNAT_OUT2IN_NEXT_DROP;
268  }
269 
270  /* Create session initiated by host from external network */
271  s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
272  node, cpu_index);
273  if (!s0)
274  return SNAT_OUT2IN_NEXT_DROP;
275  }
276  else
277  s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
278  value0.value);
279 
280  old_addr0 = ip0->dst_address.as_u32;
281  ip0->dst_address = s0->in2out.addr;
282  new_addr0 = ip0->dst_address.as_u32;
283  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
284 
285  sum0 = ip0->checksum;
286  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
287  ip4_header_t,
288  dst_address /* changed member */);
289  ip0->checksum = ip_csum_fold (sum0);
290 
291  old_id0 = echo0->identifier;
292  new_id0 = s0->in2out.port;
293  echo0->identifier = new_id0;
294 
295  sum0 = icmp0->checksum;
296  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
297  identifier);
298  icmp0->checksum = ip_csum_fold (sum0);
299 
300  /* Accounting */
301  s0->last_heard = now;
302  s0->total_pkts++;
303  s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
304  /* Per-user LRU list maintenance for dynamic translation */
305  if (!snat_is_session_static (s0))
306  {
308  s0->per_user_index);
310  s0->per_user_list_head_index,
311  s0->per_user_index);
312  }
313 
314  return next0;
315 }
316 
317 static uword
319  vlib_node_runtime_t * node,
320  vlib_frame_t * frame)
321 {
322  u32 n_left_from, * from, * to_next;
323  snat_out2in_next_t next_index;
324  u32 pkts_processed = 0;
325  snat_main_t * sm = &snat_main;
326  f64 now = vlib_time_now (vm);
327  u32 cpu_index = os_get_cpu_number ();
328 
329  from = vlib_frame_vector_args (frame);
330  n_left_from = frame->n_vectors;
331  next_index = node->cached_next_index;
332 
333  while (n_left_from > 0)
334  {
335  u32 n_left_to_next;
336 
337  vlib_get_next_frame (vm, node, next_index,
338  to_next, n_left_to_next);
339 
340  while (n_left_from >= 4 && n_left_to_next >= 2)
341  {
342  u32 bi0, bi1;
343  vlib_buffer_t * b0, * b1;
346  u32 sw_if_index0, sw_if_index1;
347  ip4_header_t * ip0, *ip1;
348  ip_csum_t sum0, sum1;
349  u32 new_addr0, old_addr0;
350  u16 new_port0, old_port0;
351  u32 new_addr1, old_addr1;
352  u16 new_port1, old_port1;
353  udp_header_t * udp0, * udp1;
354  tcp_header_t * tcp0, * tcp1;
355  icmp46_header_t * icmp0, * icmp1;
356  snat_session_key_t key0, key1, sm0, sm1;
357  u32 rx_fib_index0, rx_fib_index1;
358  u32 proto0, proto1;
359  snat_session_t * s0 = 0, * s1 = 0;
360  clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
361 
362  /* Prefetch next iteration. */
363  {
364  vlib_buffer_t * p2, * p3;
365 
366  p2 = vlib_get_buffer (vm, from[2]);
367  p3 = vlib_get_buffer (vm, from[3]);
368 
369  vlib_prefetch_buffer_header (p2, LOAD);
370  vlib_prefetch_buffer_header (p3, LOAD);
371 
374  }
375 
376  /* speculatively enqueue b0 and b1 to the current next frame */
377  to_next[0] = bi0 = from[0];
378  to_next[1] = bi1 = from[1];
379  from += 2;
380  to_next += 2;
381  n_left_from -= 2;
382  n_left_to_next -= 2;
383 
384  b0 = vlib_get_buffer (vm, bi0);
385  b1 = vlib_get_buffer (vm, bi1);
386 
387  ip0 = vlib_buffer_get_current (b0);
388  udp0 = ip4_next_header (ip0);
389  tcp0 = (tcp_header_t *) udp0;
390  icmp0 = (icmp46_header_t *) udp0;
391 
392  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
393  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
394  sw_if_index0);
395 
396  proto0 = ~0;
397  proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
398  ? SNAT_PROTOCOL_UDP : proto0;
399  proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
400  ? SNAT_PROTOCOL_TCP : proto0;
401  proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
402  ? SNAT_PROTOCOL_ICMP : proto0;
403 
404  if (PREDICT_FALSE (proto0 == ~0))
405  goto trace0;
406 
407  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
408  {
409  next0 = icmp_out2in_slow_path
410  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
411  next0, now, cpu_index);
412  goto trace0;
413  }
414 
415  key0.addr = ip0->dst_address;
416  key0.port = udp0->dst_port;
417  key0.protocol = proto0;
418  key0.fib_index = rx_fib_index0;
419 
420  kv0.key = key0.as_u64;
421 
422  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
423  {
424  /* Try to match static mapping by external address and port,
425  destination address and port in packet */
426  if (snat_static_mapping_match(sm, key0, &sm0, 1))
427  {
428  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
429  goto trace0;
430  }
431 
432  /* Create session initiated by host from external network */
433  s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
434  cpu_index);
435  if (!s0)
436  goto trace0;
437  }
438  else
439  s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
440  value0.value);
441 
442  old_addr0 = ip0->dst_address.as_u32;
443  ip0->dst_address = s0->in2out.addr;
444  new_addr0 = ip0->dst_address.as_u32;
445  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
446 
447  sum0 = ip0->checksum;
448  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
449  ip4_header_t,
450  dst_address /* changed member */);
451  ip0->checksum = ip_csum_fold (sum0);
452 
453  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
454  {
455  old_port0 = tcp0->ports.dst;
456  tcp0->ports.dst = s0->in2out.port;
457  new_port0 = tcp0->ports.dst;
458 
459  sum0 = tcp0->checksum;
460  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
461  ip4_header_t,
462  dst_address /* changed member */);
463 
464  sum0 = ip_csum_update (sum0, old_port0, new_port0,
465  ip4_header_t /* cheat */,
466  length /* changed member */);
467  tcp0->checksum = ip_csum_fold(sum0);
468  }
469  else
470  {
471  old_port0 = udp0->dst_port;
472  udp0->dst_port = s0->in2out.port;
473  udp0->checksum = 0;
474  }
475 
476  /* Accounting */
477  s0->last_heard = now;
478  s0->total_pkts++;
479  s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
480  /* Per-user LRU list maintenance for dynamic translation */
481  if (!snat_is_session_static (s0))
482  {
484  s0->per_user_index);
486  s0->per_user_list_head_index,
487  s0->per_user_index);
488  }
489  trace0:
490 
492  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
493  {
494  snat_out2in_trace_t *t =
495  vlib_add_trace (vm, node, b0, sizeof (*t));
496  t->sw_if_index = sw_if_index0;
497  t->next_index = next0;
498  t->session_index = ~0;
499  if (s0)
500  t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
501  }
502 
503  pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
504 
505 
506  ip1 = vlib_buffer_get_current (b1);
507  udp1 = ip4_next_header (ip1);
508  tcp1 = (tcp_header_t *) udp1;
509  icmp1 = (icmp46_header_t *) udp1;
510 
511  sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
512  rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
513  sw_if_index1);
514 
515  proto1 = ~0;
516  proto1 = (ip1->protocol == IP_PROTOCOL_UDP)
517  ? SNAT_PROTOCOL_UDP : proto1;
518  proto1 = (ip1->protocol == IP_PROTOCOL_TCP)
519  ? SNAT_PROTOCOL_TCP : proto1;
520  proto1 = (ip1->protocol == IP_PROTOCOL_ICMP)
521  ? SNAT_PROTOCOL_ICMP : proto1;
522 
523  if (PREDICT_FALSE (proto1 == ~0))
524  goto trace1;
525 
526  if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
527  {
528  next1 = icmp_out2in_slow_path
529  (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
530  next1, now, cpu_index);
531  goto trace1;
532  }
533 
534  key1.addr = ip1->dst_address;
535  key1.port = udp1->dst_port;
536  key1.protocol = proto1;
537  key1.fib_index = rx_fib_index1;
538 
539  kv1.key = key1.as_u64;
540 
541  if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
542  {
543  /* Try to match static mapping by external address and port,
544  destination address and port in packet */
545  if (snat_static_mapping_match(sm, key1, &sm1, 1))
546  {
547  b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
548  goto trace1;
549  }
550 
551  /* Create session initiated by host from external network */
552  s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
553  cpu_index);
554  if (!s1)
555  goto trace1;
556  }
557  else
558  s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
559  value1.value);
560 
561  old_addr1 = ip1->dst_address.as_u32;
562  ip1->dst_address = s1->in2out.addr;
563  new_addr1 = ip1->dst_address.as_u32;
564  vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
565 
566  sum1 = ip1->checksum;
567  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
568  ip4_header_t,
569  dst_address /* changed member */);
570  ip1->checksum = ip_csum_fold (sum1);
571 
572  if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
573  {
574  old_port1 = tcp1->ports.dst;
575  tcp1->ports.dst = s1->in2out.port;
576  new_port1 = tcp1->ports.dst;
577 
578  sum1 = tcp1->checksum;
579  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
580  ip4_header_t,
581  dst_address /* changed member */);
582 
583  sum1 = ip_csum_update (sum1, old_port1, new_port1,
584  ip4_header_t /* cheat */,
585  length /* changed member */);
586  tcp1->checksum = ip_csum_fold(sum1);
587  }
588  else
589  {
590  old_port1 = udp1->dst_port;
591  udp1->dst_port = s1->in2out.port;
592  udp1->checksum = 0;
593  }
594 
595  /* Accounting */
596  s1->last_heard = now;
597  s1->total_pkts++;
598  s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
599  /* Per-user LRU list maintenance for dynamic translation */
600  if (!snat_is_session_static (s1))
601  {
603  s1->per_user_index);
605  s1->per_user_list_head_index,
606  s1->per_user_index);
607  }
608  trace1:
609 
611  && (b1->flags & VLIB_BUFFER_IS_TRACED)))
612  {
613  snat_out2in_trace_t *t =
614  vlib_add_trace (vm, node, b1, sizeof (*t));
615  t->sw_if_index = sw_if_index1;
616  t->next_index = next1;
617  t->session_index = ~0;
618  if (s1)
619  t->session_index = s1 - sm->per_thread_data[cpu_index].sessions;
620  }
621 
622  pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
623 
624  /* verify speculative enqueues, maybe switch current next frame */
625  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
626  to_next, n_left_to_next,
627  bi0, bi1, next0, next1);
628  }
629 
630  while (n_left_from > 0 && n_left_to_next > 0)
631  {
632  u32 bi0;
633  vlib_buffer_t * b0;
635  u32 sw_if_index0;
636  ip4_header_t * ip0;
637  ip_csum_t sum0;
638  u32 new_addr0, old_addr0;
639  u16 new_port0, old_port0;
640  udp_header_t * udp0;
641  tcp_header_t * tcp0;
642  icmp46_header_t * icmp0;
643  snat_session_key_t key0, sm0;
644  u32 rx_fib_index0;
645  u32 proto0;
646  snat_session_t * s0 = 0;
647  clib_bihash_kv_8_8_t kv0, value0;
648 
649  /* speculatively enqueue b0 to the current next frame */
650  bi0 = from[0];
651  to_next[0] = bi0;
652  from += 1;
653  to_next += 1;
654  n_left_from -= 1;
655  n_left_to_next -= 1;
656 
657  b0 = vlib_get_buffer (vm, bi0);
658 
659  ip0 = vlib_buffer_get_current (b0);
660  udp0 = ip4_next_header (ip0);
661  tcp0 = (tcp_header_t *) udp0;
662  icmp0 = (icmp46_header_t *) udp0;
663 
664  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
665  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
666  sw_if_index0);
667 
668  proto0 = ~0;
669  proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
670  ? SNAT_PROTOCOL_UDP : proto0;
671  proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
672  ? SNAT_PROTOCOL_TCP : proto0;
673  proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
674  ? SNAT_PROTOCOL_ICMP : proto0;
675 
676  if (PREDICT_FALSE (proto0 == ~0))
677  goto trace00;
678 
679  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
680  {
681  next0 = icmp_out2in_slow_path
682  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
683  next0, now, cpu_index);
684  goto trace00;
685  }
686 
687  key0.addr = ip0->dst_address;
688  key0.port = udp0->dst_port;
689  key0.protocol = proto0;
690  key0.fib_index = rx_fib_index0;
691 
692  kv0.key = key0.as_u64;
693 
694  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
695  {
696  /* Try to match static mapping by external address and port,
697  destination address and port in packet */
698  if (snat_static_mapping_match(sm, key0, &sm0, 1))
699  {
700  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
701  goto trace00;
702  }
703 
704  /* Create session initiated by host from external network */
705  s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
706  cpu_index);
707  if (!s0)
708  goto trace00;
709  }
710  else
711  s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions,
712  value0.value);
713 
714  old_addr0 = ip0->dst_address.as_u32;
715  ip0->dst_address = s0->in2out.addr;
716  new_addr0 = ip0->dst_address.as_u32;
717  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
718 
719  sum0 = ip0->checksum;
720  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
721  ip4_header_t,
722  dst_address /* changed member */);
723  ip0->checksum = ip_csum_fold (sum0);
724 
725  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
726  {
727  old_port0 = tcp0->ports.dst;
728  tcp0->ports.dst = s0->in2out.port;
729  new_port0 = tcp0->ports.dst;
730 
731  sum0 = tcp0->checksum;
732  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
733  ip4_header_t,
734  dst_address /* changed member */);
735 
736  sum0 = ip_csum_update (sum0, old_port0, new_port0,
737  ip4_header_t /* cheat */,
738  length /* changed member */);
739  tcp0->checksum = ip_csum_fold(sum0);
740  }
741  else
742  {
743  old_port0 = udp0->dst_port;
744  udp0->dst_port = s0->in2out.port;
745  udp0->checksum = 0;
746  }
747 
748  /* Accounting */
749  s0->last_heard = now;
750  s0->total_pkts++;
751  s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
752  /* Per-user LRU list maintenance for dynamic translation */
753  if (!snat_is_session_static (s0))
754  {
756  s0->per_user_index);
758  s0->per_user_list_head_index,
759  s0->per_user_index);
760  }
761  trace00:
762 
764  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
765  {
766  snat_out2in_trace_t *t =
767  vlib_add_trace (vm, node, b0, sizeof (*t));
768  t->sw_if_index = sw_if_index0;
769  t->next_index = next0;
770  t->session_index = ~0;
771  if (s0)
772  t->session_index = s0 - sm->per_thread_data[cpu_index].sessions;
773  }
774 
775  pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
776 
777  /* verify speculative enqueue, maybe switch current next frame */
778  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
779  to_next, n_left_to_next,
780  bi0, next0);
781  }
782 
783  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
784  }
785 
787  SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
788  pkts_processed);
789  return frame->n_vectors;
790 }
791 
793  .function = snat_out2in_node_fn,
794  .name = "snat-out2in",
795  .vector_size = sizeof (u32),
796  .format_trace = format_snat_out2in_trace,
798 
800  .error_strings = snat_out2in_error_strings,
801 
802  .runtime_data_bytes = sizeof (snat_runtime_t),
803 
804  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
805 
806  /* edit / add dispositions here */
807  .next_nodes = {
808  [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
809  [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
810  },
811 };
813 
814 static uword
816  vlib_node_runtime_t * node,
817  vlib_frame_t * frame)
818 {
819  snat_main_t *sm = &snat_main;
821  u32 n_left_from, *from, *to_next = 0;
822  static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
823  static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
824  = 0;
825  vlib_frame_queue_elt_t *hf = 0;
826  vlib_frame_t *f = 0;
827  int i;
828  u32 n_left_to_next_worker = 0, *to_next_worker = 0;
829  u32 next_worker_index = 0;
830  u32 current_worker_index = ~0;
831  u32 cpu_index = os_get_cpu_number ();
832 
833  ASSERT (vec_len (sm->workers));
834 
835  if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
836  {
837  vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
838 
839  vec_validate_init_empty (congested_handoff_queue_by_worker_index,
840  sm->first_worker_index + sm->num_workers - 1,
841  (vlib_frame_queue_t *) (~0));
842  }
843 
844  from = vlib_frame_vector_args (frame);
845  n_left_from = frame->n_vectors;
846 
847  while (n_left_from > 0)
848  {
849  u32 bi0;
850  vlib_buffer_t *b0;
851  u32 sw_if_index0;
852  u32 rx_fib_index0;
853  ip4_header_t * ip0;
854  udp_header_t * udp0;
856  clib_bihash_kv_8_8_t kv0, value0;
857  u8 do_handoff;
858 
859  bi0 = from[0];
860  from += 1;
861  n_left_from -= 1;
862 
863  b0 = vlib_get_buffer (vm, bi0);
864 
865  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
866  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
867 
868  ip0 = vlib_buffer_get_current (b0);
869  udp0 = ip4_next_header (ip0);
870 
871  key0.addr = ip0->dst_address;
872  key0.port = udp0->dst_port;
873  key0.fib_index = rx_fib_index0;
874 
875  if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_ICMP))
876  {
877  icmp46_header_t * icmp0 = (icmp46_header_t *) udp0;
878  icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
879  key0.port = echo0->identifier;
880  }
881 
882  kv0.key = key0.as_u64;
883 
884  /* Ever heard of of the "user" before? */
885  if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
886  {
887  key0.port = 0;
888  kv0.key = key0.as_u64;
889 
890  if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
891  {
892  /* No, assign next available worker (RR) */
893  next_worker_index = sm->first_worker_index +
894  sm->workers[sm->next_worker++ % vec_len (sm->workers)];
895  }
896  else
897  {
898  /* Static mapping without port */
899  next_worker_index = value0.value;
900  }
901 
902  /* Add to translated packets worker lookup */
903  kv0.value = next_worker_index;
904  clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
905  }
906  else
907  next_worker_index = value0.value;
908 
909  if (PREDICT_FALSE (next_worker_index != cpu_index))
910  {
911  do_handoff = 1;
912 
913  if (next_worker_index != current_worker_index)
914  {
915  if (hf)
916  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
917 
919  next_worker_index,
920  handoff_queue_elt_by_worker_index);
921 
922  n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
923  to_next_worker = &hf->buffer_index[hf->n_vectors];
924  current_worker_index = next_worker_index;
925  }
926 
927  /* enqueue to correct worker thread */
928  to_next_worker[0] = bi0;
929  to_next_worker++;
930  n_left_to_next_worker--;
931 
932  if (n_left_to_next_worker == 0)
933  {
936  current_worker_index = ~0;
937  handoff_queue_elt_by_worker_index[next_worker_index] = 0;
938  hf = 0;
939  }
940  }
941  else
942  {
943  do_handoff = 0;
944  /* if this is 1st frame */
945  if (!f)
946  {
948  to_next = vlib_frame_vector_args (f);
949  }
950 
951  to_next[0] = bi0;
952  to_next += 1;
953  f->n_vectors++;
954  }
955 
957  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
958  {
960  vlib_add_trace (vm, node, b0, sizeof (*t));
961  t->next_worker_index = next_worker_index;
962  t->do_handoff = do_handoff;
963  }
964  }
965 
966  if (f)
968 
969  if (hf)
970  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
971 
972  /* Ship frames to the worker nodes */
973  for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
974  {
975  if (handoff_queue_elt_by_worker_index[i])
976  {
977  hf = handoff_queue_elt_by_worker_index[i];
978  /*
979  * It works better to let the handoff node
980  * rate-adapt, always ship the handoff queue element.
981  */
982  if (1 || hf->n_vectors == hf->last_n_vectors)
983  {
985  handoff_queue_elt_by_worker_index[i] = 0;
986  }
987  else
988  hf->last_n_vectors = hf->n_vectors;
989  }
990  congested_handoff_queue_by_worker_index[i] =
991  (vlib_frame_queue_t *) (~0);
992  }
993  hf = 0;
994  current_worker_index = ~0;
995  return frame->n_vectors;
996 }
997 
999  .function = snat_out2in_worker_handoff_fn,
1000  .name = "snat-out2in-worker-handoff",
1001  .vector_size = sizeof (u32),
1004 
1005  .n_next_nodes = 1,
1006 
1007  .next_nodes = {
1008  [0] = "error-drop",
1009  },
1010 };
1011 
1013 
1014 static inline u32 icmp_out2in_fast (snat_main_t *sm,
1015  vlib_buffer_t * b0,
1016  ip4_header_t * ip0,
1017  icmp46_header_t * icmp0,
1018  u32 sw_if_index0,
1019  vlib_node_runtime_t * node,
1020  u32 next0,
1021  u32 rx_fib_index0)
1022 {
1023  snat_session_key_t key0, sm0;
1024  icmp_echo_header_t *echo0;
1025  u32 new_addr0, old_addr0;
1026  u16 old_id0, new_id0;
1027  ip_csum_t sum0;
1028  snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data;
1029 
1030  echo0 = (icmp_echo_header_t *)(icmp0+1);
1031 
1032  key0.addr = ip0->dst_address;
1033  key0.port = echo0->identifier;
1034  key0.fib_index = rx_fib_index0;
1035 
1036  if (snat_static_mapping_match(sm, key0, &sm0, 1))
1037  {
1038  ip4_address_t * first_int_addr;
1039 
1040  if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
1041  {
1042  first_int_addr =
1043  ip4_interface_first_address (sm->ip4_main, sw_if_index0,
1044  0 /* just want the address */);
1045  rt->cached_sw_if_index = sw_if_index0;
1046  rt->cached_ip4_address = first_int_addr->as_u32;
1047  }
1048 
1049  /* Don't NAT packet aimed at the intfc address */
1050  if (PREDICT_FALSE(ip0->dst_address.as_u32 ==
1051  rt->cached_ip4_address))
1052  return next0;
1053 
1054  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1055  return SNAT_OUT2IN_NEXT_DROP;
1056  }
1057 
1058  new_addr0 = sm0.addr.as_u32;
1059  new_id0 = sm0.port;
1060  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1061 
1062  old_addr0 = ip0->dst_address.as_u32;
1063  ip0->dst_address.as_u32 = new_addr0;
1064 
1065  sum0 = ip0->checksum;
1066  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1067  ip4_header_t,
1068  dst_address /* changed member */);
1069  ip0->checksum = ip_csum_fold (sum0);
1070 
1071  if (PREDICT_FALSE(new_id0 != echo0->identifier))
1072  {
1073  old_id0 = echo0->identifier;
1074  echo0->identifier = new_id0;
1075 
1076  sum0 = icmp0->checksum;
1077  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
1078  identifier);
1079  icmp0->checksum = ip_csum_fold (sum0);
1080  }
1081 
1082  return next0;
1083 }
1084 
1085 static uword
1087  vlib_node_runtime_t * node,
1088  vlib_frame_t * frame)
1089 {
1090  u32 n_left_from, * from, * to_next;
1091  snat_out2in_next_t next_index;
1092  u32 pkts_processed = 0;
1093  snat_main_t * sm = &snat_main;
1094 
1095  from = vlib_frame_vector_args (frame);
1096  n_left_from = frame->n_vectors;
1097  next_index = node->cached_next_index;
1098 
1099  while (n_left_from > 0)
1100  {
1101  u32 n_left_to_next;
1102 
1103  vlib_get_next_frame (vm, node, next_index,
1104  to_next, n_left_to_next);
1105 
1106  while (n_left_from > 0 && n_left_to_next > 0)
1107  {
1108  u32 bi0;
1109  vlib_buffer_t * b0;
1110  u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1111  u32 sw_if_index0;
1112  ip4_header_t * ip0;
1113  ip_csum_t sum0;
1114  u32 new_addr0, old_addr0;
1115  u16 new_port0, old_port0;
1116  udp_header_t * udp0;
1117  tcp_header_t * tcp0;
1118  icmp46_header_t * icmp0;
1119  snat_session_key_t key0, sm0;
1120  u32 proto0;
1121  u32 rx_fib_index0;
1122 
1123  /* speculatively enqueue b0 to the current next frame */
1124  bi0 = from[0];
1125  to_next[0] = bi0;
1126  from += 1;
1127  to_next += 1;
1128  n_left_from -= 1;
1129  n_left_to_next -= 1;
1130 
1131  b0 = vlib_get_buffer (vm, bi0);
1132 
1133  ip0 = vlib_buffer_get_current (b0);
1134  udp0 = ip4_next_header (ip0);
1135  tcp0 = (tcp_header_t *) udp0;
1136  icmp0 = (icmp46_header_t *) udp0;
1137 
1138  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1139  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1140 
1141  vnet_feature_next (sw_if_index0, &next0, b0);
1142 
1143  proto0 = ~0;
1144  proto0 = (ip0->protocol == IP_PROTOCOL_UDP)
1145  ? SNAT_PROTOCOL_UDP : proto0;
1146  proto0 = (ip0->protocol == IP_PROTOCOL_TCP)
1147  ? SNAT_PROTOCOL_TCP : proto0;
1148  proto0 = (ip0->protocol == IP_PROTOCOL_ICMP)
1149  ? SNAT_PROTOCOL_ICMP : proto0;
1150 
1151  if (PREDICT_FALSE (proto0 == ~0))
1152  goto trace00;
1153 
1154  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1155  {
1156  next0 = icmp_out2in_fast
1157  (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0);
1158  goto trace00;
1159  }
1160 
1161  key0.addr = ip0->dst_address;
1162  key0.port = udp0->dst_port;
1163  key0.fib_index = rx_fib_index0;
1164 
1165  if (snat_static_mapping_match(sm, key0, &sm0, 1))
1166  {
1167  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1168  goto trace00;
1169  }
1170 
1171  new_addr0 = sm0.addr.as_u32;
1172  new_port0 = sm0.port;
1173  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1174  old_addr0 = ip0->dst_address.as_u32;
1175  ip0->dst_address.as_u32 = new_addr0;
1176 
1177  sum0 = ip0->checksum;
1178  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1179  ip4_header_t,
1180  dst_address /* changed member */);
1181  ip0->checksum = ip_csum_fold (sum0);
1182 
1183  if (PREDICT_FALSE(new_port0 != udp0->dst_port))
1184  {
1185  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1186  {
1187  old_port0 = tcp0->ports.dst;
1188  tcp0->ports.dst = new_port0;
1189 
1190  sum0 = tcp0->checksum;
1191  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1192  ip4_header_t,
1193  dst_address /* changed member */);
1194 
1195  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1196  ip4_header_t /* cheat */,
1197  length /* changed member */);
1198  tcp0->checksum = ip_csum_fold(sum0);
1199  }
1200  else
1201  {
1202  old_port0 = udp0->dst_port;
1203  udp0->dst_port = new_port0;
1204  udp0->checksum = 0;
1205  }
1206  }
1207  else
1208  {
1209  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1210  {
1211  sum0 = tcp0->checksum;
1212  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1213  ip4_header_t,
1214  dst_address /* changed member */);
1215 
1216  tcp0->checksum = ip_csum_fold(sum0);
1217  }
1218  }
1219 
1220  trace00:
1221 
1223  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1224  {
1225  snat_out2in_trace_t *t =
1226  vlib_add_trace (vm, node, b0, sizeof (*t));
1227  t->sw_if_index = sw_if_index0;
1228  t->next_index = next0;
1229  }
1230 
1231  pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1232 
1233  /* verify speculative enqueue, maybe switch current next frame */
1234  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1235  to_next, n_left_to_next,
1236  bi0, next0);
1237  }
1238 
1239  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1240  }
1241 
1243  SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1244  pkts_processed);
1245  return frame->n_vectors;
1246 }
1247 
1249  .function = snat_out2in_fast_node_fn,
1250  .name = "snat-out2in-fast",
1251  .vector_size = sizeof (u32),
1252  .format_trace = format_snat_out2in_fast_trace,
1254 
1255  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1256  .error_strings = snat_out2in_error_strings,
1257 
1258  .runtime_data_bytes = sizeof (snat_runtime_t),
1259 
1260  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1261 
1262  /* edit / add dispositions here */
1263  .next_nodes = {
1264  [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1265  [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1266  },
1267 };
vlib_node_registration_t snat_out2in_fast_node
(constructor) VLIB_REGISTER_NODE (snat_out2in_fast_node)
Definition: out2in.c:79
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:396
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:459
VLIB_NODE_FUNCTION_MULTIARCH(snat_out2in_node, snat_out2in_node_fn)
u32 sessions_per_user_list_head_index
Definition: snat.h:108
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:343
#define CLIB_UNUSED(x)
Definition: clib.h:79
vlib_main_t * vlib_main
Definition: snat.h:206
static void clib_dlist_init(dlist_elt_t *pool, u32 index)
Definition: dlist.h:36
u8 runtime_data[0]
Definition: node.h:469
static u32 icmp_out2in_slow_path(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, icmp46_header_t *icmp0, u32 sw_if_index0, u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0, f64 now, u32 cpu_index)
Definition: out2in.c:216
bad routing header type(not 4)") sr_error (NO_MORE_SEGMENTS
static u8 * format_snat_out2in_fast_trace(u8 *s, va_list *args)
Definition: out2in.c:53
#define PREDICT_TRUE(x)
Definition: clib.h:98
static u32 icmp_out2in_fast(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, icmp46_header_t *icmp0, u32 sw_if_index0, vlib_node_runtime_t *node, u32 next0, u32 rx_fib_index0)
Definition: out2in.c:1014
clib_bihash_8_8_t out2in
Definition: snat.h:147
ip4_address_t * ip4_interface_first_address(ip4_main_t *im, u32 sw_if_index, ip_interface_address_t **result_ia)
Definition: ip4_forward.c:710
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:182
u32 nstaticsessions
Definition: snat.h:110
struct _vlib_node_registration vlib_node_registration_t
uword ip_csum_t
Definition: ip_packet.h:90
u32 * fib_index_by_sw_if_index
Table index indexed by software interface.
Definition: ip4.h:105
u32 fq_out2in_index
Definition: snat.h:187
u32 buffer_index[VLIB_FRAME_SIZE]
Definition: threads.h:82
vlib_error_t * errors
Definition: node.h:419
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:100
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:200
dlist_elt_t * list_pool
Definition: snat.h:142
u32 * workers
Definition: snat.h:162
u32 cached_sw_if_index
Definition: snat.h:238
u32 ip4_fib_table_get_index_for_sw_if_index(u32 sw_if_index)
Definition: ip4_fib.c:212
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:194
ip4_address_t dst_address
Definition: ip4_packet.h:163
snat_main_per_thread_data_t * per_thread_data
Definition: snat.h:165
ip4_address_t addr
Definition: snat.h:107
int snat_static_mapping_match(snat_main_t *sm, snat_session_key_t match, snat_session_key_t *mapping, u8 by_external)
Match SNAT static mapping.
Definition: snat.c:1242
ip4_main_t * ip4_main
Definition: snat.h:208
#define clib_warning(format, args...)
Definition: error.h:59
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:232
#define foreach_snat_out2in_error
Definition: out2in.c:82
static uword snat_out2in_worker_handoff_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: out2in.c:815
clib_bihash_8_8_t worker_by_in
Definition: snat.h:154
snat_out2in_next_t
Definition: out2in.c:101
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:369
u64 key
the key
Definition: bihash_8_8.h:35
static void clib_dlist_addtail(dlist_elt_t *pool, u32 head_index, u32 new_index)
Definition: dlist.h:43
uword os_get_cpu_number(void)
Definition: unix-misc.c:224
snat_main_t snat_main
Definition: jvpp_snat.h:42
#define PREDICT_FALSE(x)
Definition: clib.h:97
static vlib_frame_queue_elt_t * vlib_get_worker_handoff_queue_elt(u32 frame_queue_index, u32 vlib_worker_index, vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index)
Definition: threads.h:445
#define VLIB_FRAME_SIZE
Definition: node.h:328
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:196
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
vlib_node_registration_t snat_out2in_node
(constructor) VLIB_REGISTER_NODE (snat_out2in_node)
Definition: out2in.c:78
static_always_inline void vnet_feature_next(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:223
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:216
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:350
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:121
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1113
u64 value
the value
Definition: bihash_8_8.h:36
snat_user_t * users
Definition: snat.h:136
u16 n_vectors
Definition: node.h:344
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
union tcp_header_t::@180::@182 ports
clib_bihash_8_8_t user_hash
Definition: snat.h:151
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
8 octet key, 8 octet key value pair
Definition: bihash_8_8.h:33
#define ARRAY_LEN(x)
Definition: clib.h:59
ip4_address_t addr
Definition: snat.h:37
u16 cached_next_index
Definition: node.h:463
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
u32 num_workers
Definition: snat.h:159
#define vnet_buffer(b)
Definition: buffer.h:361
u64 as_u64
Definition: snat.h:54
clib_bihash_8_8_t in2out
Definition: snat.h:148
ip4_address_t addr
Definition: snat.h:51
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:259
static void clib_dlist_remove(dlist_elt_t *pool, u32 index)
Definition: dlist.h:99
u32 next_worker
Definition: snat.h:161
#define VLIB_BUFFER_IS_TRACED
Definition: buffer.h:95
u32 value
Definition: dlist.h:32
u64 uword
Definition: types.h:112
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
#define vec_elt(v, i)
Get vector value at index i.
Definition: defs.h:47
unsigned short u16
Definition: types.h:57
vlib_node_registration_t snat_out2in_worker_handoff_node
(constructor) VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node)
Definition: out2in.c:80
snat_out2in_error_t
Definition: out2in.c:88
static snat_session_t * create_session_for_static_mapping(snat_main_t *sm, vlib_buffer_t *b0, snat_session_key_t in2out, snat_session_key_t out2in, vlib_node_runtime_t *node, u32 cpu_index)
Create session for static mapping.
Definition: out2in.c:122
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
static char * snat_out2in_error_strings[]
Definition: out2in.c:95
unsigned char u8
Definition: types.h:56
u32 first_worker_index
Definition: snat.h:160
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:253
#define ip_csum_update(sum, old, new, type, field)
Definition: ip_packet.h:139
u32 fib_index
Definition: snat.h:52
static u8 * format_snat_out2in_trace(u8 *s, va_list *args)
Definition: out2in.c:42
static void vlib_put_frame_queue_elt(vlib_frame_queue_elt_t *hf)
Definition: threads.h:375
ip4_address_t addr
Definition: snat.h:63
static uword snat_out2in_fast_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: out2in.c:1086
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:170
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:418
static uword snat_out2in_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: out2in.c:318
u8 data[0]
Packet data.
Definition: buffer.h:158
#define SNAT_SESSION_FLAG_STATIC_MAPPING
Definition: snat.h:79
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:187
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:445
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:85
snat_session_t * sessions
Definition: snat.h:139
u32 cached_ip4_address
Definition: snat.h:239
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
#define snat_is_session_static(s)
Check if SNAT session is created from static mapping.
Definition: snat.h:246
static u16 ip_csum_fold(ip_csum_t c)
Definition: ip_packet.h:145
clib_bihash_8_8_t worker_by_out
Definition: snat.h:157
static u8 * format_snat_out2in_worker_handoff_trace(u8 *s, va_list *args)
Definition: out2in.c:64
Definition: defs.h:46