FD.io VPP  v18.01.1-37-g7ea3975
Vector Packet Processing
in2out.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
20 
21 #include <vnet/ip/ip.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/fib/ip4_fib.h>
24 #include <nat/nat.h>
25 #include <nat/nat_ipfix_logging.h>
26 #include <nat/nat_det.h>
27 #include <nat/nat_reass.h>
28 
29 #include <vppinfra/hash.h>
30 #include <vppinfra/error.h>
31 #include <vppinfra/elog.h>
32 
33 typedef struct {
39 
40 typedef struct {
44 
45 /* packet trace format function */
46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
47 {
48  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
49  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
50  snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
51  char * tag;
52 
53  tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
54 
55  s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
57 
58  return s;
59 }
60 
61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
62 {
63  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
64  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
65  snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
66 
67  s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
68  t->sw_if_index, t->next_index);
69 
70  return s;
71 }
72 
73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
74 {
75  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
76  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
78  va_arg (*args, snat_in2out_worker_handoff_trace_t *);
79  char * m;
80 
81  m = t->do_handoff ? "next worker" : "same worker";
82  s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
83 
84  return s;
85 }
86 
87 typedef struct {
92 
93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
94 {
95  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
96  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
98 
99  s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
100  t->sw_if_index, t->next_index,
101  t->cached ? "cached" : "translated");
102 
103  return s;
104 }
105 
118 
119 
120 #define foreach_snat_in2out_error \
121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
122 _(IN2OUT_PACKETS, "Good in2out packets processed") \
123 _(OUT_OF_PORTS, "Out of ports") \
124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \
125 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
126 _(NO_TRANSLATION, "No translation") \
127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
128 _(DROP_FRAGMENT, "Drop fragment") \
129 _(MAX_REASS, "Maximum reassemblies exceeded") \
130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
131 
132 typedef enum {
133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
135 #undef _
138 
139 static char * snat_in2out_error_strings[] = {
140 #define _(sym,string) string,
142 #undef _
143 };
144 
145 typedef enum {
153 
154 typedef enum {
161 
162 /**
163  * @brief Check if packet should be translated
164  *
165  * Packets aimed at outside interface and external addresss with active session
166  * should be translated.
167  *
168  * @param sm NAT main
169  * @param rt NAT runtime data
170  * @param sw_if_index0 index of the inside interface
171  * @param ip0 IPv4 header
172  * @param proto0 NAT protocol
173  * @param rx_fib_index0 RX FIB index
174  *
175  * @returns 0 if packet should be translated otherwise 1
176  */
177 static inline int
179  u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
180  u32 rx_fib_index0)
181 {
183  fib_prefix_t pfx = {
185  .fp_len = 32,
186  .fp_addr = {
187  .ip4.as_u32 = ip0->dst_address.as_u32,
188  },
189  };
190 
191  /* Don't NAT packet aimed at the intfc address */
192  if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
193  ip0->dst_address.as_u32)))
194  return 1;
195 
196  fei = fib_table_lookup (rx_fib_index0, &pfx);
197  if (FIB_NODE_INDEX_INVALID != fei)
198  {
199  u32 sw_if_index = fib_entry_get_resolving_interface (fei);
200  if (sw_if_index == ~0)
201  {
202  fei = fib_table_lookup (sm->outside_fib_index, &pfx);
203  if (FIB_NODE_INDEX_INVALID != fei)
204  sw_if_index = fib_entry_get_resolving_interface (fei);
205  }
207  pool_foreach (i, sm->interfaces,
208  ({
209  /* NAT packet aimed at outside interface */
210  if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
211  return 0;
212  }));
213  }
214 
215  return 1;
216 }
217 
218 static inline int
220  u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
221  u32 rx_fib_index0, u32 thread_index)
222 {
223  udp_header_t * udp0 = ip4_next_header (ip0);
224  snat_session_key_t key0, sm0;
225  clib_bihash_kv_8_8_t kv0, value0;
226 
227  key0.addr = ip0->dst_address;
228  key0.port = udp0->dst_port;
229  key0.protocol = proto0;
230  key0.fib_index = sm->outside_fib_index;
231  kv0.key = key0.as_u64;
232 
233  /* NAT packet aimed at external address if */
234  /* has active sessions */
235  if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
236  &value0))
237  {
238  /* or is static mappings */
239  if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
240  return 0;
241  }
242  else
243  return 0;
244 
245  if (sm->forwarding_enabled)
246  return 1;
247 
248  return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
249  rx_fib_index0);
250 }
251 
252 static inline int
254  u32 proto0, u16 src_port, u16 dst_port,
255  u32 thread_index, u32 sw_if_index)
256 {
257  snat_session_key_t key0;
258  clib_bihash_kv_8_8_t kv0, value0;
260 
261  /* src NAT check */
262  key0.addr = ip0->src_address;
263  key0.port = src_port;
264  key0.protocol = proto0;
265  key0.fib_index = sm->outside_fib_index;
266  kv0.key = key0.as_u64;
267 
268  if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
269  &value0))
270  return 1;
271 
272  /* dst NAT check */
273  key0.addr = ip0->dst_address;
274  key0.port = dst_port;
275  key0.protocol = proto0;
276  key0.fib_index = sm->inside_fib_index;
277  kv0.key = key0.as_u64;
278  if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
279  &value0))
280  {
281  /* hairpinning */
283  ({
284  if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
285  return 0;
286  }));
287  return 1;
288  }
289 
290  return 0;
291 }
292 
294  ip4_header_t * ip0,
295  u32 rx_fib_index0,
296  snat_session_key_t * key0,
297  snat_session_t ** sessionp,
298  vlib_node_runtime_t * node,
299  u32 next0,
300  u32 thread_index)
301 {
302  snat_user_t *u;
303  snat_session_t *s;
305  snat_session_key_t key1;
306  u32 address_index = ~0;
307  u32 outside_fib_index;
308  uword * p;
309  udp_header_t * udp0 = ip4_next_header (ip0);
310  u8 is_sm = 0;
311 
312  if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
313  {
314  b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
315  return SNAT_IN2OUT_NEXT_DROP;
316  }
317 
319  if (! p)
320  {
321  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
322  return SNAT_IN2OUT_NEXT_DROP;
323  }
324  outside_fib_index = p[0];
325 
326  key1.protocol = key0->protocol;
327 
328  u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
329  thread_index);
330  if (!u)
331  {
332  clib_warning ("create NAT user failed");
333  return SNAT_IN2OUT_NEXT_DROP;
334  }
335 
336  /* First try to match static mapping by local address and port */
337  if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0))
338  {
339  /* Try to create dynamic translation */
340  if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
341  thread_index, &key1,
342  &address_index,
343  sm->port_per_thread,
344  sm->per_thread_data[thread_index].snat_thread_index))
345  {
346  b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
347  return SNAT_IN2OUT_NEXT_DROP;
348  }
349  }
350  else
351  is_sm = 1;
352 
353  s = nat_session_alloc_or_recycle (sm, u, thread_index);
354  if (!s)
355  {
356  clib_warning ("create NAT session failed");
357  return SNAT_IN2OUT_NEXT_DROP;
358  }
359 
360  if (is_sm)
362  user_session_increment (sm, u, is_sm);
363  s->outside_address_index = address_index;
364  s->in2out = *key0;
365  s->out2in = key1;
366  s->out2in.protocol = key0->protocol;
367  s->out2in.fib_index = outside_fib_index;
368  s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
369  s->ext_host_port = udp0->dst_port;
370  *sessionp = s;
371 
372  /* Add to translation hashes */
373  kv0.key = s->in2out.as_u64;
374  kv0.value = s - sm->per_thread_data[thread_index].sessions;
375  if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
376  1 /* is_add */))
377  clib_warning ("in2out key add failed");
378 
379  kv0.key = s->out2in.as_u64;
380  kv0.value = s - sm->per_thread_data[thread_index].sessions;
381 
382  if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
383  1 /* is_add */))
384  clib_warning ("out2in key add failed");
385 
386  /* log NAT event */
387  snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
388  s->out2in.addr.as_u32,
389  s->in2out.protocol,
390  s->in2out.port,
391  s->out2in.port,
392  s->in2out.fib_index);
393  return next0;
394 }
395 
398  snat_session_key_t *p_key0)
399 {
400  icmp46_header_t *icmp0;
401  snat_session_key_t key0;
402  icmp_echo_header_t *echo0, *inner_echo0 = 0;
403  ip4_header_t *inner_ip0 = 0;
404  void *l4_header = 0;
405  icmp46_header_t *inner_icmp0;
406 
407  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
408  echo0 = (icmp_echo_header_t *)(icmp0+1);
409 
410  if (!icmp_is_error_message (icmp0))
411  {
412  key0.protocol = SNAT_PROTOCOL_ICMP;
413  key0.addr = ip0->src_address;
414  key0.port = echo0->identifier;
415  }
416  else
417  {
418  inner_ip0 = (ip4_header_t *)(echo0+1);
419  l4_header = ip4_next_header (inner_ip0);
420  key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
421  key0.addr = inner_ip0->dst_address;
422  switch (key0.protocol)
423  {
424  case SNAT_PROTOCOL_ICMP:
425  inner_icmp0 = (icmp46_header_t*)l4_header;
426  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
427  key0.port = inner_echo0->identifier;
428  break;
429  case SNAT_PROTOCOL_UDP:
430  case SNAT_PROTOCOL_TCP:
431  key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
432  break;
433  default:
434  return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
435  }
436  }
437  *p_key0 = key0;
438  return -1; /* success */
439 }
440 
443 {
444  icmp46_header_t *icmp0;
445  nat_ed_ses_key_t key0;
446  icmp_echo_header_t *echo0, *inner_echo0 = 0;
447  ip4_header_t *inner_ip0 = 0;
448  void *l4_header = 0;
449  icmp46_header_t *inner_icmp0;
450 
451  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
452  echo0 = (icmp_echo_header_t *)(icmp0+1);
453 
454  if (!icmp_is_error_message (icmp0))
455  {
456  key0.proto = IP_PROTOCOL_ICMP;
457  key0.l_addr = ip0->src_address;
458  key0.r_addr = ip0->dst_address;
459  key0.l_port = key0.r_port = echo0->identifier;
460  }
461  else
462  {
463  inner_ip0 = (ip4_header_t *)(echo0+1);
464  l4_header = ip4_next_header (inner_ip0);
465  key0.proto = inner_ip0->protocol;
466  key0.r_addr = inner_ip0->src_address;
467  key0.l_addr = inner_ip0->dst_address;
468  switch (ip_proto_to_snat_proto (inner_ip0->protocol))
469  {
470  case SNAT_PROTOCOL_ICMP:
471  inner_icmp0 = (icmp46_header_t*)l4_header;
472  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
473  key0.r_port = key0.l_port = inner_echo0->identifier;
474  break;
475  case SNAT_PROTOCOL_UDP:
476  case SNAT_PROTOCOL_TCP:
477  key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
478  key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
479  break;
480  default:
481  return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
482  }
483  }
484  *p_key0 = key0;
485  return 0;
486 }
487 
488 static inline int
490 {
491  nat_ed_ses_key_t key;
492  clib_bihash_kv_16_8_t kv, value;
493  udp_header_t *udp;
494 
495  if (!sm->forwarding_enabled)
496  return 0;
497 
498  if (ip->protocol == IP_PROTOCOL_ICMP)
499  {
500  if (icmp_get_ed_key (ip, &key))
501  return 0;
502  }
503  else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
504  {
505  udp = ip4_next_header(ip);
506  key.l_addr = ip->src_address;
507  key.r_addr = ip->dst_address;
508  key.proto = ip->protocol;
509  key.r_port = udp->dst_port;
510  key.l_port = udp->src_port;
511  }
512  else
513  {
514  key.l_addr = ip->src_address;
515  key.r_addr = ip->dst_address;
516  key.proto = ip->protocol;
517  key.l_port = key.r_port = 0;
518  }
519  key.fib_index = 0;
520  kv.key[0] = key.as_u64[0];
521  kv.key[1] = key.as_u64[1];
522 
523  if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
524  return value.value == ~0ULL;
525 
526  return 0;
527 }
528 
529 /**
530  * Get address and port values to be used for ICMP packet translation
531  * and create session if needed
532  *
533  * @param[in,out] sm NAT main
534  * @param[in,out] node NAT node runtime
535  * @param[in] thread_index thread index
536  * @param[in,out] b0 buffer containing packet to be translated
537  * @param[out] p_proto protocol used for matching
538  * @param[out] p_value address and port after NAT translation
539  * @param[out] p_dont_translate if packet should not be translated
540  * @param d optional parameter
541  * @param e optional parameter
542  */
544  u32 thread_index, vlib_buffer_t *b0,
545  ip4_header_t *ip0, u8 *p_proto,
546  snat_session_key_t *p_value,
547  u8 *p_dont_translate, void *d, void *e)
548 {
549  icmp46_header_t *icmp0;
550  u32 sw_if_index0;
551  u32 rx_fib_index0;
552  snat_session_key_t key0;
553  snat_session_t *s0 = 0;
554  u8 dont_translate = 0;
555  clib_bihash_kv_8_8_t kv0, value0;
556  u32 next0 = ~0;
557  int err;
558 
559  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
560  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
561  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
562 
563  err = icmp_get_key (ip0, &key0);
564  if (err != -1)
565  {
566  b0->error = node->errors[err];
567  next0 = SNAT_IN2OUT_NEXT_DROP;
568  goto out;
569  }
570  key0.fib_index = rx_fib_index0;
571 
572  kv0.key = key0.as_u64;
573 
574  if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
575  &value0))
576  {
577  if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
578  {
580  ip0, SNAT_PROTOCOL_ICMP, key0.port, key0.port, thread_index, sw_if_index0)))
581  {
582  dont_translate = 1;
583  goto out;
584  }
585  }
586  else
587  {
588  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
589  ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
590  {
591  dont_translate = 1;
592  goto out;
593  }
594  }
595 
597  {
598  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
599  next0 = SNAT_IN2OUT_NEXT_DROP;
600  goto out;
601  }
602 
603  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
604  &s0, node, next0, thread_index);
605 
606  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
607  goto out;
608  }
609  else
610  {
611  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
612  icmp0->type != ICMP4_echo_reply &&
613  !icmp_is_error_message (icmp0)))
614  {
615  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
616  next0 = SNAT_IN2OUT_NEXT_DROP;
617  goto out;
618  }
619 
620  if (PREDICT_FALSE (value0.value == ~0ULL))
621  {
622  nat_ed_ses_key_t key;
623  clib_bihash_kv_16_8_t s_kv, s_value;
624 
625  key.as_u64[0] = 0;
626  key.as_u64[1] = 0;
627  if (icmp_get_ed_key (ip0, &key))
628  {
629  b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
630  next0 = SNAT_IN2OUT_NEXT_DROP;
631  goto out;
632  }
633  key.fib_index = rx_fib_index0;
634  s_kv.key[0] = key.as_u64[0];
635  s_kv.key[1] = key.as_u64[1];
636  if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
637  s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
638  s_value.value);
639  else
640  {
641  next0 = SNAT_IN2OUT_NEXT_DROP;
642  goto out;
643  }
644  }
645  else
646  s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
647  value0.value);
648  }
649 
650 out:
651  *p_proto = key0.protocol;
652  if (s0)
653  *p_value = s0->out2in;
654  *p_dont_translate = dont_translate;
655  if (d)
656  *(snat_session_t**)d = s0;
657  return next0;
658 }
659 
660 /**
661  * Get address and port values to be used for ICMP packet translation
662  *
663  * @param[in] sm NAT main
664  * @param[in,out] node NAT node runtime
665  * @param[in] thread_index thread index
666  * @param[in,out] b0 buffer containing packet to be translated
667  * @param[out] p_proto protocol used for matching
668  * @param[out] p_value address and port after NAT translation
669  * @param[out] p_dont_translate if packet should not be translated
670  * @param d optional parameter
671  * @param e optional parameter
672  */
674  u32 thread_index, vlib_buffer_t *b0,
675  ip4_header_t *ip0, u8 *p_proto,
676  snat_session_key_t *p_value,
677  u8 *p_dont_translate, void *d, void *e)
678 {
679  icmp46_header_t *icmp0;
680  u32 sw_if_index0;
681  u32 rx_fib_index0;
682  snat_session_key_t key0;
683  snat_session_key_t sm0;
684  u8 dont_translate = 0;
685  u8 is_addr_only;
686  u32 next0 = ~0;
687  int err;
688 
689  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
690  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
691  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
692 
693  err = icmp_get_key (ip0, &key0);
694  if (err != -1)
695  {
696  b0->error = node->errors[err];
697  next0 = SNAT_IN2OUT_NEXT_DROP;
698  goto out2;
699  }
700  key0.fib_index = rx_fib_index0;
701 
702  if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0))
703  {
704  if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
705  IP_PROTOCOL_ICMP, rx_fib_index0)))
706  {
707  dont_translate = 1;
708  goto out;
709  }
710 
711  if (icmp_is_error_message (icmp0))
712  {
713  next0 = SNAT_IN2OUT_NEXT_DROP;
714  goto out;
715  }
716 
717  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
718  next0 = SNAT_IN2OUT_NEXT_DROP;
719  goto out;
720  }
721 
722  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
723  (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
724  !icmp_is_error_message (icmp0)))
725  {
726  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
727  next0 = SNAT_IN2OUT_NEXT_DROP;
728  goto out;
729  }
730 
731 out:
732  *p_value = sm0;
733 out2:
734  *p_proto = key0.protocol;
735  *p_dont_translate = dont_translate;
736  return next0;
737 }
738 
739 static inline u32 icmp_in2out (snat_main_t *sm,
740  vlib_buffer_t * b0,
741  ip4_header_t * ip0,
742  icmp46_header_t * icmp0,
743  u32 sw_if_index0,
744  u32 rx_fib_index0,
745  vlib_node_runtime_t * node,
746  u32 next0,
747  u32 thread_index,
748  void *d,
749  void *e)
750 {
751  snat_session_key_t sm0;
752  u8 protocol;
753  icmp_echo_header_t *echo0, *inner_echo0 = 0;
754  ip4_header_t *inner_ip0;
755  void *l4_header = 0;
756  icmp46_header_t *inner_icmp0;
757  u8 dont_translate;
758  u32 new_addr0, old_addr0;
759  u16 old_id0, new_id0;
760  ip_csum_t sum0;
761  u16 checksum0;
762  u32 next0_tmp;
763 
764  echo0 = (icmp_echo_header_t *)(icmp0+1);
765 
766  next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
767  &protocol, &sm0, &dont_translate, d, e);
768  if (next0_tmp != ~0)
769  next0 = next0_tmp;
770  if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
771  goto out;
772 
773  sum0 = ip_incremental_checksum (0, icmp0,
774  ntohs(ip0->length) - ip4_header_bytes (ip0));
775  checksum0 = ~ip_csum_fold (sum0);
776  if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
777  {
778  next0 = SNAT_IN2OUT_NEXT_DROP;
779  goto out;
780  }
781 
782  old_addr0 = ip0->src_address.as_u32;
783  new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
784  if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
785  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
786 
787  sum0 = ip0->checksum;
788  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
789  src_address /* changed member */);
790  ip0->checksum = ip_csum_fold (sum0);
791 
792  if (icmp0->checksum == 0)
793  icmp0->checksum = 0xffff;
794 
795  if (!icmp_is_error_message (icmp0))
796  {
797  new_id0 = sm0.port;
798  if (PREDICT_FALSE(new_id0 != echo0->identifier))
799  {
800  old_id0 = echo0->identifier;
801  new_id0 = sm0.port;
802  echo0->identifier = new_id0;
803 
804  sum0 = icmp0->checksum;
805  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
806  identifier);
807  icmp0->checksum = ip_csum_fold (sum0);
808  }
809  }
810  else
811  {
812  inner_ip0 = (ip4_header_t *)(echo0+1);
813  l4_header = ip4_next_header (inner_ip0);
814 
815  if (!ip4_header_checksum_is_valid (inner_ip0))
816  {
817  next0 = SNAT_IN2OUT_NEXT_DROP;
818  goto out;
819  }
820 
821  old_addr0 = inner_ip0->dst_address.as_u32;
822  inner_ip0->dst_address = sm0.addr;
823  new_addr0 = inner_ip0->dst_address.as_u32;
824 
825  sum0 = icmp0->checksum;
826  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
827  dst_address /* changed member */);
828  icmp0->checksum = ip_csum_fold (sum0);
829 
830  switch (protocol)
831  {
832  case SNAT_PROTOCOL_ICMP:
833  inner_icmp0 = (icmp46_header_t*)l4_header;
834  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
835 
836  old_id0 = inner_echo0->identifier;
837  new_id0 = sm0.port;
838  inner_echo0->identifier = new_id0;
839 
840  sum0 = icmp0->checksum;
841  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
842  identifier);
843  icmp0->checksum = ip_csum_fold (sum0);
844  break;
845  case SNAT_PROTOCOL_UDP:
846  case SNAT_PROTOCOL_TCP:
847  old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
848  new_id0 = sm0.port;
849  ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
850 
851  sum0 = icmp0->checksum;
852  sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
853  dst_port);
854  icmp0->checksum = ip_csum_fold (sum0);
855  break;
856  default:
857  ASSERT(0);
858  }
859  }
860 
861 out:
862  return next0;
863 }
864 
865 /**
866  * @brief Hairpinning
867  *
868  * Hairpinning allows two endpoints on the internal side of the NAT to
869  * communicate even if they only use each other's external IP addresses
870  * and ports.
871  *
872  * @param sm NAT main.
873  * @param b0 Vlib buffer.
874  * @param ip0 IP header.
875  * @param udp0 UDP header.
876  * @param tcp0 TCP header.
877  * @param proto0 NAT protocol.
878  */
879 static inline int
881  vlib_buffer_t * b0,
882  ip4_header_t * ip0,
883  udp_header_t * udp0,
884  tcp_header_t * tcp0,
885  u32 proto0)
886 {
887  snat_session_key_t key0, sm0;
888  snat_session_t * s0;
889  clib_bihash_kv_8_8_t kv0, value0;
890  ip_csum_t sum0;
891  u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
892  u16 new_dst_port0, old_dst_port0;
893 
894  key0.addr = ip0->dst_address;
895  key0.port = udp0->dst_port;
896  key0.protocol = proto0;
897  key0.fib_index = sm->outside_fib_index;
898  kv0.key = key0.as_u64;
899 
900  /* Check if destination is static mappings */
901  if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
902  {
903  new_dst_addr0 = sm0.addr.as_u32;
904  new_dst_port0 = sm0.port;
905  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
906  }
907  /* or active session */
908  else
909  {
910  if (sm->num_workers > 1)
911  ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
912  else
913  ti = sm->num_workers;
914 
915  if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
916  {
917  si = value0.value;
918 
919  s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
920  new_dst_addr0 = s0->in2out.addr.as_u32;
921  new_dst_port0 = s0->in2out.port;
922  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
923  }
924  }
925 
926  /* Destination is behind the same NAT, use internal address and port */
927  if (new_dst_addr0)
928  {
929  old_dst_addr0 = ip0->dst_address.as_u32;
930  ip0->dst_address.as_u32 = new_dst_addr0;
931  sum0 = ip0->checksum;
932  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
933  ip4_header_t, dst_address);
934  ip0->checksum = ip_csum_fold (sum0);
935 
936  old_dst_port0 = tcp0->dst;
937  if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
938  {
939  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
940  {
941  tcp0->dst = new_dst_port0;
942  sum0 = tcp0->checksum;
943  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
944  ip4_header_t, dst_address);
945  sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
946  ip4_header_t /* cheat */, length);
947  tcp0->checksum = ip_csum_fold(sum0);
948  }
949  else
950  {
951  udp0->dst_port = new_dst_port0;
952  udp0->checksum = 0;
953  }
954  }
955  else
956  {
957  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
958  {
959  sum0 = tcp0->checksum;
960  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
961  ip4_header_t, dst_address);
962  tcp0->checksum = ip_csum_fold(sum0);
963  }
964  }
965  return 1;
966  }
967  return 0;
968 }
969 
970 static inline void
972  vlib_buffer_t * b0,
973  ip4_header_t * ip0,
974  icmp46_header_t * icmp0)
975 {
976  snat_session_key_t key0, sm0;
977  clib_bihash_kv_8_8_t kv0, value0;
978  u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
979  ip_csum_t sum0;
980  snat_session_t *s0;
981 
982  if (!icmp_is_error_message (icmp0))
983  {
984  icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
985  u16 icmp_id0 = echo0->identifier;
986  key0.addr = ip0->dst_address;
987  key0.port = icmp_id0;
988  key0.protocol = SNAT_PROTOCOL_ICMP;
989  key0.fib_index = sm->outside_fib_index;
990  kv0.key = key0.as_u64;
991 
992  if (sm->num_workers > 1)
993  ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
994  else
995  ti = sm->num_workers;
996 
997  /* Check if destination is in active sessions */
998  if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
999  &value0))
1000  {
1001  /* or static mappings */
1002  if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1003  {
1004  new_dst_addr0 = sm0.addr.as_u32;
1005  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1006  }
1007  }
1008  else
1009  {
1010  si = value0.value;
1011 
1012  s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1013  new_dst_addr0 = s0->in2out.addr.as_u32;
1014  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1015  echo0->identifier = s0->in2out.port;
1016  sum0 = icmp0->checksum;
1017  sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1018  icmp_echo_header_t, identifier);
1019  icmp0->checksum = ip_csum_fold (sum0);
1020  }
1021 
1022  /* Destination is behind the same NAT, use internal address and port */
1023  if (new_dst_addr0)
1024  {
1025  old_dst_addr0 = ip0->dst_address.as_u32;
1026  ip0->dst_address.as_u32 = new_dst_addr0;
1027  sum0 = ip0->checksum;
1028  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1029  ip4_header_t, dst_address);
1030  ip0->checksum = ip_csum_fold (sum0);
1031  }
1032  }
1033 
1034 }
1035 
1037  vlib_buffer_t * b0,
1038  ip4_header_t * ip0,
1039  icmp46_header_t * icmp0,
1040  u32 sw_if_index0,
1041  u32 rx_fib_index0,
1042  vlib_node_runtime_t * node,
1043  u32 next0,
1044  f64 now,
1045  u32 thread_index,
1046  snat_session_t ** p_s0)
1047 {
1048  next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1049  next0, thread_index, p_s0, 0);
1050  snat_session_t * s0 = *p_s0;
1051  if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1052  {
1053  /* Hairpinning */
1054  if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1055  snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1056  /* Accounting */
1057  s0->last_heard = now;
1058  s0->total_pkts++;
1059  s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1060  /* Per-user LRU list maintenance */
1061  clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1062  s0->per_user_index);
1063  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1064  s0->per_user_list_head_index,
1065  s0->per_user_index);
1066  }
1067  return next0;
1068 }
1069 static inline void
1071  vlib_buffer_t * b,
1072  ip4_header_t * ip)
1073 {
1074  u32 old_addr, new_addr = 0, ti = 0;
1075  clib_bihash_kv_8_8_t kv, value;
1076  clib_bihash_kv_16_8_t s_kv, s_value;
1077  nat_ed_ses_key_t key;
1078  snat_session_key_t m_key;
1080  ip_csum_t sum;
1081  snat_session_t *s;
1082 
1083  old_addr = ip->dst_address.as_u32;
1084  key.l_addr.as_u32 = ip->dst_address.as_u32;
1085  key.r_addr.as_u32 = ip->src_address.as_u32;
1086  key.fib_index = sm->outside_fib_index;
1087  key.proto = ip->protocol;
1088  key.r_port = 0;
1089  key.l_port = 0;
1090  s_kv.key[0] = key.as_u64[0];
1091  s_kv.key[1] = key.as_u64[1];
1092  if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1093  {
1094  m_key.addr = ip->dst_address;
1095  m_key.fib_index = sm->outside_fib_index;
1096  m_key.port = 0;
1097  m_key.protocol = 0;
1098  kv.key = m_key.as_u64;
1099  if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1100  return;
1101 
1102  m = pool_elt_at_index (sm->static_mappings, value.value);
1103  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1104  vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1105  new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1106  }
1107  else
1108  {
1109  if (sm->num_workers > 1)
1110  ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1111  else
1112  ti = sm->num_workers;
1113 
1114  s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1115  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1116  vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1117  new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1118  }
1119  sum = ip->checksum;
1120  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1121  ip->checksum = ip_csum_fold (sum);
1122 }
1123 
1124 static snat_session_t *
1126  vlib_buffer_t * b,
1127  ip4_header_t * ip,
1128  u32 rx_fib_index,
1129  u32 thread_index,
1130  f64 now,
1131  vlib_main_t * vm,
1132  vlib_node_runtime_t * node)
1133 {
1134  clib_bihash_kv_8_8_t kv, value;
1135  clib_bihash_kv_16_8_t s_kv, s_value;
1137  snat_session_key_t m_key;
1138  u32 old_addr, new_addr = 0;
1139  ip_csum_t sum;
1140  snat_user_t *u;
1141  dlist_elt_t *head, *elt;
1142  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1143  u32 elt_index, head_index, ses_index;
1144  snat_session_t * s;
1145  nat_ed_ses_key_t key;
1146  u32 address_index = ~0;
1147  int i;
1148  u8 is_sm = 0;
1149 
1150  old_addr = ip->src_address.as_u32;
1151 
1152  key.l_addr = ip->src_address;
1153  key.r_addr = ip->dst_address;
1154  key.fib_index = rx_fib_index;
1155  key.proto = ip->protocol;
1156  key.l_port = 0;
1157  key.r_port = 0;
1158  s_kv.key[0] = key.as_u64[0];
1159  s_kv.key[1] = key.as_u64[1];
1160 
1161  if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1162  {
1163  s = pool_elt_at_index (tsm->sessions, s_value.value);
1164  new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1165  }
1166  else
1167  {
1168  if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1169  {
1170  b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1171  return 0;
1172  }
1173 
1174  u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1175  thread_index);
1176  if (!u)
1177  {
1178  clib_warning ("create NAT user failed");
1179  return 0;
1180  }
1181 
1182  m_key.addr = ip->src_address;
1183  m_key.port = 0;
1184  m_key.protocol = 0;
1185  m_key.fib_index = rx_fib_index;
1186  kv.key = m_key.as_u64;
1187 
1188  /* Try to find static mapping first */
1189  if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1190  {
1191  m = pool_elt_at_index (sm->static_mappings, value.value);
1192  new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1193  is_sm = 1;
1194  goto create_ses;
1195  }
1196  /* Fallback to 3-tuple key */
1197  else
1198  {
1199  /* Choose same out address as for TCP/UDP session to same destination */
1200  if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1201  {
1202  head_index = u->sessions_per_user_list_head_index;
1203  head = pool_elt_at_index (tsm->list_pool, head_index);
1204  elt_index = head->next;
1205  elt = pool_elt_at_index (tsm->list_pool, elt_index);
1206  ses_index = elt->value;
1207  while (ses_index != ~0)
1208  {
1209  s = pool_elt_at_index (tsm->sessions, ses_index);
1210  elt_index = elt->next;
1211  elt = pool_elt_at_index (tsm->list_pool, elt_index);
1212  ses_index = elt->value;
1213 
1214  if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1215  {
1216  new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1217  address_index = s->outside_address_index;
1218 
1219  key.fib_index = sm->outside_fib_index;
1220  key.l_addr.as_u32 = new_addr;
1221  s_kv.key[0] = key.as_u64[0];
1222  s_kv.key[1] = key.as_u64[1];
1223  if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1224  break;
1225 
1226  goto create_ses;
1227  }
1228  }
1229  }
1230  key.fib_index = sm->outside_fib_index;
1231  for (i = 0; i < vec_len (sm->addresses); i++)
1232  {
1233  key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1234  s_kv.key[0] = key.as_u64[0];
1235  s_kv.key[1] = key.as_u64[1];
1236  if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1237  {
1238  new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1239  address_index = i;
1240  goto create_ses;
1241  }
1242  }
1243  return 0;
1244  }
1245 
1246 create_ses:
1247  s = nat_session_alloc_or_recycle (sm, u, thread_index);
1248  if (!s)
1249  {
1250  clib_warning ("create NAT session failed");
1251  return 0;
1252  }
1253 
1254  s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1255  s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1256  s->outside_address_index = address_index;
1257  s->out2in.addr.as_u32 = new_addr;
1258  s->out2in.fib_index = sm->outside_fib_index;
1259  s->in2out.addr.as_u32 = old_addr;
1260  s->in2out.fib_index = rx_fib_index;
1261  s->in2out.port = s->out2in.port = ip->protocol;
1262  if (is_sm)
1264  user_session_increment (sm, u, is_sm);
1265 
1266  /* Add to lookup tables */
1267  key.l_addr.as_u32 = old_addr;
1268  key.r_addr = ip->dst_address;
1269  key.proto = ip->protocol;
1270  key.fib_index = rx_fib_index;
1271  s_kv.key[0] = key.as_u64[0];
1272  s_kv.key[1] = key.as_u64[1];
1273  s_kv.value = s - tsm->sessions;
1274  if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1275  clib_warning ("in2out key add failed");
1276 
1277  key.l_addr.as_u32 = new_addr;
1278  key.fib_index = sm->outside_fib_index;
1279  s_kv.key[0] = key.as_u64[0];
1280  s_kv.key[1] = key.as_u64[1];
1281  if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1282  clib_warning ("out2in key add failed");
1283  }
1284 
1285  /* Update IP checksum */
1286  sum = ip->checksum;
1287  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1288  ip->checksum = ip_csum_fold (sum);
1289 
1290  /* Accounting */
1291  s->last_heard = now;
1292  s->total_pkts++;
1293  s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1294  /* Per-user LRU list maintenance */
1295  clib_dlist_remove (tsm->list_pool, s->per_user_index);
1296  clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1297  s->per_user_index);
1298 
1299  /* Hairpinning */
1300  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1301  snat_hairpinning_unknown_proto(sm, b, ip);
1302 
1303  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1304  vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1305 
1306  return s;
1307 }
1308 
1309 static snat_session_t *
1311  vlib_buffer_t * b,
1312  ip4_header_t * ip,
1313  u32 rx_fib_index,
1314  u32 thread_index,
1315  f64 now,
1316  vlib_main_t * vm,
1317  vlib_node_runtime_t * node)
1318 {
1319  nat_ed_ses_key_t key;
1320  clib_bihash_kv_16_8_t s_kv, s_value;
1321  udp_header_t *udp = ip4_next_header (ip);
1322  tcp_header_t *tcp = (tcp_header_t *) udp;
1323  snat_session_t *s = 0;
1324  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1325  u32 old_addr, new_addr;
1326  u16 new_port, old_port;
1327  ip_csum_t sum;
1328  u32 proto = ip_proto_to_snat_proto (ip->protocol);
1329  snat_session_key_t e_key, l_key;
1330  snat_user_t *u;
1331 
1332  old_addr = ip->src_address.as_u32;
1333 
1334  key.l_addr = ip->src_address;
1335  key.r_addr = ip->dst_address;
1336  key.fib_index = rx_fib_index;
1337  key.proto = ip->protocol;
1338  key.r_port = udp->dst_port;
1339  key.l_port = udp->src_port;
1340  s_kv.key[0] = key.as_u64[0];
1341  s_kv.key[1] = key.as_u64[1];
1342 
1343  if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1344  {
1345  if (s_value.value == ~0ULL)
1346  return 0;
1347  s = pool_elt_at_index (tsm->sessions, s_value.value);
1348  }
1349  else
1350  {
1351  if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1352  {
1353  b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1354  return 0;
1355  }
1356 
1357  l_key.addr = ip->src_address;
1358  l_key.port = udp->src_port;
1359  l_key.protocol = proto;
1360  l_key.fib_index = rx_fib_index;
1361  if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0))
1362  return 0;
1363 
1364  u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1365  thread_index);
1366  if (!u)
1367  {
1368  clib_warning ("create NAT user failed");
1369  return 0;
1370  }
1371 
1372  s = nat_session_alloc_or_recycle (sm, u, thread_index);
1373  if (!s)
1374  {
1375  clib_warning ("create NAT session failed");
1376  return 0;
1377  }
1378 
1379  s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1382  s->outside_address_index = ~0;
1383  s->in2out = l_key;
1384  s->out2in = e_key;
1385  s->out2in.protocol = l_key.protocol;
1386  user_session_increment (sm, u, 1 /* static */);
1387 
1388  /* Add to lookup tables */
1389  s_kv.value = s - tsm->sessions;
1390  if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1391  clib_warning ("in2out-ed key add failed");
1392 
1393  key.l_addr = e_key.addr;
1394  key.fib_index = e_key.fib_index;
1395  key.l_port = e_key.port;
1396  s_kv.key[0] = key.as_u64[0];
1397  s_kv.key[1] = key.as_u64[1];
1398  if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1399  clib_warning ("out2in-ed key add failed");
1400  }
1401 
1402  new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1403 
1404  /* Update IP checksum */
1405  sum = ip->checksum;
1406  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1407  if (is_twice_nat_session (s))
1408  sum = ip_csum_update (sum, ip->dst_address.as_u32,
1409  s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1410  ip->checksum = ip_csum_fold (sum);
1411 
1412  if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1413  {
1414  old_port = tcp->src_port;
1415  tcp->src_port = s->out2in.port;
1416  new_port = tcp->src_port;
1417 
1418  sum = tcp->checksum;
1419  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1420  sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1421  if (is_twice_nat_session (s))
1422  {
1423  sum = ip_csum_update (sum, ip->dst_address.as_u32,
1424  s->ext_host_addr.as_u32, ip4_header_t,
1425  dst_address);
1426  sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1427  ip4_header_t, length);
1428  tcp->dst_port = s->ext_host_port;
1429  ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1430  }
1431  tcp->checksum = ip_csum_fold(sum);
1432  }
1433  else
1434  {
1435  udp->src_port = s->out2in.port;
1436  if (is_twice_nat_session (s))
1437  {
1438  udp->dst_port = s->ext_host_port;
1439  ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1440  }
1441  udp->checksum = 0;
1442  }
1443 
1444  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1445  vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1446 
1447  /* Accounting */
1448  s->last_heard = now;
1449  s->total_pkts++;
1450  s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1451  /* Per-user LRU list maintenance */
1452  clib_dlist_remove (tsm->list_pool, s->per_user_index);
1453  clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1454  s->per_user_index);
1455  return s;
1456 }
1457 
1458 static inline uword
1460  vlib_node_runtime_t * node,
1461  vlib_frame_t * frame, int is_slow_path,
1462  int is_output_feature)
1463 {
1464  u32 n_left_from, * from, * to_next;
1465  snat_in2out_next_t next_index;
1466  u32 pkts_processed = 0;
1467  snat_main_t * sm = &snat_main;
1468  f64 now = vlib_time_now (vm);
1469  u32 stats_node_index;
1470  u32 thread_index = vlib_get_thread_index ();
1471 
1472  stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1473  snat_in2out_node.index;
1474 
1475  from = vlib_frame_vector_args (frame);
1476  n_left_from = frame->n_vectors;
1477  next_index = node->cached_next_index;
1478 
1479  while (n_left_from > 0)
1480  {
1481  u32 n_left_to_next;
1482 
1483  vlib_get_next_frame (vm, node, next_index,
1484  to_next, n_left_to_next);
1485 
1486  while (n_left_from >= 4 && n_left_to_next >= 2)
1487  {
1488  u32 bi0, bi1;
1489  vlib_buffer_t * b0, * b1;
1490  u32 next0, next1;
1491  u32 sw_if_index0, sw_if_index1;
1492  ip4_header_t * ip0, * ip1;
1493  ip_csum_t sum0, sum1;
1494  u32 new_addr0, old_addr0, new_addr1, old_addr1;
1495  u16 old_port0, new_port0, old_port1, new_port1;
1496  udp_header_t * udp0, * udp1;
1497  tcp_header_t * tcp0, * tcp1;
1498  icmp46_header_t * icmp0, * icmp1;
1499  snat_session_key_t key0, key1;
1500  u32 rx_fib_index0, rx_fib_index1;
1501  u32 proto0, proto1;
1502  snat_session_t * s0 = 0, * s1 = 0;
1503  clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1504  u32 iph_offset0 = 0, iph_offset1 = 0;
1505 
1506  /* Prefetch next iteration. */
1507  {
1508  vlib_buffer_t * p2, * p3;
1509 
1510  p2 = vlib_get_buffer (vm, from[2]);
1511  p3 = vlib_get_buffer (vm, from[3]);
1512 
1513  vlib_prefetch_buffer_header (p2, LOAD);
1514  vlib_prefetch_buffer_header (p3, LOAD);
1515 
1518  }
1519 
1520  /* speculatively enqueue b0 and b1 to the current next frame */
1521  to_next[0] = bi0 = from[0];
1522  to_next[1] = bi1 = from[1];
1523  from += 2;
1524  to_next += 2;
1525  n_left_from -= 2;
1526  n_left_to_next -= 2;
1527 
1528  b0 = vlib_get_buffer (vm, bi0);
1529  b1 = vlib_get_buffer (vm, bi1);
1530 
1531  if (is_output_feature)
1532  iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1533 
1534  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1535  iph_offset0);
1536 
1537  udp0 = ip4_next_header (ip0);
1538  tcp0 = (tcp_header_t *) udp0;
1539  icmp0 = (icmp46_header_t *) udp0;
1540 
1541  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1542  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1543  sw_if_index0);
1544 
1545  next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1546 
1547  if (PREDICT_FALSE(ip0->ttl == 1))
1548  {
1549  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1550  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1551  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1552  0);
1554  goto trace00;
1555  }
1556 
1557  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1558 
1559  /* Next configured feature, probably ip4-lookup */
1560  if (is_slow_path)
1561  {
1562  if (PREDICT_FALSE (proto0 == ~0))
1563  {
1564  s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1565  thread_index, now, vm, node);
1566  if (!s0)
1567  next0 = SNAT_IN2OUT_NEXT_DROP;
1568  goto trace00;
1569  }
1570 
1571  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1572  {
1573  next0 = icmp_in2out_slow_path
1574  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1575  node, next0, now, thread_index, &s0);
1576  goto trace00;
1577  }
1578  }
1579  else
1580  {
1581  if (is_output_feature)
1582  {
1584  goto trace00;
1585  }
1586 
1587  if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1588  {
1590  goto trace00;
1591  }
1592 
1593  if (ip4_is_fragment (ip0))
1594  {
1595  next0 = SNAT_IN2OUT_NEXT_REASS;
1596  goto trace00;
1597  }
1598  }
1599 
1600  key0.addr = ip0->src_address;
1601  key0.port = udp0->src_port;
1602  key0.protocol = proto0;
1603  key0.fib_index = rx_fib_index0;
1604 
1605  kv0.key = key0.as_u64;
1606 
1607  if (PREDICT_FALSE (clib_bihash_search_8_8 (
1608  &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1609  {
1610  if (is_slow_path)
1611  {
1612  if (is_output_feature)
1613  {
1615  ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1616  goto trace00;
1617  }
1618  else
1619  {
1620  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1621  ip0, proto0, rx_fib_index0, thread_index)))
1622  goto trace00;
1623  }
1624 
1625  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1626  &s0, node, next0, thread_index);
1627  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1628  goto trace00;
1629  }
1630  else
1631  {
1633  goto trace00;
1634  }
1635  }
1636  else
1637  {
1638  if (PREDICT_FALSE (value0.value == ~0ULL))
1639  {
1640  if (is_slow_path)
1641  {
1642  s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1643  thread_index, now, vm, node);
1644  if (!s0)
1645  next0 = SNAT_IN2OUT_NEXT_DROP;
1646  goto trace00;
1647  }
1648  else
1649  {
1651  goto trace00;
1652  }
1653  }
1654  else
1655  {
1656  s0 = pool_elt_at_index (
1657  sm->per_thread_data[thread_index].sessions,
1658  value0.value);
1659  }
1660  }
1661 
1662  b0->flags |= VNET_BUFFER_F_IS_NATED;
1663 
1664  old_addr0 = ip0->src_address.as_u32;
1665  ip0->src_address = s0->out2in.addr;
1666  new_addr0 = ip0->src_address.as_u32;
1667  if (!is_output_feature)
1668  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1669 
1670  sum0 = ip0->checksum;
1671  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1672  ip4_header_t,
1673  src_address /* changed member */);
1674  ip0->checksum = ip_csum_fold (sum0);
1675 
1676  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1677  {
1678  old_port0 = tcp0->src_port;
1679  tcp0->src_port = s0->out2in.port;
1680  new_port0 = tcp0->src_port;
1681 
1682  sum0 = tcp0->checksum;
1683  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1684  ip4_header_t,
1685  dst_address /* changed member */);
1686  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1687  ip4_header_t /* cheat */,
1688  length /* changed member */);
1689  tcp0->checksum = ip_csum_fold(sum0);
1690  }
1691  else
1692  {
1693  old_port0 = udp0->src_port;
1694  udp0->src_port = s0->out2in.port;
1695  udp0->checksum = 0;
1696  }
1697 
1698  /* Accounting */
1699  s0->last_heard = now;
1700  s0->total_pkts++;
1701  s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1702  /* Per-user LRU list maintenance */
1703  clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1704  s0->per_user_index);
1705  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1706  s0->per_user_list_head_index,
1707  s0->per_user_index);
1708  trace00:
1709 
1711  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1712  {
1713  snat_in2out_trace_t *t =
1714  vlib_add_trace (vm, node, b0, sizeof (*t));
1715  t->is_slow_path = is_slow_path;
1716  t->sw_if_index = sw_if_index0;
1717  t->next_index = next0;
1718  t->session_index = ~0;
1719  if (s0)
1720  t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1721  }
1722 
1723  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1724 
1725  if (is_output_feature)
1726  iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1727 
1728  ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1729  iph_offset1);
1730 
1731  udp1 = ip4_next_header (ip1);
1732  tcp1 = (tcp_header_t *) udp1;
1733  icmp1 = (icmp46_header_t *) udp1;
1734 
1735  sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1736  rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1737  sw_if_index1);
1738 
1739  if (PREDICT_FALSE(ip1->ttl == 1))
1740  {
1741  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1742  icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1743  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1744  0);
1746  goto trace01;
1747  }
1748 
1749  proto1 = ip_proto_to_snat_proto (ip1->protocol);
1750 
1751  /* Next configured feature, probably ip4-lookup */
1752  if (is_slow_path)
1753  {
1754  if (PREDICT_FALSE (proto1 == ~0))
1755  {
1756  s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1757  thread_index, now, vm, node);
1758  if (!s1)
1759  next1 = SNAT_IN2OUT_NEXT_DROP;
1760  goto trace01;
1761  }
1762 
1763  if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1764  {
1765  next1 = icmp_in2out_slow_path
1766  (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1767  next1, now, thread_index, &s1);
1768  goto trace01;
1769  }
1770  }
1771  else
1772  {
1773  if (is_output_feature)
1774  {
1776  goto trace01;
1777  }
1778 
1779  if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1780  {
1782  goto trace01;
1783  }
1784 
1785  if (ip4_is_fragment (ip1))
1786  {
1787  next1 = SNAT_IN2OUT_NEXT_REASS;
1788  goto trace01;
1789  }
1790  }
1791 
1792  key1.addr = ip1->src_address;
1793  key1.port = udp1->src_port;
1794  key1.protocol = proto1;
1795  key1.fib_index = rx_fib_index1;
1796 
1797  kv1.key = key1.as_u64;
1798 
1799  if (PREDICT_FALSE(clib_bihash_search_8_8 (
1800  &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1801  {
1802  if (is_slow_path)
1803  {
1804  if (is_output_feature)
1805  {
1807  ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1808  goto trace01;
1809  }
1810  else
1811  {
1812  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1813  ip1, proto1, rx_fib_index1, thread_index)))
1814  goto trace01;
1815  }
1816 
1817  next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1818  &s1, node, next1, thread_index);
1819  if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1820  goto trace01;
1821  }
1822  else
1823  {
1825  goto trace01;
1826  }
1827  }
1828  else
1829  {
1830  if (PREDICT_FALSE (value1.value == ~0ULL))
1831  {
1832  if (is_slow_path)
1833  {
1834  s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1835  thread_index, now, vm, node);
1836  if (!s1)
1837  next1 = SNAT_IN2OUT_NEXT_DROP;
1838  goto trace01;
1839  }
1840  else
1841  {
1843  goto trace01;
1844  }
1845  }
1846  else
1847  {
1848  s1 = pool_elt_at_index (
1849  sm->per_thread_data[thread_index].sessions,
1850  value1.value);
1851  }
1852  }
1853 
1854  b1->flags |= VNET_BUFFER_F_IS_NATED;
1855 
1856  old_addr1 = ip1->src_address.as_u32;
1857  ip1->src_address = s1->out2in.addr;
1858  new_addr1 = ip1->src_address.as_u32;
1859  if (!is_output_feature)
1860  vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1861 
1862  sum1 = ip1->checksum;
1863  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1864  ip4_header_t,
1865  src_address /* changed member */);
1866  ip1->checksum = ip_csum_fold (sum1);
1867 
1868  if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1869  {
1870  old_port1 = tcp1->src_port;
1871  tcp1->src_port = s1->out2in.port;
1872  new_port1 = tcp1->src_port;
1873 
1874  sum1 = tcp1->checksum;
1875  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1876  ip4_header_t,
1877  dst_address /* changed member */);
1878  sum1 = ip_csum_update (sum1, old_port1, new_port1,
1879  ip4_header_t /* cheat */,
1880  length /* changed member */);
1881  tcp1->checksum = ip_csum_fold(sum1);
1882  }
1883  else
1884  {
1885  old_port1 = udp1->src_port;
1886  udp1->src_port = s1->out2in.port;
1887  udp1->checksum = 0;
1888  }
1889 
1890  /* Accounting */
1891  s1->last_heard = now;
1892  s1->total_pkts++;
1893  s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1894  /* Per-user LRU list maintenance */
1895  clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1896  s1->per_user_index);
1897  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1898  s1->per_user_list_head_index,
1899  s1->per_user_index);
1900  trace01:
1901 
1903  && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1904  {
1905  snat_in2out_trace_t *t =
1906  vlib_add_trace (vm, node, b1, sizeof (*t));
1907  t->sw_if_index = sw_if_index1;
1908  t->next_index = next1;
1909  t->session_index = ~0;
1910  if (s1)
1911  t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1912  }
1913 
1914  pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1915 
1916  /* verify speculative enqueues, maybe switch current next frame */
1917  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1918  to_next, n_left_to_next,
1919  bi0, bi1, next0, next1);
1920  }
1921 
1922  while (n_left_from > 0 && n_left_to_next > 0)
1923  {
1924  u32 bi0;
1925  vlib_buffer_t * b0;
1926  u32 next0;
1927  u32 sw_if_index0;
1928  ip4_header_t * ip0;
1929  ip_csum_t sum0;
1930  u32 new_addr0, old_addr0;
1931  u16 old_port0, new_port0;
1932  udp_header_t * udp0;
1933  tcp_header_t * tcp0;
1934  icmp46_header_t * icmp0;
1935  snat_session_key_t key0;
1936  u32 rx_fib_index0;
1937  u32 proto0;
1938  snat_session_t * s0 = 0;
1939  clib_bihash_kv_8_8_t kv0, value0;
1940  u32 iph_offset0 = 0;
1941 
1942  /* speculatively enqueue b0 to the current next frame */
1943  bi0 = from[0];
1944  to_next[0] = bi0;
1945  from += 1;
1946  to_next += 1;
1947  n_left_from -= 1;
1948  n_left_to_next -= 1;
1949 
1950  b0 = vlib_get_buffer (vm, bi0);
1951  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1952 
1953  if (is_output_feature)
1954  iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1955 
1956  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1957  iph_offset0);
1958 
1959  udp0 = ip4_next_header (ip0);
1960  tcp0 = (tcp_header_t *) udp0;
1961  icmp0 = (icmp46_header_t *) udp0;
1962 
1963  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1964  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1965  sw_if_index0);
1966 
1967  if (PREDICT_FALSE(ip0->ttl == 1))
1968  {
1969  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1970  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1971  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1972  0);
1974  goto trace0;
1975  }
1976 
1977  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1978 
1979  /* Next configured feature, probably ip4-lookup */
1980  if (is_slow_path)
1981  {
1982  if (PREDICT_FALSE (proto0 == ~0))
1983  {
1984  s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1985  thread_index, now, vm, node);
1986  if (!s0)
1987  next0 = SNAT_IN2OUT_NEXT_DROP;
1988  goto trace0;
1989  }
1990 
1991  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1992  {
1993  next0 = icmp_in2out_slow_path
1994  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1995  next0, now, thread_index, &s0);
1996  goto trace0;
1997  }
1998  }
1999  else
2000  {
2001  if (is_output_feature)
2002  {
2004  goto trace0;
2005  }
2006 
2007  if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2008  {
2010  goto trace0;
2011  }
2012 
2013  if (ip4_is_fragment (ip0))
2014  {
2015  next0 = SNAT_IN2OUT_NEXT_REASS;
2016  goto trace0;
2017  }
2018  }
2019 
2020  key0.addr = ip0->src_address;
2021  key0.port = udp0->src_port;
2022  key0.protocol = proto0;
2023  key0.fib_index = rx_fib_index0;
2024 
2025  kv0.key = key0.as_u64;
2026 
2027  if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2028  &kv0, &value0))
2029  {
2030  if (is_slow_path)
2031  {
2032  if (is_output_feature)
2033  {
2035  ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
2036  goto trace0;
2037  }
2038  else
2039  {
2040  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2041  ip0, proto0, rx_fib_index0, thread_index)))
2042  goto trace0;
2043  }
2044 
2045  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2046  &s0, node, next0, thread_index);
2047 
2048  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2049  goto trace0;
2050  }
2051  else
2052  {
2054  goto trace0;
2055  }
2056  }
2057  else
2058  {
2059  if (PREDICT_FALSE (value0.value == ~0ULL))
2060  {
2061  if (is_slow_path)
2062  {
2063  s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2064  thread_index, now, vm, node);
2065  if (!s0)
2066  next0 = SNAT_IN2OUT_NEXT_DROP;
2067  goto trace0;
2068  }
2069  else
2070  {
2072  goto trace0;
2073  }
2074  }
2075  else
2076  {
2077  s0 = pool_elt_at_index (
2078  sm->per_thread_data[thread_index].sessions,
2079  value0.value);
2080  }
2081  }
2082 
2083  b0->flags |= VNET_BUFFER_F_IS_NATED;
2084 
2085  old_addr0 = ip0->src_address.as_u32;
2086  ip0->src_address = s0->out2in.addr;
2087  new_addr0 = ip0->src_address.as_u32;
2088  if (!is_output_feature)
2089  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2090 
2091  sum0 = ip0->checksum;
2092  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2093  ip4_header_t,
2094  src_address /* changed member */);
2095  ip0->checksum = ip_csum_fold (sum0);
2096 
2097  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2098  {
2099  old_port0 = tcp0->src_port;
2100  tcp0->src_port = s0->out2in.port;
2101  new_port0 = tcp0->src_port;
2102 
2103  sum0 = tcp0->checksum;
2104  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2105  ip4_header_t,
2106  dst_address /* changed member */);
2107  sum0 = ip_csum_update (sum0, old_port0, new_port0,
2108  ip4_header_t /* cheat */,
2109  length /* changed member */);
2110  tcp0->checksum = ip_csum_fold(sum0);
2111  }
2112  else
2113  {
2114  old_port0 = udp0->src_port;
2115  udp0->src_port = s0->out2in.port;
2116  udp0->checksum = 0;
2117  }
2118 
2119  /* Accounting */
2120  s0->last_heard = now;
2121  s0->total_pkts++;
2122  s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2123  /* Per-user LRU list maintenance */
2124  clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2125  s0->per_user_index);
2126  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2127  s0->per_user_list_head_index,
2128  s0->per_user_index);
2129 
2130  trace0:
2132  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2133  {
2134  snat_in2out_trace_t *t =
2135  vlib_add_trace (vm, node, b0, sizeof (*t));
2136  t->is_slow_path = is_slow_path;
2137  t->sw_if_index = sw_if_index0;
2138  t->next_index = next0;
2139  t->session_index = ~0;
2140  if (s0)
2141  t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2142  }
2143 
2144  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2145 
2146  /* verify speculative enqueue, maybe switch current next frame */
2147  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2148  to_next, n_left_to_next,
2149  bi0, next0);
2150  }
2151 
2152  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2153  }
2154 
2155  vlib_node_increment_counter (vm, stats_node_index,
2156  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2157  pkts_processed);
2158  return frame->n_vectors;
2159 }
2160 
2161 static uword
2163  vlib_node_runtime_t * node,
2164  vlib_frame_t * frame)
2165 {
2166  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2167 }
2168 
2170  .function = snat_in2out_fast_path_fn,
2171  .name = "nat44-in2out",
2172  .vector_size = sizeof (u32),
2173  .format_trace = format_snat_in2out_trace,
2174  .type = VLIB_NODE_TYPE_INTERNAL,
2175 
2176  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2177  .error_strings = snat_in2out_error_strings,
2178 
2179  .runtime_data_bytes = sizeof (snat_runtime_t),
2180 
2181  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2182 
2183  /* edit / add dispositions here */
2184  .next_nodes = {
2185  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2186  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2187  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2188  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2189  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2190  },
2191 };
2192 
2194 
2195 static uword
2197  vlib_node_runtime_t * node,
2198  vlib_frame_t * frame)
2199 {
2200  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2201 }
2202 
2204  .function = snat_in2out_output_fast_path_fn,
2205  .name = "nat44-in2out-output",
2206  .vector_size = sizeof (u32),
2207  .format_trace = format_snat_in2out_trace,
2208  .type = VLIB_NODE_TYPE_INTERNAL,
2209 
2210  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2211  .error_strings = snat_in2out_error_strings,
2212 
2213  .runtime_data_bytes = sizeof (snat_runtime_t),
2214 
2215  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2216 
2217  /* edit / add dispositions here */
2218  .next_nodes = {
2219  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2220  [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2221  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2222  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2223  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2224  },
2225 };
2226 
2229 
2230 static uword
2232  vlib_node_runtime_t * node,
2233  vlib_frame_t * frame)
2234 {
2235  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2236 }
2237 
2239  .function = snat_in2out_slow_path_fn,
2240  .name = "nat44-in2out-slowpath",
2241  .vector_size = sizeof (u32),
2242  .format_trace = format_snat_in2out_trace,
2243  .type = VLIB_NODE_TYPE_INTERNAL,
2244 
2245  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2246  .error_strings = snat_in2out_error_strings,
2247 
2248  .runtime_data_bytes = sizeof (snat_runtime_t),
2249 
2250  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2251 
2252  /* edit / add dispositions here */
2253  .next_nodes = {
2254  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2255  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2256  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2257  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2258  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2259  },
2260 };
2261 
2264 
2265 static uword
2267  vlib_node_runtime_t * node,
2268  vlib_frame_t * frame)
2269 {
2270  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2271 }
2272 
2274  .function = snat_in2out_output_slow_path_fn,
2275  .name = "nat44-in2out-output-slowpath",
2276  .vector_size = sizeof (u32),
2277  .format_trace = format_snat_in2out_trace,
2278  .type = VLIB_NODE_TYPE_INTERNAL,
2279 
2280  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2281  .error_strings = snat_in2out_error_strings,
2282 
2283  .runtime_data_bytes = sizeof (snat_runtime_t),
2284 
2285  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2286 
2287  /* edit / add dispositions here */
2288  .next_nodes = {
2289  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2290  [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2291  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2292  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2293  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2294  },
2295 };
2296 
2299 
2301 
2302 static uword
2304  vlib_node_runtime_t * node,
2305  vlib_frame_t * frame)
2306 {
2307  u32 n_left_from, * from, * to_next;
2308  snat_in2out_next_t next_index;
2309  u32 pkts_processed = 0;
2310  snat_main_t * sm = &snat_main;
2312  u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2313  vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2314 
2315  from = vlib_frame_vector_args (frame);
2316  n_left_from = frame->n_vectors;
2317  next_index = node->cached_next_index;
2318 
2319  while (n_left_from > 0)
2320  {
2321  u32 n_left_to_next;
2322 
2323  vlib_get_next_frame (vm, node, next_index,
2324  to_next, n_left_to_next);
2325 
2326  while (n_left_from > 0 && n_left_to_next > 0)
2327  {
2328  u32 bi0;
2329  vlib_buffer_t * b0;
2330  u32 next0;
2331  ip4_header_t * ip0;
2332  u32 proto0;
2333  udp_header_t * udp0;
2334  tcp_header_t * tcp0;
2335 
2336  /* speculatively enqueue b0 to the current next frame */
2337  bi0 = from[0];
2338  to_next[0] = bi0;
2339  from += 1;
2340  to_next += 1;
2341  n_left_from -= 1;
2342  n_left_to_next -= 1;
2343 
2344  b0 = vlib_get_buffer (vm, bi0);
2345  ip0 = vlib_buffer_get_current (b0);
2346  udp0 = ip4_next_header (ip0);
2347  tcp0 = (tcp_header_t *) udp0;
2348 
2349  proto0 = ip_proto_to_snat_proto (ip0->protocol);
2350 
2352  &next0, 0);
2353 
2354  if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2355  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2356 
2357  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2358 
2359  /* verify speculative enqueue, maybe switch current next frame */
2360  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2361  to_next, n_left_to_next,
2362  bi0, next0);
2363  }
2364 
2365  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2366  }
2367 
2369  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2370  pkts_processed);
2371  return frame->n_vectors;
2372 }
2373 
2375  .function = nat44_hairpinning_fn,
2376  .name = "nat44-hairpinning",
2377  .vector_size = sizeof (u32),
2378  .type = VLIB_NODE_TYPE_INTERNAL,
2379  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2380  .error_strings = snat_in2out_error_strings,
2381  .n_next_nodes = 2,
2382  .next_nodes = {
2383  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2384  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2385  },
2386 };
2387 
2390 
2391 static inline void
2393  vlib_buffer_t * b0,
2394  ip4_header_t * ip0,
2395  u16 sport,
2396  u16 dport,
2397  u32 proto0)
2398 {
2399  snat_session_key_t key0, sm0;
2400  snat_session_t * s0;
2401  clib_bihash_kv_8_8_t kv0, value0;
2402  ip_csum_t sum0;
2403  u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2404  u16 new_dst_port0, old_dst_port0;
2405  udp_header_t * udp0;
2406  tcp_header_t * tcp0;
2407 
2408  key0.addr = ip0->dst_address;
2409  key0.port = dport;
2410  key0.protocol = proto0;
2411  key0.fib_index = sm->outside_fib_index;
2412  kv0.key = key0.as_u64;
2413 
2414  udp0 = ip4_next_header (ip0);
2415 
2416  /* Check if destination is static mappings */
2417  if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
2418  {
2419  new_dst_addr0 = sm0.addr.as_u32;
2420  new_dst_port0 = sm0.port;
2421  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2422  }
2423  /* or active sessions */
2424  else
2425  {
2426  if (sm->num_workers > 1)
2427  ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2428  else
2429  ti = sm->num_workers;
2430 
2431  if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2432  {
2433  si = value0.value;
2434  s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2435  new_dst_addr0 = s0->in2out.addr.as_u32;
2436  new_dst_port0 = s0->in2out.port;
2437  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2438  }
2439  }
2440 
2441  /* Destination is behind the same NAT, use internal address and port */
2442  if (new_dst_addr0)
2443  {
2444  old_dst_addr0 = ip0->dst_address.as_u32;
2445  ip0->dst_address.as_u32 = new_dst_addr0;
2446  sum0 = ip0->checksum;
2447  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2448  ip4_header_t, dst_address);
2449  ip0->checksum = ip_csum_fold (sum0);
2450 
2451  old_dst_port0 = dport;
2452  if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2453  ip4_is_first_fragment (ip0)))
2454  {
2455  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2456  {
2457  tcp0 = ip4_next_header (ip0);
2458  tcp0->dst = new_dst_port0;
2459  sum0 = tcp0->checksum;
2460  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2461  ip4_header_t, dst_address);
2462  sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2463  ip4_header_t /* cheat */, length);
2464  tcp0->checksum = ip_csum_fold(sum0);
2465  }
2466  else
2467  {
2468  udp0->dst_port = new_dst_port0;
2469  udp0->checksum = 0;
2470  }
2471  }
2472  else
2473  {
2474  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2475  {
2476  tcp0 = ip4_next_header (ip0);
2477  sum0 = tcp0->checksum;
2478  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2479  ip4_header_t, dst_address);
2480  tcp0->checksum = ip_csum_fold(sum0);
2481  }
2482  }
2483  }
2484 }
2485 
2486 static uword
2488  vlib_node_runtime_t * node,
2489  vlib_frame_t * frame)
2490 {
2491  u32 n_left_from, *from, *to_next;
2492  snat_in2out_next_t next_index;
2493  u32 pkts_processed = 0;
2494  snat_main_t *sm = &snat_main;
2495  f64 now = vlib_time_now (vm);
2496  u32 thread_index = vlib_get_thread_index ();
2497  snat_main_per_thread_data_t *per_thread_data =
2498  &sm->per_thread_data[thread_index];
2499  u32 *fragments_to_drop = 0;
2500  u32 *fragments_to_loopback = 0;
2501 
2502  from = vlib_frame_vector_args (frame);
2503  n_left_from = frame->n_vectors;
2504  next_index = node->cached_next_index;
2505 
2506  while (n_left_from > 0)
2507  {
2508  u32 n_left_to_next;
2509 
2510  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2511 
2512  while (n_left_from > 0 && n_left_to_next > 0)
2513  {
2514  u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2515  vlib_buffer_t *b0;
2516  u32 next0;
2517  u8 cached0 = 0;
2518  ip4_header_t *ip0;
2519  nat_reass_ip4_t *reass0;
2520  udp_header_t * udp0;
2521  tcp_header_t * tcp0;
2522  snat_session_key_t key0;
2523  clib_bihash_kv_8_8_t kv0, value0;
2524  snat_session_t * s0 = 0;
2525  u16 old_port0, new_port0;
2526  ip_csum_t sum0;
2527 
2528  /* speculatively enqueue b0 to the current next frame */
2529  bi0 = from[0];
2530  to_next[0] = bi0;
2531  from += 1;
2532  to_next += 1;
2533  n_left_from -= 1;
2534  n_left_to_next -= 1;
2535 
2536  b0 = vlib_get_buffer (vm, bi0);
2537  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2538 
2539  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2541  sw_if_index0);
2542 
2544  {
2545  next0 = SNAT_IN2OUT_NEXT_DROP;
2546  b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2547  goto trace0;
2548  }
2549 
2550  ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2551  udp0 = ip4_next_header (ip0);
2552  tcp0 = (tcp_header_t *) udp0;
2553  proto0 = ip_proto_to_snat_proto (ip0->protocol);
2554 
2556  ip0->dst_address,
2557  ip0->fragment_id,
2558  ip0->protocol,
2559  1,
2560  &fragments_to_drop);
2561 
2562  if (PREDICT_FALSE (!reass0))
2563  {
2564  next0 = SNAT_IN2OUT_NEXT_DROP;
2565  b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2566  goto trace0;
2567  }
2568 
2570  {
2571  key0.addr = ip0->src_address;
2572  key0.port = udp0->src_port;
2573  key0.protocol = proto0;
2574  key0.fib_index = rx_fib_index0;
2575  kv0.key = key0.as_u64;
2576 
2577  if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2578  {
2579  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2580  ip0, proto0, rx_fib_index0, thread_index)))
2581  goto trace0;
2582 
2583  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2584  &s0, node, next0, thread_index);
2585 
2586  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2587  goto trace0;
2588 
2589  reass0->sess_index = s0 - per_thread_data->sessions;
2590  }
2591  else
2592  {
2593  s0 = pool_elt_at_index (per_thread_data->sessions,
2594  value0.value);
2595  reass0->sess_index = value0.value;
2596  }
2597  nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2598  }
2599  else
2600  {
2601  if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2602  {
2603  if (nat_ip4_reass_add_fragment (reass0, bi0))
2604  {
2605  b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2606  next0 = SNAT_IN2OUT_NEXT_DROP;
2607  goto trace0;
2608  }
2609  cached0 = 1;
2610  goto trace0;
2611  }
2612  s0 = pool_elt_at_index (per_thread_data->sessions,
2613  reass0->sess_index);
2614  }
2615 
2616  old_addr0 = ip0->src_address.as_u32;
2617  ip0->src_address = s0->out2in.addr;
2618  new_addr0 = ip0->src_address.as_u32;
2619  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2620 
2621  sum0 = ip0->checksum;
2622  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2623  ip4_header_t,
2624  src_address /* changed member */);
2625  ip0->checksum = ip_csum_fold (sum0);
2626 
2628  {
2629  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2630  {
2631  old_port0 = tcp0->src_port;
2632  tcp0->src_port = s0->out2in.port;
2633  new_port0 = tcp0->src_port;
2634 
2635  sum0 = tcp0->checksum;
2636  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2637  ip4_header_t,
2638  dst_address /* changed member */);
2639  sum0 = ip_csum_update (sum0, old_port0, new_port0,
2640  ip4_header_t /* cheat */,
2641  length /* changed member */);
2642  tcp0->checksum = ip_csum_fold(sum0);
2643  }
2644  else
2645  {
2646  old_port0 = udp0->src_port;
2647  udp0->src_port = s0->out2in.port;
2648  udp0->checksum = 0;
2649  }
2650  }
2651 
2652  /* Hairpinning */
2653  nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2654  s0->ext_host_port, proto0);
2655 
2656  /* Accounting */
2657  s0->last_heard = now;
2658  s0->total_pkts++;
2659  s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2660  /* Per-user LRU list maintenance */
2661  clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2662  s0->per_user_index);
2663  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2664  s0->per_user_list_head_index,
2665  s0->per_user_index);
2666 
2667  trace0:
2669  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2670  {
2672  vlib_add_trace (vm, node, b0, sizeof (*t));
2673  t->cached = cached0;
2674  t->sw_if_index = sw_if_index0;
2675  t->next_index = next0;
2676  }
2677 
2678  if (cached0)
2679  {
2680  n_left_to_next++;
2681  to_next--;
2682  }
2683  else
2684  {
2685  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2686 
2687  /* verify speculative enqueue, maybe switch current next frame */
2688  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2689  to_next, n_left_to_next,
2690  bi0, next0);
2691  }
2692 
2693  if (n_left_from == 0 && vec_len (fragments_to_loopback))
2694  {
2695  from = vlib_frame_vector_args (frame);
2696  u32 len = vec_len (fragments_to_loopback);
2697  if (len <= VLIB_FRAME_SIZE)
2698  {
2699  clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2700  n_left_from = len;
2701  vec_reset_length (fragments_to_loopback);
2702  }
2703  else
2704  {
2705  clib_memcpy (from,
2706  fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2707  sizeof (u32) * VLIB_FRAME_SIZE);
2708  n_left_from = VLIB_FRAME_SIZE;
2709  _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2710  }
2711  }
2712  }
2713 
2714  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2715  }
2716 
2718  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2719  pkts_processed);
2720 
2721  nat_send_all_to_node (vm, fragments_to_drop, node,
2722  &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2724 
2725  vec_free (fragments_to_drop);
2726  vec_free (fragments_to_loopback);
2727  return frame->n_vectors;
2728 }
2729 
2731  .function = nat44_in2out_reass_node_fn,
2732  .name = "nat44-in2out-reass",
2733  .vector_size = sizeof (u32),
2734  .format_trace = format_nat44_in2out_reass_trace,
2735  .type = VLIB_NODE_TYPE_INTERNAL,
2736 
2737  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2738  .error_strings = snat_in2out_error_strings,
2739 
2740  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2741  .next_nodes = {
2742  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2743  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2744  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2745  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2746  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2747  },
2748 };
2749 
2752 
2753 /**************************/
2754 /*** deterministic mode ***/
2755 /**************************/
2756 static uword
2758  vlib_node_runtime_t * node,
2759  vlib_frame_t * frame)
2760 {
2761  u32 n_left_from, * from, * to_next;
2762  snat_in2out_next_t next_index;
2763  u32 pkts_processed = 0;
2764  snat_main_t * sm = &snat_main;
2765  u32 now = (u32) vlib_time_now (vm);
2766  u32 thread_index = vlib_get_thread_index ();
2767 
2768  from = vlib_frame_vector_args (frame);
2769  n_left_from = frame->n_vectors;
2770  next_index = node->cached_next_index;
2771 
2772  while (n_left_from > 0)
2773  {
2774  u32 n_left_to_next;
2775 
2776  vlib_get_next_frame (vm, node, next_index,
2777  to_next, n_left_to_next);
2778 
2779  while (n_left_from >= 4 && n_left_to_next >= 2)
2780  {
2781  u32 bi0, bi1;
2782  vlib_buffer_t * b0, * b1;
2783  u32 next0, next1;
2784  u32 sw_if_index0, sw_if_index1;
2785  ip4_header_t * ip0, * ip1;
2786  ip_csum_t sum0, sum1;
2787  ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2788  u16 old_port0, new_port0, lo_port0, i0;
2789  u16 old_port1, new_port1, lo_port1, i1;
2790  udp_header_t * udp0, * udp1;
2791  tcp_header_t * tcp0, * tcp1;
2792  u32 proto0, proto1;
2793  snat_det_out_key_t key0, key1;
2794  snat_det_map_t * dm0, * dm1;
2795  snat_det_session_t * ses0 = 0, * ses1 = 0;
2796  u32 rx_fib_index0, rx_fib_index1;
2797  icmp46_header_t * icmp0, * icmp1;
2798 
2799  /* Prefetch next iteration. */
2800  {
2801  vlib_buffer_t * p2, * p3;
2802 
2803  p2 = vlib_get_buffer (vm, from[2]);
2804  p3 = vlib_get_buffer (vm, from[3]);
2805 
2806  vlib_prefetch_buffer_header (p2, LOAD);
2807  vlib_prefetch_buffer_header (p3, LOAD);
2808 
2811  }
2812 
2813  /* speculatively enqueue b0 and b1 to the current next frame */
2814  to_next[0] = bi0 = from[0];
2815  to_next[1] = bi1 = from[1];
2816  from += 2;
2817  to_next += 2;
2818  n_left_from -= 2;
2819  n_left_to_next -= 2;
2820 
2821  b0 = vlib_get_buffer (vm, bi0);
2822  b1 = vlib_get_buffer (vm, bi1);
2823 
2824  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2825  next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2826 
2827  ip0 = vlib_buffer_get_current (b0);
2828  udp0 = ip4_next_header (ip0);
2829  tcp0 = (tcp_header_t *) udp0;
2830 
2831  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2832 
2833  if (PREDICT_FALSE(ip0->ttl == 1))
2834  {
2835  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2836  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2837  ICMP4_time_exceeded_ttl_exceeded_in_transit,
2838  0);
2840  goto trace0;
2841  }
2842 
2843  proto0 = ip_proto_to_snat_proto (ip0->protocol);
2844 
2845  if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2846  {
2847  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2848  icmp0 = (icmp46_header_t *) udp0;
2849 
2850  next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2851  rx_fib_index0, node, next0, thread_index,
2852  &ses0, &dm0);
2853  goto trace0;
2854  }
2855 
2856  dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2857  if (PREDICT_FALSE(!dm0))
2858  {
2859  clib_warning("no match for internal host %U",
2861  next0 = SNAT_IN2OUT_NEXT_DROP;
2862  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2863  goto trace0;
2864  }
2865 
2866  snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2867 
2868  key0.ext_host_addr = ip0->dst_address;
2869  key0.ext_host_port = tcp0->dst;
2870 
2871  ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2872  if (PREDICT_FALSE(!ses0))
2873  {
2874  for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2875  {
2876  key0.out_port = clib_host_to_net_u16 (lo_port0 +
2877  ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2878 
2879  if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2880  continue;
2881 
2882  ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2883  break;
2884  }
2885  if (PREDICT_FALSE(!ses0))
2886  {
2887  /* too many sessions for user, send ICMP error packet */
2888 
2889  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2890  icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2891  ICMP4_destination_unreachable_destination_unreachable_host,
2892  0);
2894  goto trace0;
2895  }
2896  }
2897 
2898  new_port0 = ses0->out.out_port;
2899 
2900  old_addr0.as_u32 = ip0->src_address.as_u32;
2901  ip0->src_address.as_u32 = new_addr0.as_u32;
2902  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2903 
2904  sum0 = ip0->checksum;
2905  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2906  ip4_header_t,
2907  src_address /* changed member */);
2908  ip0->checksum = ip_csum_fold (sum0);
2909 
2910  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2911  {
2912  if (tcp0->flags & TCP_FLAG_SYN)
2913  ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2914  else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2915  ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2916  else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2917  ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2918  else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2919  snat_det_ses_close(dm0, ses0);
2920  else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2921  ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2922  else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2923  ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2924 
2925  old_port0 = tcp0->src;
2926  tcp0->src = new_port0;
2927 
2928  sum0 = tcp0->checksum;
2929  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2930  ip4_header_t,
2931  dst_address /* changed member */);
2932  sum0 = ip_csum_update (sum0, old_port0, new_port0,
2933  ip4_header_t /* cheat */,
2934  length /* changed member */);
2935  tcp0->checksum = ip_csum_fold(sum0);
2936  }
2937  else
2938  {
2939  ses0->state = SNAT_SESSION_UDP_ACTIVE;
2940  old_port0 = udp0->src_port;
2941  udp0->src_port = new_port0;
2942  udp0->checksum = 0;
2943  }
2944 
2945  switch(ses0->state)
2946  {
2947  case SNAT_SESSION_UDP_ACTIVE:
2948  ses0->expire = now + sm->udp_timeout;
2949  break;
2950  case SNAT_SESSION_TCP_SYN_SENT:
2951  case SNAT_SESSION_TCP_FIN_WAIT:
2952  case SNAT_SESSION_TCP_CLOSE_WAIT:
2953  case SNAT_SESSION_TCP_LAST_ACK:
2954  ses0->expire = now + sm->tcp_transitory_timeout;
2955  break;
2956  case SNAT_SESSION_TCP_ESTABLISHED:
2957  ses0->expire = now + sm->tcp_established_timeout;
2958  break;
2959  }
2960 
2961  trace0:
2963  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2964  {
2965  snat_in2out_trace_t *t =
2966  vlib_add_trace (vm, node, b0, sizeof (*t));
2967  t->is_slow_path = 0;
2968  t->sw_if_index = sw_if_index0;
2969  t->next_index = next0;
2970  t->session_index = ~0;
2971  if (ses0)
2972  t->session_index = ses0 - dm0->sessions;
2973  }
2974 
2975  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2976 
2977  ip1 = vlib_buffer_get_current (b1);
2978  udp1 = ip4_next_header (ip1);
2979  tcp1 = (tcp_header_t *) udp1;
2980 
2981  sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2982 
2983  if (PREDICT_FALSE(ip1->ttl == 1))
2984  {
2985  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2986  icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2987  ICMP4_time_exceeded_ttl_exceeded_in_transit,
2988  0);
2990  goto trace1;
2991  }
2992 
2993  proto1 = ip_proto_to_snat_proto (ip1->protocol);
2994 
2995  if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2996  {
2997  rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2998  icmp1 = (icmp46_header_t *) udp1;
2999 
3000  next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
3001  rx_fib_index1, node, next1, thread_index,
3002  &ses1, &dm1);
3003  goto trace1;
3004  }
3005 
3006  dm1 = snat_det_map_by_user(sm, &ip1->src_address);
3007  if (PREDICT_FALSE(!dm1))
3008  {
3009  clib_warning("no match for internal host %U",
3011  next1 = SNAT_IN2OUT_NEXT_DROP;
3012  b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3013  goto trace1;
3014  }
3015 
3016  snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3017 
3018  key1.ext_host_addr = ip1->dst_address;
3019  key1.ext_host_port = tcp1->dst;
3020 
3021  ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3022  if (PREDICT_FALSE(!ses1))
3023  {
3024  for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3025  {
3026  key1.out_port = clib_host_to_net_u16 (lo_port1 +
3027  ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3028 
3029  if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3030  continue;
3031 
3032  ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3033  break;
3034  }
3035  if (PREDICT_FALSE(!ses1))
3036  {
3037  /* too many sessions for user, send ICMP error packet */
3038 
3039  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3040  icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3041  ICMP4_destination_unreachable_destination_unreachable_host,
3042  0);
3044  goto trace1;
3045  }
3046  }
3047 
3048  new_port1 = ses1->out.out_port;
3049 
3050  old_addr1.as_u32 = ip1->src_address.as_u32;
3051  ip1->src_address.as_u32 = new_addr1.as_u32;
3052  vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3053 
3054  sum1 = ip1->checksum;
3055  sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3056  ip4_header_t,
3057  src_address /* changed member */);
3058  ip1->checksum = ip_csum_fold (sum1);
3059 
3060  if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3061  {
3062  if (tcp1->flags & TCP_FLAG_SYN)
3063  ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3064  else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3065  ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3066  else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3067  ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3068  else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3069  snat_det_ses_close(dm1, ses1);
3070  else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3071  ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3072  else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3073  ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3074 
3075  old_port1 = tcp1->src;
3076  tcp1->src = new_port1;
3077 
3078  sum1 = tcp1->checksum;
3079  sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3080  ip4_header_t,
3081  dst_address /* changed member */);
3082  sum1 = ip_csum_update (sum1, old_port1, new_port1,
3083  ip4_header_t /* cheat */,
3084  length /* changed member */);
3085  tcp1->checksum = ip_csum_fold(sum1);
3086  }
3087  else
3088  {
3089  ses1->state = SNAT_SESSION_UDP_ACTIVE;
3090  old_port1 = udp1->src_port;
3091  udp1->src_port = new_port1;
3092  udp1->checksum = 0;
3093  }
3094 
3095  switch(ses1->state)
3096  {
3097  case SNAT_SESSION_UDP_ACTIVE:
3098  ses1->expire = now + sm->udp_timeout;
3099  break;
3100  case SNAT_SESSION_TCP_SYN_SENT:
3101  case SNAT_SESSION_TCP_FIN_WAIT:
3102  case SNAT_SESSION_TCP_CLOSE_WAIT:
3103  case SNAT_SESSION_TCP_LAST_ACK:
3104  ses1->expire = now + sm->tcp_transitory_timeout;
3105  break;
3106  case SNAT_SESSION_TCP_ESTABLISHED:
3107  ses1->expire = now + sm->tcp_established_timeout;
3108  break;
3109  }
3110 
3111  trace1:
3113  && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3114  {
3115  snat_in2out_trace_t *t =
3116  vlib_add_trace (vm, node, b1, sizeof (*t));
3117  t->is_slow_path = 0;
3118  t->sw_if_index = sw_if_index1;
3119  t->next_index = next1;
3120  t->session_index = ~0;
3121  if (ses1)
3122  t->session_index = ses1 - dm1->sessions;
3123  }
3124 
3125  pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3126 
3127  /* verify speculative enqueues, maybe switch current next frame */
3128  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3129  to_next, n_left_to_next,
3130  bi0, bi1, next0, next1);
3131  }
3132 
3133  while (n_left_from > 0 && n_left_to_next > 0)
3134  {
3135  u32 bi0;
3136  vlib_buffer_t * b0;
3137  u32 next0;
3138  u32 sw_if_index0;
3139  ip4_header_t * ip0;
3140  ip_csum_t sum0;
3141  ip4_address_t new_addr0, old_addr0;
3142  u16 old_port0, new_port0, lo_port0, i0;
3143  udp_header_t * udp0;
3144  tcp_header_t * tcp0;
3145  u32 proto0;
3146  snat_det_out_key_t key0;
3147  snat_det_map_t * dm0;
3148  snat_det_session_t * ses0 = 0;
3149  u32 rx_fib_index0;
3150  icmp46_header_t * icmp0;
3151 
3152  /* speculatively enqueue b0 to the current next frame */
3153  bi0 = from[0];
3154  to_next[0] = bi0;
3155  from += 1;
3156  to_next += 1;
3157  n_left_from -= 1;
3158  n_left_to_next -= 1;
3159 
3160  b0 = vlib_get_buffer (vm, bi0);
3161  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3162 
3163  ip0 = vlib_buffer_get_current (b0);
3164  udp0 = ip4_next_header (ip0);
3165  tcp0 = (tcp_header_t *) udp0;
3166 
3167  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3168 
3169  if (PREDICT_FALSE(ip0->ttl == 1))
3170  {
3171  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3172  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3173  ICMP4_time_exceeded_ttl_exceeded_in_transit,
3174  0);
3176  goto trace00;
3177  }
3178 
3179  proto0 = ip_proto_to_snat_proto (ip0->protocol);
3180 
3181  if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3182  {
3183  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3184  icmp0 = (icmp46_header_t *) udp0;
3185 
3186  next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3187  rx_fib_index0, node, next0, thread_index,
3188  &ses0, &dm0);
3189  goto trace00;
3190  }
3191 
3192  dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3193  if (PREDICT_FALSE(!dm0))
3194  {
3195  clib_warning("no match for internal host %U",
3197  next0 = SNAT_IN2OUT_NEXT_DROP;
3198  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3199  goto trace00;
3200  }
3201 
3202  snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3203 
3204  key0.ext_host_addr = ip0->dst_address;
3205  key0.ext_host_port = tcp0->dst;
3206 
3207  ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3208  if (PREDICT_FALSE(!ses0))
3209  {
3210  for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3211  {
3212  key0.out_port = clib_host_to_net_u16 (lo_port0 +
3213  ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3214 
3215  if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3216  continue;
3217 
3218  ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3219  break;
3220  }
3221  if (PREDICT_FALSE(!ses0))
3222  {
3223  /* too many sessions for user, send ICMP error packet */
3224 
3225  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3226  icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3227  ICMP4_destination_unreachable_destination_unreachable_host,
3228  0);
3230  goto trace00;
3231  }
3232  }
3233 
3234  new_port0 = ses0->out.out_port;
3235 
3236  old_addr0.as_u32 = ip0->src_address.as_u32;
3237  ip0->src_address.as_u32 = new_addr0.as_u32;
3238  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3239 
3240  sum0 = ip0->checksum;
3241  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3242  ip4_header_t,
3243  src_address /* changed member */);
3244  ip0->checksum = ip_csum_fold (sum0);
3245 
3246  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3247  {
3248  if (tcp0->flags & TCP_FLAG_SYN)
3249  ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3250  else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3251  ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3252  else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3253  ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3254  else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3255  snat_det_ses_close(dm0, ses0);
3256  else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3257  ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3258  else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3259  ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3260 
3261  old_port0 = tcp0->src;
3262  tcp0->src = new_port0;
3263 
3264  sum0 = tcp0->checksum;
3265  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3266  ip4_header_t,
3267  dst_address /* changed member */);
3268  sum0 = ip_csum_update (sum0, old_port0, new_port0,
3269  ip4_header_t /* cheat */,
3270  length /* changed member */);
3271  tcp0->checksum = ip_csum_fold(sum0);
3272  }
3273  else
3274  {
3275  ses0->state = SNAT_SESSION_UDP_ACTIVE;
3276  old_port0 = udp0->src_port;
3277  udp0->src_port = new_port0;
3278  udp0->checksum = 0;
3279  }
3280 
3281  switch(ses0->state)
3282  {
3283  case SNAT_SESSION_UDP_ACTIVE:
3284  ses0->expire = now + sm->udp_timeout;
3285  break;
3286  case SNAT_SESSION_TCP_SYN_SENT:
3287  case SNAT_SESSION_TCP_FIN_WAIT:
3288  case SNAT_SESSION_TCP_CLOSE_WAIT:
3289  case SNAT_SESSION_TCP_LAST_ACK:
3290  ses0->expire = now + sm->tcp_transitory_timeout;
3291  break;
3292  case SNAT_SESSION_TCP_ESTABLISHED:
3293  ses0->expire = now + sm->tcp_established_timeout;
3294  break;
3295  }
3296 
3297  trace00:
3299  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3300  {
3301  snat_in2out_trace_t *t =
3302  vlib_add_trace (vm, node, b0, sizeof (*t));
3303  t->is_slow_path = 0;
3304  t->sw_if_index = sw_if_index0;
3305  t->next_index = next0;
3306  t->session_index = ~0;
3307  if (ses0)
3308  t->session_index = ses0 - dm0->sessions;
3309  }
3310 
3311  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3312 
3313  /* verify speculative enqueue, maybe switch current next frame */
3314  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3315  to_next, n_left_to_next,
3316  bi0, next0);
3317  }
3318 
3319  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3320  }
3321 
3323  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3324  pkts_processed);
3325  return frame->n_vectors;
3326 }
3327 
3329  .function = snat_det_in2out_node_fn,
3330  .name = "nat44-det-in2out",
3331  .vector_size = sizeof (u32),
3332  .format_trace = format_snat_in2out_trace,
3333  .type = VLIB_NODE_TYPE_INTERNAL,
3334 
3335  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3336  .error_strings = snat_in2out_error_strings,
3337 
3338  .runtime_data_bytes = sizeof (snat_runtime_t),
3339 
3340  .n_next_nodes = 3,
3341 
3342  /* edit / add dispositions here */
3343  .next_nodes = {
3344  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3345  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3346  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3347  },
3348 };
3349 
3351 
3352 /**
3353  * Get address and port values to be used for ICMP packet translation
3354  * and create session if needed
3355  *
3356  * @param[in,out] sm NAT main
3357  * @param[in,out] node NAT node runtime
3358  * @param[in] thread_index thread index
3359  * @param[in,out] b0 buffer containing packet to be translated
3360  * @param[out] p_proto protocol used for matching
3361  * @param[out] p_value address and port after NAT translation
3362  * @param[out] p_dont_translate if packet should not be translated
3363  * @param d optional parameter
3364  * @param e optional parameter
3365  */
3367  u32 thread_index, vlib_buffer_t *b0,
3368  ip4_header_t *ip0, u8 *p_proto,
3369  snat_session_key_t *p_value,
3370  u8 *p_dont_translate, void *d, void *e)
3371 {
3372  icmp46_header_t *icmp0;
3373  u32 sw_if_index0;
3374  u32 rx_fib_index0;
3375  u8 protocol;
3376  snat_det_out_key_t key0;
3377  u8 dont_translate = 0;
3378  u32 next0 = ~0;
3379  icmp_echo_header_t *echo0, *inner_echo0 = 0;
3380  ip4_header_t *inner_ip0;
3381  void *l4_header = 0;
3382  icmp46_header_t *inner_icmp0;
3383  snat_det_map_t * dm0 = 0;
3384  ip4_address_t new_addr0;
3385  u16 lo_port0, i0;
3386  snat_det_session_t * ses0 = 0;
3387  ip4_address_t in_addr;
3388  u16 in_port;
3389 
3390  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3391  echo0 = (icmp_echo_header_t *)(icmp0+1);
3392  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3393  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3394 
3395  if (!icmp_is_error_message (icmp0))
3396  {
3397  protocol = SNAT_PROTOCOL_ICMP;
3398  in_addr = ip0->src_address;
3399  in_port = echo0->identifier;
3400  }
3401  else
3402  {
3403  inner_ip0 = (ip4_header_t *)(echo0+1);
3404  l4_header = ip4_next_header (inner_ip0);
3405  protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3406  in_addr = inner_ip0->dst_address;
3407  switch (protocol)
3408  {
3409  case SNAT_PROTOCOL_ICMP:
3410  inner_icmp0 = (icmp46_header_t*)l4_header;
3411  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3412  in_port = inner_echo0->identifier;
3413  break;
3414  case SNAT_PROTOCOL_UDP:
3415  case SNAT_PROTOCOL_TCP:
3416  in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3417  break;
3418  default:
3419  b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3420  next0 = SNAT_IN2OUT_NEXT_DROP;
3421  goto out;
3422  }
3423  }
3424 
3425  dm0 = snat_det_map_by_user(sm, &in_addr);
3426  if (PREDICT_FALSE(!dm0))
3427  {
3428  clib_warning("no match for internal host %U",
3429  format_ip4_address, &in_addr);
3430  if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3431  IP_PROTOCOL_ICMP, rx_fib_index0)))
3432  {
3433  dont_translate = 1;
3434  goto out;
3435  }
3436  next0 = SNAT_IN2OUT_NEXT_DROP;
3437  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3438  goto out;
3439  }
3440 
3441  snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3442 
3443  key0.ext_host_addr = ip0->dst_address;
3444  key0.ext_host_port = 0;
3445 
3446  ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3447  if (PREDICT_FALSE(!ses0))
3448  {
3449  if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3450  IP_PROTOCOL_ICMP, rx_fib_index0)))
3451  {
3452  dont_translate = 1;
3453  goto out;
3454  }
3455  if (icmp0->type != ICMP4_echo_request)
3456  {
3457  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3458  next0 = SNAT_IN2OUT_NEXT_DROP;
3459  goto out;
3460  }
3461  for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3462  {
3463  key0.out_port = clib_host_to_net_u16 (lo_port0 +
3464  ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3465 
3466  if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3467  continue;
3468 
3469  ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3470  break;
3471  }
3472  if (PREDICT_FALSE(!ses0))
3473  {
3474  next0 = SNAT_IN2OUT_NEXT_DROP;
3475  b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3476  goto out;
3477  }
3478  }
3479 
3480  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3481  !icmp_is_error_message (icmp0)))
3482  {
3483  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3484  next0 = SNAT_IN2OUT_NEXT_DROP;
3485  goto out;
3486  }
3487 
3488  u32 now = (u32) vlib_time_now (sm->vlib_main);
3489 
3490  ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3491  ses0->expire = now + sm->icmp_timeout;
3492 
3493 out:
3494  *p_proto = protocol;
3495  if (ses0)
3496  {
3497  p_value->addr = new_addr0;
3498  p_value->fib_index = sm->outside_fib_index;
3499  p_value->port = ses0->out.out_port;
3500  }
3501  *p_dont_translate = dont_translate;
3502  if (d)
3503  *(snat_det_session_t**)d = ses0;
3504  if (e)
3505  *(snat_det_map_t**)e = dm0;
3506  return next0;
3507 }
3508 
3509 /**********************/
3510 /*** worker handoff ***/
3511 /**********************/
3512 static inline uword
3514  vlib_node_runtime_t * node,
3515  vlib_frame_t * frame,
3516  u8 is_output)
3517 {
3518  snat_main_t *sm = &snat_main;
3520  u32 n_left_from, *from, *to_next = 0;
3521  static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3522  static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3523  = 0;
3524  vlib_frame_queue_elt_t *hf = 0;
3525  vlib_frame_t *f = 0;
3526  int i;
3527  u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3528  u32 next_worker_index = 0;
3529  u32 current_worker_index = ~0;
3530  u32 thread_index = vlib_get_thread_index ();
3531  u32 fq_index;
3532  u32 to_node_index;
3533 
3534  ASSERT (vec_len (sm->workers));
3535 
3536  if (is_output)
3537  {
3538  fq_index = sm->fq_in2out_output_index;
3539  to_node_index = sm->in2out_output_node_index;
3540  }
3541  else
3542  {
3543  fq_index = sm->fq_in2out_index;
3544  to_node_index = sm->in2out_node_index;
3545  }
3546 
3547  if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3548  {
3549  vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3550 
3551  vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3552  sm->first_worker_index + sm->num_workers - 1,
3553  (vlib_frame_queue_t *) (~0));
3554  }
3555 
3556  from = vlib_frame_vector_args (frame);
3557  n_left_from = frame->n_vectors;
3558 
3559  while (n_left_from > 0)
3560  {
3561  u32 bi0;
3562  vlib_buffer_t *b0;
3563  u32 sw_if_index0;
3564  u32 rx_fib_index0;
3565  ip4_header_t * ip0;
3566  u8 do_handoff;
3567 
3568  bi0 = from[0];
3569  from += 1;
3570  n_left_from -= 1;
3571 
3572  b0 = vlib_get_buffer (vm, bi0);
3573 
3574  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3575  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3576 
3577  ip0 = vlib_buffer_get_current (b0);
3578 
3579  next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3580 
3581  if (PREDICT_FALSE (next_worker_index != thread_index))
3582  {
3583  do_handoff = 1;
3584 
3585  if (next_worker_index != current_worker_index)
3586  {
3587  if (hf)
3588  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3589 
3590  hf = vlib_get_worker_handoff_queue_elt (fq_index,
3591  next_worker_index,
3592  handoff_queue_elt_by_worker_index);
3593 
3594  n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3595  to_next_worker = &hf->buffer_index[hf->n_vectors];
3596  current_worker_index = next_worker_index;
3597  }
3598 
3599  /* enqueue to correct worker thread */
3600  to_next_worker[0] = bi0;
3601  to_next_worker++;
3602  n_left_to_next_worker--;
3603 
3604  if (n_left_to_next_worker == 0)
3605  {
3606  hf->n_vectors = VLIB_FRAME_SIZE;
3608  current_worker_index = ~0;
3609  handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3610  hf = 0;
3611  }
3612  }
3613  else
3614  {
3615  do_handoff = 0;
3616  /* if this is 1st frame */
3617  if (!f)
3618  {
3619  f = vlib_get_frame_to_node (vm, to_node_index);
3620  to_next = vlib_frame_vector_args (f);
3621  }
3622 
3623  to_next[0] = bi0;
3624  to_next += 1;
3625  f->n_vectors++;
3626  }
3627 
3629  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3630  {
3632  vlib_add_trace (vm, node, b0, sizeof (*t));
3633  t->next_worker_index = next_worker_index;
3634  t->do_handoff = do_handoff;
3635  }
3636  }
3637 
3638  if (f)
3639  vlib_put_frame_to_node (vm, to_node_index, f);
3640 
3641  if (hf)
3642  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3643 
3644  /* Ship frames to the worker nodes */
3645  for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3646  {
3647  if (handoff_queue_elt_by_worker_index[i])
3648  {
3649  hf = handoff_queue_elt_by_worker_index[i];
3650  /*
3651  * It works better to let the handoff node
3652  * rate-adapt, always ship the handoff queue element.
3653  */
3654  if (1 || hf->n_vectors == hf->last_n_vectors)
3655  {
3657  handoff_queue_elt_by_worker_index[i] = 0;
3658  }
3659  else
3660  hf->last_n_vectors = hf->n_vectors;
3661  }
3662  congested_handoff_queue_by_worker_index[i] =
3663  (vlib_frame_queue_t *) (~0);
3664  }
3665  hf = 0;
3666  current_worker_index = ~0;
3667  return frame->n_vectors;
3668 }
3669 
3670 static uword
3672  vlib_node_runtime_t * node,
3673  vlib_frame_t * frame)
3674 {
3675  return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3676 }
3677 
3679  .function = snat_in2out_worker_handoff_fn,
3680  .name = "nat44-in2out-worker-handoff",
3681  .vector_size = sizeof (u32),
3683  .type = VLIB_NODE_TYPE_INTERNAL,
3684 
3685  .n_next_nodes = 1,
3686 
3687  .next_nodes = {
3688  [0] = "error-drop",
3689  },
3690 };
3691 
3694 
3695 static uword
3697  vlib_node_runtime_t * node,
3698  vlib_frame_t * frame)
3699 {
3700  return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3701 }
3702 
3705  .name = "nat44-in2out-output-worker-handoff",
3706  .vector_size = sizeof (u32),
3708  .type = VLIB_NODE_TYPE_INTERNAL,
3709 
3710  .n_next_nodes = 1,
3711 
3712  .next_nodes = {
3713  [0] = "error-drop",
3714  },
3715 };
3716 
3719 
3722 {
3723  snat_address_t * ap;
3724  clib_bihash_kv_8_8_t kv, value;
3725  snat_session_key_t m_key;
3726 
3727  vec_foreach (ap, sm->addresses)
3728  {
3729  if (ap->addr.as_u32 == dst_addr->as_u32)
3730  return 1;
3731  }
3732 
3733  m_key.addr.as_u32 = dst_addr->as_u32;
3734  m_key.fib_index = sm->outside_fib_index;
3735  m_key.port = 0;
3736  m_key.protocol = 0;
3737  kv.key = m_key.as_u64;
3738  if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3739  return 1;
3740 
3741  return 0;
3742 }
3743 
3744 static uword
3746  vlib_node_runtime_t * node,
3747  vlib_frame_t * frame)
3748 {
3749  u32 n_left_from, * from, * to_next;
3750  snat_in2out_next_t next_index;
3751  u32 pkts_processed = 0;
3752  snat_main_t * sm = &snat_main;
3753 
3754  from = vlib_frame_vector_args (frame);
3755  n_left_from = frame->n_vectors;
3756  next_index = node->cached_next_index;
3757 
3758  while (n_left_from > 0)
3759  {
3760  u32 n_left_to_next;
3761 
3762  vlib_get_next_frame (vm, node, next_index,
3763  to_next, n_left_to_next);
3764 
3765  while (n_left_from > 0 && n_left_to_next > 0)
3766  {
3767  u32 bi0;
3768  vlib_buffer_t * b0;
3769  u32 next0;
3770  ip4_header_t * ip0;
3771  u32 proto0;
3772 
3773  /* speculatively enqueue b0 to the current next frame */
3774  bi0 = from[0];
3775  to_next[0] = bi0;
3776  from += 1;
3777  to_next += 1;
3778  n_left_from -= 1;
3779  n_left_to_next -= 1;
3780 
3781  b0 = vlib_get_buffer (vm, bi0);
3782  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3783  ip0 = vlib_buffer_get_current (b0);
3784 
3785  proto0 = ip_proto_to_snat_proto (ip0->protocol);
3786 
3787  vnet_buffer (b0)->snat.flags = 0;
3788  if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3789  {
3790  if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3791  {
3792  udp_header_t * udp0 = ip4_next_header (ip0);
3793  tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3794 
3795  snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3796  }
3797  else if (proto0 == SNAT_PROTOCOL_ICMP)
3798  {
3799  icmp46_header_t * icmp0 = ip4_next_header (ip0);
3800 
3801  snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3802  }
3803  else
3804  {
3805  snat_hairpinning_unknown_proto (sm, b0, ip0);
3806  }
3807 
3808  vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3809  }
3810 
3811  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3812 
3813  /* verify speculative enqueue, maybe switch current next frame */
3814  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3815  to_next, n_left_to_next,
3816  bi0, next0);
3817  }
3818 
3819  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3820  }
3821 
3823  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3824  pkts_processed);
3825  return frame->n_vectors;
3826 }
3827 
3829  .function = snat_hairpin_dst_fn,
3830  .name = "nat44-hairpin-dst",
3831  .vector_size = sizeof (u32),
3832  .type = VLIB_NODE_TYPE_INTERNAL,
3833  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3834  .error_strings = snat_in2out_error_strings,
3835  .n_next_nodes = 2,
3836  .next_nodes = {
3837  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3838  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3839  },
3840 };
3841 
3844 
3845 static uword
3847  vlib_node_runtime_t * node,
3848  vlib_frame_t * frame)
3849 {
3850  u32 n_left_from, * from, * to_next;
3851  snat_in2out_next_t next_index;
3852  u32 pkts_processed = 0;
3853  snat_main_t *sm = &snat_main;
3854 
3855  from = vlib_frame_vector_args (frame);
3856  n_left_from = frame->n_vectors;
3857  next_index = node->cached_next_index;
3858 
3859  while (n_left_from > 0)
3860  {
3861  u32 n_left_to_next;
3862 
3863  vlib_get_next_frame (vm, node, next_index,
3864  to_next, n_left_to_next);
3865 
3866  while (n_left_from > 0 && n_left_to_next > 0)
3867  {
3868  u32 bi0;
3869  vlib_buffer_t * b0;
3870  u32 next0;
3872  u32 sw_if_index0;
3873 
3874  /* speculatively enqueue b0 to the current next frame */
3875  bi0 = from[0];
3876  to_next[0] = bi0;
3877  from += 1;
3878  to_next += 1;
3879  n_left_from -= 1;
3880  n_left_to_next -= 1;
3881 
3882  b0 = vlib_get_buffer (vm, bi0);
3883  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3885 
3887  ({
3888  /* Only packets from NAT inside interface */
3889  if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3890  {
3891  if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3892  SNAT_FLAG_HAIRPINNING))
3893  {
3894  if (PREDICT_TRUE (sm->num_workers > 1))
3895  next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3896  else
3897  next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3898  }
3899  break;
3900  }
3901  }));
3902 
3903  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3904 
3905  /* verify speculative enqueue, maybe switch current next frame */
3906  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3907  to_next, n_left_to_next,
3908  bi0, next0);
3909  }
3910 
3911  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3912  }
3913 
3915  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3916  pkts_processed);
3917  return frame->n_vectors;
3918 }
3919 
3921  .function = snat_hairpin_src_fn,
3922  .name = "nat44-hairpin-src",
3923  .vector_size = sizeof (u32),
3924  .type = VLIB_NODE_TYPE_INTERNAL,
3925  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3926  .error_strings = snat_in2out_error_strings,
3927  .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3928  .next_nodes = {
3929  [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3930  [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3931  [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3932  [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3933  },
3934 };
3935 
3938 
3939 static uword
3941  vlib_node_runtime_t * node,
3942  vlib_frame_t * frame)
3943 {
3944  u32 n_left_from, * from, * to_next;
3945  snat_in2out_next_t next_index;
3946  u32 pkts_processed = 0;
3947  snat_main_t * sm = &snat_main;
3948  u32 stats_node_index;
3949 
3950  stats_node_index = snat_in2out_fast_node.index;
3951 
3952  from = vlib_frame_vector_args (frame);
3953  n_left_from = frame->n_vectors;
3954  next_index = node->cached_next_index;
3955 
3956  while (n_left_from > 0)
3957  {
3958  u32 n_left_to_next;
3959 
3960  vlib_get_next_frame (vm, node, next_index,
3961  to_next, n_left_to_next);
3962 
3963  while (n_left_from > 0 && n_left_to_next > 0)
3964  {
3965  u32 bi0;
3966  vlib_buffer_t * b0;
3967  u32 next0;
3968  u32 sw_if_index0;
3969  ip4_header_t * ip0;
3970  ip_csum_t sum0;
3971  u32 new_addr0, old_addr0;
3972  u16 old_port0, new_port0;
3973  udp_header_t * udp0;
3974  tcp_header_t * tcp0;
3975  icmp46_header_t * icmp0;
3976  snat_session_key_t key0, sm0;
3977  u32 proto0;
3978  u32 rx_fib_index0;
3979 
3980  /* speculatively enqueue b0 to the current next frame */
3981  bi0 = from[0];
3982  to_next[0] = bi0;
3983  from += 1;
3984  to_next += 1;
3985  n_left_from -= 1;
3986  n_left_to_next -= 1;
3987 
3988  b0 = vlib_get_buffer (vm, bi0);
3989  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3990 
3991  ip0 = vlib_buffer_get_current (b0);
3992  udp0 = ip4_next_header (ip0);
3993  tcp0 = (tcp_header_t *) udp0;
3994  icmp0 = (icmp46_header_t *) udp0;
3995 
3996  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3997  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3998 
3999  if (PREDICT_FALSE(ip0->ttl == 1))
4000  {
4001  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4002  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4003  ICMP4_time_exceeded_ttl_exceeded_in_transit,
4004  0);
4006  goto trace0;
4007  }
4008 
4009  proto0 = ip_proto_to_snat_proto (ip0->protocol);
4010 
4011  if (PREDICT_FALSE (proto0 == ~0))
4012  goto trace0;
4013 
4014  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4015  {
4016  next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4017  rx_fib_index0, node, next0, ~0, 0, 0);
4018  goto trace0;
4019  }
4020 
4021  key0.addr = ip0->src_address;
4022  key0.protocol = proto0;
4023  key0.port = udp0->src_port;
4024  key0.fib_index = rx_fib_index0;
4025 
4026  if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0))
4027  {
4028  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4029  next0= SNAT_IN2OUT_NEXT_DROP;
4030  goto trace0;
4031  }
4032 
4033  new_addr0 = sm0.addr.as_u32;
4034  new_port0 = sm0.port;
4035  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4036  old_addr0 = ip0->src_address.as_u32;
4037  ip0->src_address.as_u32 = new_addr0;
4038 
4039  sum0 = ip0->checksum;
4040  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4041  ip4_header_t,
4042  src_address /* changed member */);
4043  ip0->checksum = ip_csum_fold (sum0);
4044 
4045  if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4046  {
4047  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4048  {
4049  old_port0 = tcp0->src_port;
4050  tcp0->src_port = new_port0;
4051 
4052  sum0 = tcp0->checksum;
4053  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4054  ip4_header_t,
4055  dst_address /* changed member */);
4056  sum0 = ip_csum_update (sum0, old_port0, new_port0,
4057  ip4_header_t /* cheat */,
4058  length /* changed member */);
4059  tcp0->checksum = ip_csum_fold(sum0);
4060  }
4061  else
4062  {
4063  old_port0 = udp0->src_port;
4064  udp0->src_port = new_port0;
4065  udp0->checksum = 0;
4066  }
4067  }
4068  else
4069  {
4070  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4071  {
4072  sum0 = tcp0->checksum;
4073  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4074  ip4_header_t,
4075  dst_address /* changed member */);
4076  tcp0->checksum = ip_csum_fold(sum0);
4077  }
4078  }
4079 
4080  /* Hairpinning */
4081  snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4082 
4083  trace0:
4085  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4086  {
4087  snat_in2out_trace_t *t =
4088  vlib_add_trace (vm, node, b0, sizeof (*t));
4089  t->sw_if_index = sw_if_index0;
4090  t->next_index = next0;
4091  }
4092 
4093  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4094 
4095  /* verify speculative enqueue, maybe switch current next frame */
4096  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4097  to_next, n_left_to_next,
4098  bi0, next0);
4099  }
4100 
4101  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4102  }
4103 
4104  vlib_node_increment_counter (vm, stats_node_index,
4105  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4106  pkts_processed);
4107  return frame->n_vectors;
4108 }
4109 
4110 
4112  .function = snat_in2out_fast_static_map_fn,
4113  .name = "nat44-in2out-fast",
4114  .vector_size = sizeof (u32),
4115  .format_trace = format_snat_in2out_fast_trace,
4116  .type = VLIB_NODE_TYPE_INTERNAL,
4117 
4118  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4119  .error_strings = snat_in2out_error_strings,
4120 
4121  .runtime_data_bytes = sizeof (snat_runtime_t),
4122 
4123  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4124 
4125  /* edit / add dispositions here */
4126  .next_nodes = {
4127  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4128  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4129  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4130  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4131  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4132  },
4133 };
4134 
ip4_address_t external_addr
Definition: nat.h:215
vnet_config_main_t config_main
Definition: feature.h:65
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:432
static snat_session_t * snat_in2out_unknown_proto(snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip, u32 rx_fib_index, u32 thread_index, f64 now, vlib_main_t *vm, vlib_node_runtime_t *node)
Definition: in2out.c:1125
u32 next
Definition: dlist.h:30
fib_protocol_t fp_proto
protocol type
Definition: fib_types.h:181
u32 sessions_per_user_list_head_index
Definition: nat.h:171
static u8 * format_snat_in2out_trace(u8 *s, va_list *args)
Definition: in2out.c:46
clib_bihash_16_8_t out2in_ed
Definition: nat.h:289
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
#define CLIB_UNUSED(x)
Definition: clib.h:79
snat_in2out_error_t
Definition: in2out.c:132
u16 ext_host_port
Definition: nat.h:79
static uword snat_in2out_slow_path_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:2231
u16 out_port
Definition: nat.h:80
u32 icmp_timeout
Definition: nat.h:375
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:116
ip4_address_t src_address
Definition: ip4_packet.h:164
static int nat_not_translate_output_feature(snat_main_t *sm, ip4_header_t *ip0, u32 proto0, u16 src_port, u16 dst_port, u32 thread_index, u32 sw_if_index)
Definition: in2out.c:253
u32 fq_in2out_output_index
Definition: nat.h:342
static u8 * format_snat_in2out_fast_trace(u8 *s, va_list *args)
Definition: in2out.c:61
#define TCP_FLAG_SYN
Definition: fa_node.h:8
static uword snat_in2out_node_fn_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_slow_path, int is_output_feature)
Definition: in2out.c:1459
#define PREDICT_TRUE(x)
Definition: clib.h:106
static int ip4_header_bytes(ip4_header_t *i)
Definition: ip4_packet.h:227
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:224
static char * snat_in2out_error_strings[]
Definition: in2out.c:139
u32 fib_table_get_index_for_sw_if_index(fib_protocol_t proto, u32 sw_if_index)
Get the index of the FIB bound to the interface.
Definition: fib_table.c:931
int nat_ip4_reass_add_fragment(nat_reass_ip4_t *reass, u32 bi)
Cache fragment.
Definition: nat_reass.c:328
u16 port_per_thread
Definition: nat.h:301
static uword snat_in2out_worker_handoff_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:3671
static void snat_icmp_hairpinning(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, icmp46_header_t *icmp0)
Definition: in2out.c:971
static void snat_det_ses_close(snat_det_map_t *dm, snat_det_session_t *ses)
Definition: nat_det.h:179
struct _vlib_node_registration vlib_node_registration_t
static snat_det_session_t * snat_det_find_ses_by_in(snat_det_map_t *dm, ip4_address_t *in_addr, u16 in_port, snat_det_out_key_t out_key)
Definition: nat_det.h:129
static void snat_det_forward(snat_det_map_t *dm, ip4_address_t *in_addr, ip4_address_t *out_addr, u16 *lo_port)
Definition: nat_det.h:75
uword ip_csum_t
Definition: ip_packet.h:90
snat_in2out_next_t
Definition: in2out.c:145
u32 * fib_index_by_sw_if_index
Table index indexed by software interface.
Definition: ip4.h:111
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
clib_bihash_16_8_t in2out_ed
Definition: nat.h:290
int snat_static_mapping_match(snat_main_t *sm, snat_session_key_t match, snat_session_key_t *mapping, u8 by_external, u8 *is_addr_only, u8 *twice_nat)
Match NAT44 static mapping.
Definition: nat.c:1785
u32 buffer_index[VLIB_FRAME_SIZE]
Definition: threads.h:98
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:415
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:107
dlist_elt_t * list_pool
Definition: nat.h:259
struct _tcp_header tcp_header_t
u32 proto
Definition: nat.h:64
VLIB_NODE_FUNCTION_MULTIARCH(snat_in2out_node, snat_in2out_fast_path_fn)
u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node, u32 thread_index, vlib_buffer_t *b0, ip4_header_t *ip0, u8 *p_proto, snat_session_key_t *p_value, u8 *p_dont_translate, void *d, void *e)
Get address and port values to be used for ICMP packet translation.
Definition: in2out.c:673
u16 l_port
Definition: nat.h:66
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_bihash_8_8_t user_hash
Definition: nat.h:250
u16 identifier
Definition: nat.h:474
vlib_node_registration_t snat_hairpin_dst_node
(constructor) VLIB_REGISTER_NODE (snat_hairpin_dst_node)
Definition: in2out.c:114
static uword snat_in2out_output_slow_path_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:2266
clib_bihash_8_8_t in2out
Definition: nat.h:247
vlib_node_registration_t snat_in2out_output_slowpath_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node)
Definition: in2out.c:112
static void user_session_increment(snat_main_t *sm, snat_user_t *u, u8 is_static)
Definition: nat.h:645
u32 in2out_output_node_index
Definition: nat.h:347
u32 ip4_fib_table_get_index_for_sw_if_index(u32 sw_if_index)
Definition: ip4_fib.c:226
format_function_t format_ip4_address
Definition: format.h:79
#define static_always_inline
Definition: clib.h:93
static uword ip4_header_checksum_is_valid(ip4_header_t *i)
Definition: ip4_packet.h:260
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:438
vlib_node_registration_t snat_in2out_output_worker_handoff_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node)
Definition: in2out.c:113
u16 r_port
Definition: nat.h:67
ip_csum_t ip_incremental_checksum(ip_csum_t sum, void *_data, uword n_bytes)
Definition: ip_checksum.c:43
vlib_node_registration_t snat_in2out_fast_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_fast_node)
Definition: in2out.c:108
static uword snat_in2out_output_worker_handoff_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:3696
ip4_address_t ext_host_addr
Definition: nat.h:78
static int snat_not_translate(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0, ip4_header_t *ip0, u32 proto0, u32 rx_fib_index0, u32 thread_index)
Definition: in2out.c:219
ip4_address_t dst_address
Definition: ip4_packet.h:164
#define TCP_FLAG_ACK
Definition: fa_node.h:11
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:171
static uword snat_det_in2out_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:2757
Aggregrate type for a prefix.
Definition: fib_types.h:172
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:182
static u8 * format_snat_in2out_worker_handoff_trace(u8 *s, va_list *args)
Definition: in2out.c:73
ip4_main_t * ip4_main
Definition: nat.h:383
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:233
static u32 slow_path(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, u32 rx_fib_index0, snat_session_key_t *key0, snat_session_t **sessionp, vlib_node_runtime_t *node, u32 next0, u32 thread_index)
Definition: in2out.c:293
vlib_node_registration_t snat_in2out_slowpath_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_slowpath_node)
Definition: in2out.c:107
ip4_address_t local_addr
Definition: nat.h:214
static uword snat_in2out_worker_handoff_fn_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, u8 is_output)
Definition: in2out.c:3513
fib_node_index_t fib_table_lookup(u32 fib_index, const fib_prefix_t *prefix)
Perfom a longest prefix match in the non-forwarding table.
Definition: fib_table.c:66
u64 as_u64[2]
Definition: nat.h:69
static u8 * format_nat44_in2out_reass_trace(u8 *s, va_list *args)
Definition: in2out.c:93
#define foreach_snat_in2out_error
Definition: in2out.c:120
static int ip4_is_fragment(ip4_header_t *i)
Definition: ip4_packet.h:205
static void nat44_reass_hairpinning(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, u16 sport, u16 dport, u32 proto0)
Definition: in2out.c:2392
u32 fib_index
Definition: nat.h:64
#define hash_get(h, key)
Definition: hash.h:248
snat_user_t * nat_user_get_or_create(snat_main_t *sm, ip4_address_t *addr, u32 fib_index, u32 thread_index)
Definition: nat.c:239
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:459
uword * fib_index_by_table_id
Hash table mapping table id to fib index.
Definition: ip4.h:121
static uword nat44_in2out_reass_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:2487
snat_det_session_t * sessions
Definition: nat.h:203
u64 key
the key
Definition: bihash_8_8.h:35
static u32 icmp_in2out_slow_path(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, icmp46_header_t *icmp0, u32 sw_if_index0, u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0, f64 now, u32 thread_index, snat_session_t **p_s0)
Definition: in2out.c:1036
vlib_main_t * vlib_main
Definition: nat.h:381
static void clib_dlist_addtail(dlist_elt_t *pool, u32 head_index, u32 new_index)
Definition: dlist.h:43
u16 protocol
Definition: nat.h:50
snat_static_mapping_t * static_mappings
Definition: nat.h:314
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:191
u32 inside_fib_index
Definition: nat.h:369
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:195
static_always_inline int icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
Definition: in2out.c:442
static void * vnet_get_config_data(vnet_config_main_t *cm, u32 *config_index, u32 *next_index, u32 n_data_bytes)
Definition: config.h:122
u32 udp_timeout
Definition: nat.h:372
#define PREDICT_FALSE(x)
Definition: clib.h:105
void snat_ipfix_logging_nat44_ses_create(u32 src_ip, u32 nat_src_ip, snat_protocol_t snat_proto, u16 src_port, u16 nat_src_port, u32 vrf_id)
Generate NAT44 session create event.
static vlib_frame_queue_elt_t * vlib_get_worker_handoff_queue_elt(u32 frame_queue_index, u32 vlib_worker_index, vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index)
Definition: threads.h:502
clib_bihash_8_8_t static_mapping_by_external
Definition: nat.h:311
#define TCP_FLAG_FIN
Definition: fa_node.h:7
#define VLIB_FRAME_SIZE
Definition: node.h:328
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
#define SNAT_SESSION_FLAG_UNKNOWN_PROTO
Definition: nat.h:129
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:364
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:113
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1158
u32 fib_entry_get_resolving_interface(fib_node_index_t entry_index)
Definition: fib_entry.c:1352
snat_interface_t * output_feature_interfaces
Definition: nat.h:318
snat_main_t snat_main
Definition: nat.c:33
static void snat_hairpinning_unknown_proto(snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip)
Definition: in2out.c:1070
The fine-grained event logger allows lightweight, thread-safe event logging at minimum cost...
u64 value
the value
Definition: bihash_8_8.h:36
static_always_inline int is_hairpinning(snat_main_t *sm, ip4_address_t *dst_addr)
Definition: in2out.c:3721
vlib_node_registration_t nat44_in2out_reass_node
(constructor) VLIB_REGISTER_NODE (nat44_in2out_reass_node)
Definition: in2out.c:117
u16 n_vectors
Definition: node.h:344
clib_bihash_8_8_t out2in
Definition: nat.h:246
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:221
u8 nat_reass_is_drop_frag(u8 is_ip6)
Get status of virtual fragmentation reassembly.
Definition: nat_reass.c:167
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
vlib_main_t * vm
Definition: buffer.c:283
static uword snat_in2out_output_fast_path_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:2196
u32 outside_vrf_id
Definition: nat.h:366
vlib_node_registration_t snat_in2out_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_node)
Definition: in2out.c:106
ip4_address_t l_addr
Definition: nat.h:62
snat_get_worker_function_t * worker_in2out_cb
Definition: nat.h:299
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:336
void icmp4_error_set_vnet_buffer(vlib_buffer_t *b, u8 type, u8 code, u32 data)
Definition: icmp4.c:431
deterministic NAT definitions
#define clib_warning(format, args...)
Definition: error.h:59
#define VLIB_BUFFER_IS_TRACED
Definition: buffer.h:93
static int nat_not_translate_output_feature_fwd(snat_main_t *sm, ip4_header_t *ip)
Definition: in2out.c:489
#define clib_memcpy(a, b, c)
Definition: string.h:75
static int ip4_is_first_fragment(ip4_header_t *i)
Definition: ip4_packet.h:212
u32 outside_fib_index
Definition: nat.h:367
static uword snat_in2out_fast_static_map_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:3940
u32 fib_node_index_t
A typedef of a node index.
Definition: fib_types.h:29
8 octet key, 8 octet key value pair
Definition: bihash_8_8.h:33
#define ARRAY_LEN(x)
Definition: clib.h:59
ip4_address_t addr
Definition: nat.h:48
static_always_inline snat_in2out_error_t icmp_get_key(ip4_header_t *ip0, snat_session_key_t *p_key0)
Definition: in2out.c:397
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:454
u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node, u32 thread_index, vlib_buffer_t *b0, ip4_header_t *ip0, u8 *p_proto, snat_session_key_t *p_value, u8 *p_dont_translate, void *d, void *e)
Get address and port values to be used for ICMP packet translation and create session if needed...
Definition: in2out.c:543
u32 tcp_transitory_timeout
Definition: nat.h:374
ip4_address_t r_addr
Definition: nat.h:63
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:456
static snat_det_map_t * snat_det_map_by_user(snat_main_t *sm, ip4_address_t *user_addr)
Definition: nat_det.h:45
static uword nat44_hairpinning_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:2303
#define ASSERT(truth)
u32 num_workers
Definition: nat.h:295
unsigned int u32
Definition: types.h:88
u32 first_worker_index
Definition: nat.h:296
snat_get_worker_function_t * worker_out2in_cb
Definition: nat.h:300
static u32 ip_proto_to_snat_proto(u8 ip_proto)
Definition: nat.h:479
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:259
u32 fq_in2out_index
Definition: nat.h:341
static void clib_dlist_remove(dlist_elt_t *pool, u32 index)
Definition: dlist.h:99
static int snat_not_translate_fast(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0, ip4_header_t *ip0, u32 proto0, u32 rx_fib_index0)
Check if packet should be translated.
Definition: in2out.c:178
ip4_address_t addr
Definition: nat.h:177
vlib_node_registration_t nat44_hairpinning_node
(constructor) VLIB_REGISTER_NODE (nat44_hairpinning_node)
Definition: in2out.c:116
u32 value
Definition: dlist.h:32
u64 uword
Definition: types.h:112
snat_session_t * nat_session_alloc_or_recycle(snat_main_t *sm, snat_user_t *u, u32 thread_index)
Definition: nat.c:283
vlib_node_registration_t snat_det_in2out_node
(constructor) VLIB_REGISTER_NODE (snat_det_in2out_node)
Definition: in2out.c:110
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
#define vec_elt(v, i)
Get vector value at index i.
Definition: defs.h:47
unsigned short u16
Definition: types.h:57
static uword snat_hairpin_src_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:3846
#define FIB_NODE_INDEX_INVALID
Definition: fib_types.h:30
int snat_alloc_outside_address_and_port(snat_address_t *addresses, u32 fib_index, u32 thread_index, snat_session_key_t *k, u32 *address_indexp, u16 port_per_thread, u32 snat_thread_index)
Definition: nat.c:1873
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
unsigned char u8
Definition: types.h:56
u16 ports_per_host
Definition: nat.h:200
vlib_node_registration_t snat_in2out_worker_handoff_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node)
Definition: in2out.c:109
static_always_inline u8 is_interface_addr(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0, u32 ip4_addr)
Definition: nat.h:585
u32 * workers
Definition: nat.h:298
snat_main_per_thread_data_t * per_thread_data
Definition: nat.h:305
#define SNAT_FLAG_HAIRPINNING
Definition: nat.h:40
static int snat_hairpinning(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, udp_header_t *udp0, tcp_header_t *tcp0, u32 proto0)
Hairpinning.
Definition: in2out.c:880
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:267
#define ip_csum_update(sum, old, new, type, field)
Definition: ip_packet.h:139
snat_det_out_key_t out
Definition: nat.h:189
vlib_node_registration_t snat_hairpin_src_node
(constructor) VLIB_REGISTER_NODE (snat_hairpin_src_node)
Definition: in2out.c:115
static snat_session_t * snat_in2out_lb(snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip, u32 rx_fib_index, u32 thread_index, f64 now, vlib_main_t *vm, vlib_node_runtime_t *node)
Definition: in2out.c:1310
nat_reass_ip4_t * nat_ip4_reass_find_or_create(ip4_address_t src, ip4_address_t dst, u16 frag_id, u8 proto, u8 reset_timeout, u32 **bi_to_drop)
Find or create reassembly.
Definition: nat_reass.c:219
snat_address_t * addresses
Definition: nat.h:321
static void vlib_put_frame_queue_elt(vlib_frame_queue_elt_t *hf)
Definition: threads.h:432
u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node, u32 thread_index, vlib_buffer_t *b0, ip4_header_t *ip0, u8 *p_proto, snat_session_key_t *p_value, u8 *p_dont_translate, void *d, void *e)
Get address and port values to be used for ICMP packet translation and create session if needed...
Definition: in2out.c:3366
struct _vnet_feature_arc_registration vnet_feature_arc_registration_t
feature registration object
vnet_feature_arc_registration_t vnet_feat_arc_ip4_local
static uword snat_in2out_fast_path_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:2162
u32 in2out_node_index
Definition: nat.h:346
static snat_det_session_t * snat_det_get_ses_by_out(snat_det_map_t *dm, ip4_address_t *in_addr, u64 out_key)
Definition: nat_det.h:112
#define vnet_buffer(b)
Definition: buffer.h:326
#define SNAT_SESSION_FLAG_STATIC_MAPPING
Definition: nat.h:128
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
u8 data[0]
Packet data.
Definition: buffer.h:159
u8 forwarding_enabled
Definition: nat.h:354
#define vec_foreach(var, vec)
Vector iterator.
vlib_node_registration_t snat_in2out_output_node
(constructor) VLIB_REGISTER_NODE (snat_in2out_output_node)
Definition: in2out.c:111
static_always_inline void nat_send_all_to_node(vlib_main_t *vm, u32 *bi_vector, vlib_node_runtime_t *node, vlib_error_t *error, u32 next)
Definition: nat.h:619
u16 flags
Copy of main node flags.
Definition: node.h:450
#define is_twice_nat_session(s)
Check if NAT session is twice NAT.
Definition: nat.h:450
static uword snat_hairpin_dst_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:3745
snat_hairpin_next_t
Definition: in2out.c:154
void nat_ip4_reass_get_frags(nat_reass_ip4_t *reass, u32 **bi)
Get cached fragments.
Definition: nat_reass.c:353
NAT plugin virtual fragmentation reassembly.
vnet_feature_config_main_t * feature_config_mains
feature config main objects
Definition: feature.h:81
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:481
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:75
static snat_det_session_t * snat_det_ses_create(snat_det_map_t *dm, ip4_address_t *in_addr, u16 in_port, snat_det_out_key_t *out)
Definition: nat_det.h:150
static u8 maximum_sessions_exceeded(snat_main_t *sm, u32 thread_index)
Definition: nat.h:610
vnet_feature_main_t feature_main
Definition: feature.c:19
snat_session_t * sessions
Definition: nat.h:256
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
snat_icmp_match_function_t * icmp_match_in2out_cb
Definition: nat.h:292
#define SNAT_SESSION_FLAG_LOAD_BALANCING
Definition: nat.h:130
clib_bihash_8_8_t static_mapping_by_local
Definition: nat.h:308
static u16 ip_csum_fold(ip_csum_t c)
Definition: ip_packet.h:145
snat_interface_t * interfaces
Definition: nat.h:317
Definition: defs.h:46
u16 fib_index
Definition: nat.h:50
static_always_inline u8 icmp_is_error_message(icmp46_header_t *icmp)
Definition: nat.h:569
u32 tcp_established_timeout
Definition: nat.h:373
static u32 icmp_in2out(snat_main_t *sm, vlib_buffer_t *b0, ip4_header_t *ip0, icmp46_header_t *icmp0, u32 sw_if_index0, u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0, u32 thread_index, void *d, void *e)
Definition: in2out.c:739