FD.io VPP  v18.07.1-13-g909ba93
Vector Packet Processing
in2out.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
20 
21 #include <vnet/ip/ip.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/fib/ip4_fib.h>
24 #include <nat/nat.h>
25 #include <nat/nat_ipfix_logging.h>
26 #include <nat/nat_det.h>
27 #include <nat/nat_reass.h>
28 #include <nat/nat_inlines.h>
29 
30 #include <vppinfra/hash.h>
31 #include <vppinfra/error.h>
32 #include <vppinfra/elog.h>
33 
34 typedef struct {
40 
41 typedef struct {
45 
46 /* packet trace format function */
47 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
48 {
49  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
50  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
51  snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
52  char * tag;
53 
54  tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
55 
56  s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
58 
59  return s;
60 }
61 
62 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
63 {
64  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
65  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
66  snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
67 
68  s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
69  t->sw_if_index, t->next_index);
70 
71  return s;
72 }
73 
74 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
75 {
76  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
77  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
79  va_arg (*args, snat_in2out_worker_handoff_trace_t *);
80  char * m;
81 
82  m = t->do_handoff ? "next worker" : "same worker";
83  s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
84 
85  return s;
86 }
87 
88 typedef struct {
93 
94 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
95 {
96  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
97  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
99 
100  s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
101  t->sw_if_index, t->next_index,
102  t->cached ? "cached" : "translated");
103 
104  return s;
105 }
106 
126 
127 #define foreach_snat_in2out_error \
128 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
129 _(IN2OUT_PACKETS, "Good in2out packets processed") \
130 _(OUT_OF_PORTS, "Out of ports") \
131 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \
132 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
133 _(NO_TRANSLATION, "No translation") \
134 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
135 _(DROP_FRAGMENT, "Drop fragment") \
136 _(MAX_REASS, "Maximum reassemblies exceeded") \
137 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
138 _(FQ_CONGESTED, "Handoff frame queue congested")
139 
140 typedef enum {
141 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
143 #undef _
146 
147 static char * snat_in2out_error_strings[] = {
148 #define _(sym,string) string,
150 #undef _
151 };
152 
153 typedef enum {
161 
162 typedef enum {
169 
170 /**
171  * @brief Check if packet should be translated
172  *
173  * Packets aimed at outside interface and external address with active session
174  * should be translated.
175  *
176  * @param sm NAT main
177  * @param rt NAT runtime data
178  * @param sw_if_index0 index of the inside interface
179  * @param ip0 IPv4 header
180  * @param proto0 NAT protocol
181  * @param rx_fib_index0 RX FIB index
182  *
183  * @returns 0 if packet should be translated otherwise 1
184  */
185 static inline int
187  u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
188  u32 rx_fib_index0)
189 {
190  if (sm->out2in_dpo)
191  return 0;
192 
194  nat_outside_fib_t *outside_fib;
195  fib_prefix_t pfx = {
197  .fp_len = 32,
198  .fp_addr = {
199  .ip4.as_u32 = ip0->dst_address.as_u32,
200  },
201  };
202 
203  /* Don't NAT packet aimed at the intfc address */
204  if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
205  ip0->dst_address.as_u32)))
206  return 1;
207 
208  fei = fib_table_lookup (rx_fib_index0, &pfx);
209  if (FIB_NODE_INDEX_INVALID != fei)
210  {
211  u32 sw_if_index = fib_entry_get_resolving_interface (fei);
212  if (sw_if_index == ~0)
213  {
214  vec_foreach (outside_fib, sm->outside_fibs)
215  {
216  fei = fib_table_lookup (outside_fib->fib_index, &pfx);
217  if (FIB_NODE_INDEX_INVALID != fei)
218  {
219  sw_if_index = fib_entry_get_resolving_interface (fei);
220  if (sw_if_index != ~0)
221  break;
222  }
223  }
224  }
225  if (sw_if_index == ~0)
226  return 1;
227 
229  pool_foreach (i, sm->interfaces,
230  ({
231  /* NAT packet aimed at outside interface */
232  if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
233  return 0;
234  }));
235  }
236 
237  return 1;
238 }
239 
240 static inline int
242  u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
243  u32 rx_fib_index0, u32 thread_index)
244 {
245  udp_header_t * udp0 = ip4_next_header (ip0);
246  snat_session_key_t key0, sm0;
247  clib_bihash_kv_8_8_t kv0, value0;
248 
249  key0.addr = ip0->dst_address;
250  key0.port = udp0->dst_port;
251  key0.protocol = proto0;
252  key0.fib_index = sm->outside_fib_index;
253  kv0.key = key0.as_u64;
254 
255  /* NAT packet aimed at external address if */
256  /* has active sessions */
257  if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
258  &value0))
259  {
260  /* or is static mappings */
261  if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
262  return 0;
263  }
264  else
265  return 0;
266 
267  if (sm->forwarding_enabled)
268  return 1;
269 
270  return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
271  rx_fib_index0);
272 }
273 
274 static inline int
276  u32 proto0, u16 src_port, u16 dst_port,
277  u32 thread_index, u32 sw_if_index)
278 {
279  snat_session_key_t key0;
280  clib_bihash_kv_8_8_t kv0, value0;
282 
283  /* src NAT check */
284  key0.addr = ip0->src_address;
285  key0.port = src_port;
286  key0.protocol = proto0;
288  kv0.key = key0.as_u64;
289 
290  if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
291  &value0))
292  return 1;
293 
294  /* dst NAT check */
295  key0.addr = ip0->dst_address;
296  key0.port = dst_port;
297  key0.protocol = proto0;
298  kv0.key = key0.as_u64;
299  if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
300  &value0))
301  {
302  /* hairpinning */
304  ({
305  if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
306  return 0;
307  }));
308  return 1;
309  }
310 
311  return 0;
312 }
313 
315  ip4_header_t * ip0,
316  u32 rx_fib_index0,
317  snat_session_key_t * key0,
318  snat_session_t ** sessionp,
319  vlib_node_runtime_t * node,
320  u32 next0,
321  u32 thread_index)
322 {
323  snat_user_t *u;
324  snat_session_t *s;
326  snat_session_key_t key1;
327  u32 address_index = ~0;
328  udp_header_t * udp0 = ip4_next_header (ip0);
329  u8 is_sm = 0;
330  nat_outside_fib_t *outside_fib;
332  fib_prefix_t pfx = {
334  .fp_len = 32,
335  .fp_addr = {
336  .ip4.as_u32 = ip0->dst_address.as_u32,
337  },
338  };
339 
340  if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
341  {
342  b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
344  nat_log_notice ("maximum sessions exceeded");
345  return SNAT_IN2OUT_NEXT_DROP;
346  }
347 
348  key1.protocol = key0->protocol;
349 
350  u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
351  thread_index);
352  if (!u)
353  {
354  nat_log_warn ("create NAT user failed");
355  return SNAT_IN2OUT_NEXT_DROP;
356  }
357 
358  /* First try to match static mapping by local address and port */
359  if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
360  {
361  /* Try to create dynamic translation */
362  if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
363  thread_index, &key1,
364  &address_index,
365  sm->port_per_thread,
366  sm->per_thread_data[thread_index].snat_thread_index))
367  {
368  b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
369  return SNAT_IN2OUT_NEXT_DROP;
370  }
371  }
372  else
373  is_sm = 1;
374 
375  s = nat_session_alloc_or_recycle (sm, u, thread_index);
376  if (!s)
377  {
378  nat_log_warn ("create NAT session failed");
379  return SNAT_IN2OUT_NEXT_DROP;
380  }
381 
382  if (is_sm)
384  user_session_increment (sm, u, is_sm);
385  s->outside_address_index = address_index;
386  s->in2out = *key0;
387  s->out2in = key1;
388  s->out2in.protocol = key0->protocol;
389  s->out2in.fib_index = sm->outside_fib_index;
390  switch (vec_len (sm->outside_fibs))
391  {
392  case 0:
393  s->out2in.fib_index = sm->outside_fib_index;
394  break;
395  case 1:
396  s->out2in.fib_index = sm->outside_fibs[0].fib_index;
397  break;
398  default:
399  vec_foreach (outside_fib, sm->outside_fibs)
400  {
401  fei = fib_table_lookup (outside_fib->fib_index, &pfx);
402  if (FIB_NODE_INDEX_INVALID != fei)
403  {
404  if (fib_entry_get_resolving_interface (fei) != ~0)
405  {
406  s->out2in.fib_index = outside_fib->fib_index;
407  break;
408  }
409  }
410  }
411  break;
412  }
413  s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
414  s->ext_host_port = udp0->dst_port;
415  *sessionp = s;
416 
417  /* Add to translation hashes */
418  kv0.key = s->in2out.as_u64;
419  kv0.value = s - sm->per_thread_data[thread_index].sessions;
420  if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
421  1 /* is_add */))
422  nat_log_notice ("in2out key add failed");
423 
424  kv0.key = s->out2in.as_u64;
425  kv0.value = s - sm->per_thread_data[thread_index].sessions;
426 
427  if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
428  1 /* is_add */))
429  nat_log_notice ("out2in key add failed");
430 
431  /* log NAT event */
432  snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
433  s->out2in.addr.as_u32,
434  s->in2out.protocol,
435  s->in2out.port,
436  s->out2in.port,
437  s->in2out.fib_index);
438  return next0;
439 }
440 
443  snat_session_key_t *p_key0)
444 {
445  icmp46_header_t *icmp0;
446  snat_session_key_t key0;
447  icmp_echo_header_t *echo0, *inner_echo0 = 0;
448  ip4_header_t *inner_ip0 = 0;
449  void *l4_header = 0;
450  icmp46_header_t *inner_icmp0;
451 
452  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
453  echo0 = (icmp_echo_header_t *)(icmp0+1);
454 
455  if (!icmp_is_error_message (icmp0))
456  {
457  key0.protocol = SNAT_PROTOCOL_ICMP;
458  key0.addr = ip0->src_address;
459  key0.port = echo0->identifier;
460  }
461  else
462  {
463  inner_ip0 = (ip4_header_t *)(echo0+1);
464  l4_header = ip4_next_header (inner_ip0);
465  key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
466  key0.addr = inner_ip0->dst_address;
467  switch (key0.protocol)
468  {
469  case SNAT_PROTOCOL_ICMP:
470  inner_icmp0 = (icmp46_header_t*)l4_header;
471  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
472  key0.port = inner_echo0->identifier;
473  break;
474  case SNAT_PROTOCOL_UDP:
475  case SNAT_PROTOCOL_TCP:
476  key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
477  break;
478  default:
479  return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
480  }
481  }
482  *p_key0 = key0;
483  return -1; /* success */
484 }
485 
486 /**
487  * Get address and port values to be used for ICMP packet translation
488  * and create session if needed
489  *
490  * @param[in,out] sm NAT main
491  * @param[in,out] node NAT node runtime
492  * @param[in] thread_index thread index
493  * @param[in,out] b0 buffer containing packet to be translated
494  * @param[out] p_proto protocol used for matching
495  * @param[out] p_value address and port after NAT translation
496  * @param[out] p_dont_translate if packet should not be translated
497  * @param d optional parameter
498  * @param e optional parameter
499  */
501  u32 thread_index, vlib_buffer_t *b0,
502  ip4_header_t *ip0, u8 *p_proto,
503  snat_session_key_t *p_value,
504  u8 *p_dont_translate, void *d, void *e)
505 {
506  icmp46_header_t *icmp0;
507  u32 sw_if_index0;
508  u32 rx_fib_index0;
509  snat_session_key_t key0;
510  snat_session_t *s0 = 0;
511  u8 dont_translate = 0;
512  clib_bihash_kv_8_8_t kv0, value0;
513  u32 next0 = ~0;
514  int err;
515 
516  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
517  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
518  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
519 
520  err = icmp_get_key (ip0, &key0);
521  if (err != -1)
522  {
523  b0->error = node->errors[err];
524  next0 = SNAT_IN2OUT_NEXT_DROP;
525  goto out;
526  }
527  key0.fib_index = rx_fib_index0;
528 
529  kv0.key = key0.as_u64;
530 
531  if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
532  &value0))
533  {
534  if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
535  {
537  key0.protocol, key0.port, key0.port, thread_index, sw_if_index0)))
538  {
539  dont_translate = 1;
540  goto out;
541  }
542  }
543  else
544  {
545  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
546  ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
547  {
548  dont_translate = 1;
549  goto out;
550  }
551  }
552 
554  {
555  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
556  next0 = SNAT_IN2OUT_NEXT_DROP;
557  goto out;
558  }
559 
560  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
561  &s0, node, next0, thread_index);
562 
563  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
564  goto out;
565  }
566  else
567  {
568  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
569  icmp0->type != ICMP4_echo_reply &&
570  !icmp_is_error_message (icmp0)))
571  {
572  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
573  next0 = SNAT_IN2OUT_NEXT_DROP;
574  goto out;
575  }
576 
577  s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
578  value0.value);
579  }
580 
581 out:
582  *p_proto = key0.protocol;
583  if (s0)
584  *p_value = s0->out2in;
585  *p_dont_translate = dont_translate;
586  if (d)
587  *(snat_session_t**)d = s0;
588  return next0;
589 }
590 
591 /**
592  * Get address and port values to be used for ICMP packet translation
593  *
594  * @param[in] sm NAT main
595  * @param[in,out] node NAT node runtime
596  * @param[in] thread_index thread index
597  * @param[in,out] b0 buffer containing packet to be translated
598  * @param[out] p_proto protocol used for matching
599  * @param[out] p_value address and port after NAT translation
600  * @param[out] p_dont_translate if packet should not be translated
601  * @param d optional parameter
602  * @param e optional parameter
603  */
605  u32 thread_index, vlib_buffer_t *b0,
606  ip4_header_t *ip0, u8 *p_proto,
607  snat_session_key_t *p_value,
608  u8 *p_dont_translate, void *d, void *e)
609 {
610  icmp46_header_t *icmp0;
611  u32 sw_if_index0;
612  u32 rx_fib_index0;
613  snat_session_key_t key0;
614  snat_session_key_t sm0;
615  u8 dont_translate = 0;
616  u8 is_addr_only;
617  u32 next0 = ~0;
618  int err;
619 
620  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
621  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
622  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
623 
624  err = icmp_get_key (ip0, &key0);
625  if (err != -1)
626  {
627  b0->error = node->errors[err];
628  next0 = SNAT_IN2OUT_NEXT_DROP;
629  goto out2;
630  }
631  key0.fib_index = rx_fib_index0;
632 
633  if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
634  {
635  if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
636  IP_PROTOCOL_ICMP, rx_fib_index0)))
637  {
638  dont_translate = 1;
639  goto out;
640  }
641 
642  if (icmp_is_error_message (icmp0))
643  {
644  next0 = SNAT_IN2OUT_NEXT_DROP;
645  goto out;
646  }
647 
648  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
649  next0 = SNAT_IN2OUT_NEXT_DROP;
650  goto out;
651  }
652 
653  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
654  (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
655  !icmp_is_error_message (icmp0)))
656  {
657  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
658  next0 = SNAT_IN2OUT_NEXT_DROP;
659  goto out;
660  }
661 
662 out:
663  *p_value = sm0;
664 out2:
665  *p_proto = key0.protocol;
666  *p_dont_translate = dont_translate;
667  return next0;
668 }
669 
670 static inline u32 icmp_in2out (snat_main_t *sm,
671  vlib_buffer_t * b0,
672  ip4_header_t * ip0,
673  icmp46_header_t * icmp0,
674  u32 sw_if_index0,
675  u32 rx_fib_index0,
676  vlib_node_runtime_t * node,
677  u32 next0,
678  u32 thread_index,
679  void *d,
680  void *e)
681 {
682  snat_session_key_t sm0;
683  u8 protocol;
684  icmp_echo_header_t *echo0, *inner_echo0 = 0;
685  ip4_header_t *inner_ip0;
686  void *l4_header = 0;
687  icmp46_header_t *inner_icmp0;
688  u8 dont_translate;
689  u32 new_addr0, old_addr0;
690  u16 old_id0, new_id0;
691  ip_csum_t sum0;
692  u16 checksum0;
693  u32 next0_tmp;
694 
695  echo0 = (icmp_echo_header_t *)(icmp0+1);
696 
697  next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
698  &protocol, &sm0, &dont_translate, d, e);
699  if (next0_tmp != ~0)
700  next0 = next0_tmp;
701  if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
702  goto out;
703 
704  sum0 = ip_incremental_checksum (0, icmp0,
705  ntohs(ip0->length) - ip4_header_bytes (ip0));
706  checksum0 = ~ip_csum_fold (sum0);
707  if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
708  {
709  next0 = SNAT_IN2OUT_NEXT_DROP;
710  goto out;
711  }
712 
713  old_addr0 = ip0->src_address.as_u32;
714  new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
715  if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
716  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
717 
718  sum0 = ip0->checksum;
719  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
720  src_address /* changed member */);
721  ip0->checksum = ip_csum_fold (sum0);
722 
723  if (icmp0->checksum == 0)
724  icmp0->checksum = 0xffff;
725 
726  if (!icmp_is_error_message (icmp0))
727  {
728  new_id0 = sm0.port;
729  if (PREDICT_FALSE(new_id0 != echo0->identifier))
730  {
731  old_id0 = echo0->identifier;
732  new_id0 = sm0.port;
733  echo0->identifier = new_id0;
734 
735  sum0 = icmp0->checksum;
736  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
737  identifier);
738  icmp0->checksum = ip_csum_fold (sum0);
739  }
740  }
741  else
742  {
743  inner_ip0 = (ip4_header_t *)(echo0+1);
744  l4_header = ip4_next_header (inner_ip0);
745 
746  if (!ip4_header_checksum_is_valid (inner_ip0))
747  {
748  next0 = SNAT_IN2OUT_NEXT_DROP;
749  goto out;
750  }
751 
752  old_addr0 = inner_ip0->dst_address.as_u32;
753  inner_ip0->dst_address = sm0.addr;
754  new_addr0 = inner_ip0->dst_address.as_u32;
755 
756  sum0 = icmp0->checksum;
757  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
758  dst_address /* changed member */);
759  icmp0->checksum = ip_csum_fold (sum0);
760 
761  switch (protocol)
762  {
763  case SNAT_PROTOCOL_ICMP:
764  inner_icmp0 = (icmp46_header_t*)l4_header;
765  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
766 
767  old_id0 = inner_echo0->identifier;
768  new_id0 = sm0.port;
769  inner_echo0->identifier = new_id0;
770 
771  sum0 = icmp0->checksum;
772  sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
773  identifier);
774  icmp0->checksum = ip_csum_fold (sum0);
775  break;
776  case SNAT_PROTOCOL_UDP:
777  case SNAT_PROTOCOL_TCP:
778  old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
779  new_id0 = sm0.port;
780  ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
781 
782  sum0 = icmp0->checksum;
783  sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
784  dst_port);
785  icmp0->checksum = ip_csum_fold (sum0);
786  break;
787  default:
788  ASSERT(0);
789  }
790  }
791 
792 out:
793  return next0;
794 }
795 
796 /**
797  * @brief Hairpinning
798  *
799  * Hairpinning allows two endpoints on the internal side of the NAT to
800  * communicate even if they only use each other's external IP addresses
801  * and ports.
802  *
803  * @param sm NAT main.
804  * @param b0 Vlib buffer.
805  * @param ip0 IP header.
806  * @param udp0 UDP header.
807  * @param tcp0 TCP header.
808  * @param proto0 NAT protocol.
809  */
810 static inline int
812  vlib_buffer_t * b0,
813  ip4_header_t * ip0,
814  udp_header_t * udp0,
815  tcp_header_t * tcp0,
816  u32 proto0,
817  int is_ed)
818 {
819  snat_session_key_t key0, sm0;
820  snat_session_t * s0;
821  clib_bihash_kv_8_8_t kv0, value0;
822  ip_csum_t sum0;
823  u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
824  u16 new_dst_port0, old_dst_port0;
825  int rv;
826 
827  key0.addr = ip0->dst_address;
828  key0.port = udp0->dst_port;
829  key0.protocol = proto0;
830  key0.fib_index = sm->outside_fib_index;
831  kv0.key = key0.as_u64;
832 
833  /* Check if destination is static mappings */
834  if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
835  {
836  new_dst_addr0 = sm0.addr.as_u32;
837  new_dst_port0 = sm0.port;
838  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
839  }
840  /* or active session */
841  else
842  {
843  if (sm->num_workers > 1)
844  ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
845  else
846  ti = sm->num_workers;
847 
848  if (is_ed)
849  {
850  clib_bihash_kv_16_8_t ed_kv, ed_value;
851  make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
852  ip0->protocol, sm->outside_fib_index, udp0->dst_port,
853  udp0->src_port);
854  rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
855  &ed_kv, &ed_value);
856  si = ed_value.value;
857  }
858  else
859  {
860  rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
861  &value0);
862  si = value0.value;
863  }
864  if (rv)
865  return 0;
866 
867  s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
868  new_dst_addr0 = s0->in2out.addr.as_u32;
869  new_dst_port0 = s0->in2out.port;
870  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
871  }
872 
873  /* Destination is behind the same NAT, use internal address and port */
874  if (new_dst_addr0)
875  {
876  old_dst_addr0 = ip0->dst_address.as_u32;
877  ip0->dst_address.as_u32 = new_dst_addr0;
878  sum0 = ip0->checksum;
879  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
880  ip4_header_t, dst_address);
881  ip0->checksum = ip_csum_fold (sum0);
882 
883  old_dst_port0 = tcp0->dst;
884  if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
885  {
886  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
887  {
888  tcp0->dst = new_dst_port0;
889  sum0 = tcp0->checksum;
890  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
891  ip4_header_t, dst_address);
892  sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
893  ip4_header_t /* cheat */, length);
894  tcp0->checksum = ip_csum_fold(sum0);
895  }
896  else
897  {
898  udp0->dst_port = new_dst_port0;
899  udp0->checksum = 0;
900  }
901  }
902  else
903  {
904  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
905  {
906  sum0 = tcp0->checksum;
907  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
908  ip4_header_t, dst_address);
909  tcp0->checksum = ip_csum_fold(sum0);
910  }
911  }
912  return 1;
913  }
914  return 0;
915 }
916 
917 static inline void
919  vlib_buffer_t * b0,
920  ip4_header_t * ip0,
921  icmp46_header_t * icmp0,
922  int is_ed)
923 {
924  snat_session_key_t key0, sm0;
925  clib_bihash_kv_8_8_t kv0, value0;
926  u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
927  ip_csum_t sum0;
928  snat_session_t *s0;
929  int rv;
930 
931  if (!icmp_is_error_message (icmp0))
932  {
933  icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
934  u16 icmp_id0 = echo0->identifier;
935  key0.addr = ip0->dst_address;
936  key0.port = icmp_id0;
937  key0.protocol = SNAT_PROTOCOL_ICMP;
938  key0.fib_index = sm->outside_fib_index;
939  kv0.key = key0.as_u64;
940 
941  if (sm->num_workers > 1)
942  ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
943  else
944  ti = sm->num_workers;
945 
946  /* Check if destination is in active sessions */
947  if (is_ed)
948  {
949  clib_bihash_kv_16_8_t ed_kv, ed_value;
950  make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
951  IP_PROTOCOL_ICMP, sm->outside_fib_index, icmp_id0, 0);
952  rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
953  &ed_kv, &ed_value);
954  si = ed_value.value;
955  }
956  else
957  {
958  rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
959  &value0);
960  si = value0.value;
961  }
962  if (rv)
963  {
964  /* or static mappings */
965  if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
966  {
967  new_dst_addr0 = sm0.addr.as_u32;
968  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
969  }
970  }
971  else
972  {
973  s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
974  new_dst_addr0 = s0->in2out.addr.as_u32;
975  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
976  echo0->identifier = s0->in2out.port;
977  sum0 = icmp0->checksum;
978  sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
979  icmp_echo_header_t, identifier);
980  icmp0->checksum = ip_csum_fold (sum0);
981  }
982 
983  /* Destination is behind the same NAT, use internal address and port */
984  if (new_dst_addr0)
985  {
986  old_dst_addr0 = ip0->dst_address.as_u32;
987  ip0->dst_address.as_u32 = new_dst_addr0;
988  sum0 = ip0->checksum;
989  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
990  ip4_header_t, dst_address);
991  ip0->checksum = ip_csum_fold (sum0);
992  }
993  }
994 
995 }
996 
998  vlib_buffer_t * b0,
999  ip4_header_t * ip0,
1000  icmp46_header_t * icmp0,
1001  u32 sw_if_index0,
1002  u32 rx_fib_index0,
1003  vlib_node_runtime_t * node,
1004  u32 next0,
1005  f64 now,
1006  u32 thread_index,
1007  snat_session_t ** p_s0)
1008 {
1009  next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1010  next0, thread_index, p_s0, 0);
1011  snat_session_t * s0 = *p_s0;
1012  if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1013  {
1014  /* Hairpinning */
1015  if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1016  snat_icmp_hairpinning(sm, b0, ip0, icmp0, sm->endpoint_dependent);
1017  /* Accounting */
1020  /* Per-user LRU list maintenance */
1021  nat44_session_update_lru (sm, s0, thread_index);
1022  }
1023  return next0;
1024 }
1025 
1026 static inline void
1028  vlib_buffer_t * b,
1029  ip4_header_t * ip)
1030 {
1031  clib_bihash_kv_8_8_t kv, value;
1033  u32 old_addr, new_addr;
1034  ip_csum_t sum;
1035 
1036  make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1037  if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1038  return;
1039 
1040  m = pool_elt_at_index (sm->static_mappings, value.value);
1041 
1042  old_addr = ip->dst_address.as_u32;
1043  new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1044  sum = ip->checksum;
1045  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1046  ip->checksum = ip_csum_fold (sum);
1047 
1048  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1049  vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1050 }
1051 
1052 static int
1054  vlib_buffer_t * b,
1055  ip4_header_t * ip,
1056  u32 rx_fib_index)
1057 {
1058  clib_bihash_kv_8_8_t kv, value;
1060  snat_session_key_t m_key;
1061  u32 old_addr, new_addr;
1062  ip_csum_t sum;
1063 
1064  m_key.addr = ip->src_address;
1065  m_key.port = 0;
1066  m_key.protocol = 0;
1067  m_key.fib_index = rx_fib_index;
1068  kv.key = m_key.as_u64;
1069  if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1070  return 1;
1071 
1072  m = pool_elt_at_index (sm->static_mappings, value.value);
1073 
1074  old_addr = ip->src_address.as_u32;
1075  new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1076  sum = ip->checksum;
1077  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1078  ip->checksum = ip_csum_fold (sum);
1079 
1080 
1081  /* Hairpinning */
1082  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1083  {
1084  vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1086  }
1087 
1088  return 0;
1089 }
1090 
1091 static inline uword
1093  vlib_node_runtime_t * node,
1094  vlib_frame_t * frame, int is_slow_path,
1095  int is_output_feature)
1096 {
1097  u32 n_left_from, * from, * to_next;
1098  snat_in2out_next_t next_index;
1099  u32 pkts_processed = 0;
1100  snat_main_t * sm = &snat_main;
1101  f64 now = vlib_time_now (vm);
1102  u32 stats_node_index;
1103  u32 thread_index = vm->thread_index;
1104 
1105  stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1106  snat_in2out_node.index;
1107 
1108  from = vlib_frame_vector_args (frame);
1109  n_left_from = frame->n_vectors;
1110  next_index = node->cached_next_index;
1111 
1112  while (n_left_from > 0)
1113  {
1114  u32 n_left_to_next;
1115 
1116  vlib_get_next_frame (vm, node, next_index,
1117  to_next, n_left_to_next);
1118 
1119  while (n_left_from >= 4 && n_left_to_next >= 2)
1120  {
1121  u32 bi0, bi1;
1122  vlib_buffer_t * b0, * b1;
1123  u32 next0, next1;
1124  u32 sw_if_index0, sw_if_index1;
1125  ip4_header_t * ip0, * ip1;
1126  ip_csum_t sum0, sum1;
1127  u32 new_addr0, old_addr0, new_addr1, old_addr1;
1128  u16 old_port0, new_port0, old_port1, new_port1;
1129  udp_header_t * udp0, * udp1;
1130  tcp_header_t * tcp0, * tcp1;
1131  icmp46_header_t * icmp0, * icmp1;
1132  snat_session_key_t key0, key1;
1133  u32 rx_fib_index0, rx_fib_index1;
1134  u32 proto0, proto1;
1135  snat_session_t * s0 = 0, * s1 = 0;
1136  clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1137  u32 iph_offset0 = 0, iph_offset1 = 0;
1138 
1139  /* Prefetch next iteration. */
1140  {
1141  vlib_buffer_t * p2, * p3;
1142 
1143  p2 = vlib_get_buffer (vm, from[2]);
1144  p3 = vlib_get_buffer (vm, from[3]);
1145 
1146  vlib_prefetch_buffer_header (p2, LOAD);
1147  vlib_prefetch_buffer_header (p3, LOAD);
1148 
1151  }
1152 
1153  /* speculatively enqueue b0 and b1 to the current next frame */
1154  to_next[0] = bi0 = from[0];
1155  to_next[1] = bi1 = from[1];
1156  from += 2;
1157  to_next += 2;
1158  n_left_from -= 2;
1159  n_left_to_next -= 2;
1160 
1161  b0 = vlib_get_buffer (vm, bi0);
1162  b1 = vlib_get_buffer (vm, bi1);
1163 
1164  if (is_output_feature)
1165  iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1166 
1167  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1168  iph_offset0);
1169 
1170  udp0 = ip4_next_header (ip0);
1171  tcp0 = (tcp_header_t *) udp0;
1172  icmp0 = (icmp46_header_t *) udp0;
1173 
1174  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1175  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1176  sw_if_index0);
1177 
1178  next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1179 
1180  if (PREDICT_FALSE(ip0->ttl == 1))
1181  {
1182  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1183  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1184  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1185  0);
1187  goto trace00;
1188  }
1189 
1190  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1191 
1192  /* Next configured feature, probably ip4-lookup */
1193  if (is_slow_path)
1194  {
1195  if (PREDICT_FALSE (proto0 == ~0))
1196  {
1197  if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1198  {
1199  next0 = SNAT_IN2OUT_NEXT_DROP;
1200  b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1201  }
1202  goto trace00;
1203  }
1204 
1205  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1206  {
1207  next0 = icmp_in2out_slow_path
1208  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1209  node, next0, now, thread_index, &s0);
1210  goto trace00;
1211  }
1212  }
1213  else
1214  {
1215  if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1216  {
1218  goto trace00;
1219  }
1220 
1221  if (ip4_is_fragment (ip0))
1222  {
1223  next0 = SNAT_IN2OUT_NEXT_REASS;
1224  goto trace00;
1225  }
1226  }
1227 
1228  key0.addr = ip0->src_address;
1229  key0.port = udp0->src_port;
1230  key0.protocol = proto0;
1231  key0.fib_index = rx_fib_index0;
1232 
1233  kv0.key = key0.as_u64;
1234 
1235  if (PREDICT_FALSE (clib_bihash_search_8_8 (
1236  &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1237  {
1238  if (is_slow_path)
1239  {
1240  if (is_output_feature)
1241  {
1243  ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1244  goto trace00;
1245  }
1246  else
1247  {
1248  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1249  ip0, proto0, rx_fib_index0, thread_index)))
1250  goto trace00;
1251  }
1252 
1253  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1254  &s0, node, next0, thread_index);
1255  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1256  goto trace00;
1257  }
1258  else
1259  {
1261  goto trace00;
1262  }
1263  }
1264  else
1265  s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1266  value0.value);
1267 
1268  b0->flags |= VNET_BUFFER_F_IS_NATED;
1269 
1270  old_addr0 = ip0->src_address.as_u32;
1271  ip0->src_address = s0->out2in.addr;
1272  new_addr0 = ip0->src_address.as_u32;
1273  if (!is_output_feature)
1274  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1275 
1276  sum0 = ip0->checksum;
1277  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1278  ip4_header_t,
1279  src_address /* changed member */);
1280  ip0->checksum = ip_csum_fold (sum0);
1281 
1282  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1283  {
1284  old_port0 = tcp0->src_port;
1285  tcp0->src_port = s0->out2in.port;
1286  new_port0 = tcp0->src_port;
1287 
1288  sum0 = tcp0->checksum;
1289  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1290  ip4_header_t,
1291  dst_address /* changed member */);
1292  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1293  ip4_header_t /* cheat */,
1294  length /* changed member */);
1295  tcp0->checksum = ip_csum_fold(sum0);
1296  }
1297  else
1298  {
1299  old_port0 = udp0->src_port;
1300  udp0->src_port = s0->out2in.port;
1301  udp0->checksum = 0;
1302  }
1303 
1304  /* Accounting */
1306  vlib_buffer_length_in_chain (vm, b0));
1307  /* Per-user LRU list maintenance */
1308  nat44_session_update_lru (sm, s0, thread_index);
1309  trace00:
1310 
1312  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1313  {
1314  snat_in2out_trace_t *t =
1315  vlib_add_trace (vm, node, b0, sizeof (*t));
1316  t->is_slow_path = is_slow_path;
1317  t->sw_if_index = sw_if_index0;
1318  t->next_index = next0;
1319  t->session_index = ~0;
1320  if (s0)
1321  t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1322  }
1323 
1324  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1325 
1326  if (is_output_feature)
1327  iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1328 
1329  ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1330  iph_offset1);
1331 
1332  udp1 = ip4_next_header (ip1);
1333  tcp1 = (tcp_header_t *) udp1;
1334  icmp1 = (icmp46_header_t *) udp1;
1335 
1336  sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1337  rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1338  sw_if_index1);
1339 
1340  if (PREDICT_FALSE(ip1->ttl == 1))
1341  {
1342  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1343  icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1344  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1345  0);
1347  goto trace01;
1348  }
1349 
1350  proto1 = ip_proto_to_snat_proto (ip1->protocol);
1351 
1352  /* Next configured feature, probably ip4-lookup */
1353  if (is_slow_path)
1354  {
1355  if (PREDICT_FALSE (proto1 == ~0))
1356  {
1357  if (nat_in2out_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1358  {
1359  next1 = SNAT_IN2OUT_NEXT_DROP;
1360  b1->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1361  }
1362  goto trace01;
1363  }
1364 
1365  if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1366  {
1367  next1 = icmp_in2out_slow_path
1368  (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1369  next1, now, thread_index, &s1);
1370  goto trace01;
1371  }
1372  }
1373  else
1374  {
1375  if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1376  {
1378  goto trace01;
1379  }
1380 
1381  if (ip4_is_fragment (ip1))
1382  {
1383  next1 = SNAT_IN2OUT_NEXT_REASS;
1384  goto trace01;
1385  }
1386  }
1387 
1388  key1.addr = ip1->src_address;
1389  key1.port = udp1->src_port;
1390  key1.protocol = proto1;
1391  key1.fib_index = rx_fib_index1;
1392 
1393  kv1.key = key1.as_u64;
1394 
1395  if (PREDICT_FALSE(clib_bihash_search_8_8 (
1396  &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1397  {
1398  if (is_slow_path)
1399  {
1400  if (is_output_feature)
1401  {
1403  ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1404  goto trace01;
1405  }
1406  else
1407  {
1408  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1409  ip1, proto1, rx_fib_index1, thread_index)))
1410  goto trace01;
1411  }
1412 
1413  next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1414  &s1, node, next1, thread_index);
1415  if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1416  goto trace01;
1417  }
1418  else
1419  {
1421  goto trace01;
1422  }
1423  }
1424  else
1425  s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1426  value1.value);
1427 
1428  b1->flags |= VNET_BUFFER_F_IS_NATED;
1429 
1430  old_addr1 = ip1->src_address.as_u32;
1431  ip1->src_address = s1->out2in.addr;
1432  new_addr1 = ip1->src_address.as_u32;
1433  if (!is_output_feature)
1434  vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1435 
1436  sum1 = ip1->checksum;
1437  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1438  ip4_header_t,
1439  src_address /* changed member */);
1440  ip1->checksum = ip_csum_fold (sum1);
1441 
1442  if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1443  {
1444  old_port1 = tcp1->src_port;
1445  tcp1->src_port = s1->out2in.port;
1446  new_port1 = tcp1->src_port;
1447 
1448  sum1 = tcp1->checksum;
1449  sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1450  ip4_header_t,
1451  dst_address /* changed member */);
1452  sum1 = ip_csum_update (sum1, old_port1, new_port1,
1453  ip4_header_t /* cheat */,
1454  length /* changed member */);
1455  tcp1->checksum = ip_csum_fold(sum1);
1456  }
1457  else
1458  {
1459  old_port1 = udp1->src_port;
1460  udp1->src_port = s1->out2in.port;
1461  udp1->checksum = 0;
1462  }
1463 
1464  /* Accounting */
1466  vlib_buffer_length_in_chain (vm, b1));
1467  /* Per-user LRU list maintenance */
1468  nat44_session_update_lru (sm, s1, thread_index);
1469  trace01:
1470 
1472  && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1473  {
1474  snat_in2out_trace_t *t =
1475  vlib_add_trace (vm, node, b1, sizeof (*t));
1476  t->sw_if_index = sw_if_index1;
1477  t->next_index = next1;
1478  t->session_index = ~0;
1479  if (s1)
1480  t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1481  }
1482 
1483  pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1484 
1485  /* verify speculative enqueues, maybe switch current next frame */
1486  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1487  to_next, n_left_to_next,
1488  bi0, bi1, next0, next1);
1489  }
1490 
1491  while (n_left_from > 0 && n_left_to_next > 0)
1492  {
1493  u32 bi0;
1494  vlib_buffer_t * b0;
1495  u32 next0;
1496  u32 sw_if_index0;
1497  ip4_header_t * ip0;
1498  ip_csum_t sum0;
1499  u32 new_addr0, old_addr0;
1500  u16 old_port0, new_port0;
1501  udp_header_t * udp0;
1502  tcp_header_t * tcp0;
1503  icmp46_header_t * icmp0;
1504  snat_session_key_t key0;
1505  u32 rx_fib_index0;
1506  u32 proto0;
1507  snat_session_t * s0 = 0;
1508  clib_bihash_kv_8_8_t kv0, value0;
1509  u32 iph_offset0 = 0;
1510 
1511  /* speculatively enqueue b0 to the current next frame */
1512  bi0 = from[0];
1513  to_next[0] = bi0;
1514  from += 1;
1515  to_next += 1;
1516  n_left_from -= 1;
1517  n_left_to_next -= 1;
1518 
1519  b0 = vlib_get_buffer (vm, bi0);
1520  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1521 
1522  if (is_output_feature)
1523  iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1524 
1525  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1526  iph_offset0);
1527 
1528  udp0 = ip4_next_header (ip0);
1529  tcp0 = (tcp_header_t *) udp0;
1530  icmp0 = (icmp46_header_t *) udp0;
1531 
1532  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1533  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1534  sw_if_index0);
1535 
1536  if (PREDICT_FALSE(ip0->ttl == 1))
1537  {
1538  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1539  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1540  ICMP4_time_exceeded_ttl_exceeded_in_transit,
1541  0);
1543  goto trace0;
1544  }
1545 
1546  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1547 
1548  /* Next configured feature, probably ip4-lookup */
1549  if (is_slow_path)
1550  {
1551  if (PREDICT_FALSE (proto0 == ~0))
1552  {
1553  if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1554  {
1555  next0 = SNAT_IN2OUT_NEXT_DROP;
1556  b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1557  }
1558  goto trace0;
1559  }
1560 
1561  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1562  {
1563  next0 = icmp_in2out_slow_path
1564  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1565  next0, now, thread_index, &s0);
1566  goto trace0;
1567  }
1568  }
1569  else
1570  {
1571  if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1572  {
1574  goto trace0;
1575  }
1576 
1577  if (ip4_is_fragment (ip0))
1578  {
1579  next0 = SNAT_IN2OUT_NEXT_REASS;
1580  goto trace0;
1581  }
1582  }
1583 
1584  key0.addr = ip0->src_address;
1585  key0.port = udp0->src_port;
1586  key0.protocol = proto0;
1587  key0.fib_index = rx_fib_index0;
1588 
1589  kv0.key = key0.as_u64;
1590 
1591  if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1592  &kv0, &value0))
1593  {
1594  if (is_slow_path)
1595  {
1596  if (is_output_feature)
1597  {
1599  ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1600  goto trace0;
1601  }
1602  else
1603  {
1604  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1605  ip0, proto0, rx_fib_index0, thread_index)))
1606  goto trace0;
1607  }
1608 
1609  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1610  &s0, node, next0, thread_index);
1611 
1612  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1613  goto trace0;
1614  }
1615  else
1616  {
1618  goto trace0;
1619  }
1620  }
1621  else
1622  s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1623  value0.value);
1624 
1625  b0->flags |= VNET_BUFFER_F_IS_NATED;
1626 
1627  old_addr0 = ip0->src_address.as_u32;
1628  ip0->src_address = s0->out2in.addr;
1629  new_addr0 = ip0->src_address.as_u32;
1630  if (!is_output_feature)
1631  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1632 
1633  sum0 = ip0->checksum;
1634  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1635  ip4_header_t,
1636  src_address /* changed member */);
1637  ip0->checksum = ip_csum_fold (sum0);
1638 
1639  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1640  {
1641  old_port0 = tcp0->src_port;
1642  tcp0->src_port = s0->out2in.port;
1643  new_port0 = tcp0->src_port;
1644 
1645  sum0 = tcp0->checksum;
1646  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1647  ip4_header_t,
1648  dst_address /* changed member */);
1649  sum0 = ip_csum_update (sum0, old_port0, new_port0,
1650  ip4_header_t /* cheat */,
1651  length /* changed member */);
1652  tcp0->checksum = ip_csum_fold(sum0);
1653  }
1654  else
1655  {
1656  old_port0 = udp0->src_port;
1657  udp0->src_port = s0->out2in.port;
1658  udp0->checksum = 0;
1659  }
1660 
1661  /* Accounting */
1663  vlib_buffer_length_in_chain (vm, b0));
1664  /* Per-user LRU list maintenance */
1665  nat44_session_update_lru (sm, s0, thread_index);
1666 
1667  trace0:
1669  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1670  {
1671  snat_in2out_trace_t *t =
1672  vlib_add_trace (vm, node, b0, sizeof (*t));
1673  t->is_slow_path = is_slow_path;
1674  t->sw_if_index = sw_if_index0;
1675  t->next_index = next0;
1676  t->session_index = ~0;
1677  if (s0)
1678  t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1679  }
1680 
1681  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1682 
1683  /* verify speculative enqueue, maybe switch current next frame */
1684  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1685  to_next, n_left_to_next,
1686  bi0, next0);
1687  }
1688 
1689  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1690  }
1691 
1692  vlib_node_increment_counter (vm, stats_node_index,
1693  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1694  pkts_processed);
1695  return frame->n_vectors;
1696 }
1697 
1698 static uword
1700  vlib_node_runtime_t * node,
1701  vlib_frame_t * frame)
1702 {
1703  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1704 }
1705 
1707  .function = snat_in2out_fast_path_fn,
1708  .name = "nat44-in2out",
1709  .vector_size = sizeof (u32),
1710  .format_trace = format_snat_in2out_trace,
1711  .type = VLIB_NODE_TYPE_INTERNAL,
1712 
1713  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1714  .error_strings = snat_in2out_error_strings,
1715 
1716  .runtime_data_bytes = sizeof (snat_runtime_t),
1717 
1718  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1719 
1720  /* edit / add dispositions here */
1721  .next_nodes = {
1722  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1723  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1724  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1725  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1726  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1727  },
1728 };
1729 
1731 
1732 static uword
1734  vlib_node_runtime_t * node,
1735  vlib_frame_t * frame)
1736 {
1737  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1738 }
1739 
1741  .function = snat_in2out_output_fast_path_fn,
1742  .name = "nat44-in2out-output",
1743  .vector_size = sizeof (u32),
1744  .format_trace = format_snat_in2out_trace,
1745  .type = VLIB_NODE_TYPE_INTERNAL,
1746 
1747  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1748  .error_strings = snat_in2out_error_strings,
1749 
1750  .runtime_data_bytes = sizeof (snat_runtime_t),
1751 
1752  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1753 
1754  /* edit / add dispositions here */
1755  .next_nodes = {
1756  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1757  [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1758  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1759  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1760  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1761  },
1762 };
1763 
1766 
1767 static uword
1769  vlib_node_runtime_t * node,
1770  vlib_frame_t * frame)
1771 {
1772  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
1773 }
1774 
1776  .function = snat_in2out_slow_path_fn,
1777  .name = "nat44-in2out-slowpath",
1778  .vector_size = sizeof (u32),
1779  .format_trace = format_snat_in2out_trace,
1780  .type = VLIB_NODE_TYPE_INTERNAL,
1781 
1782  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1783  .error_strings = snat_in2out_error_strings,
1784 
1785  .runtime_data_bytes = sizeof (snat_runtime_t),
1786 
1787  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1788 
1789  /* edit / add dispositions here */
1790  .next_nodes = {
1791  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1792  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1793  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1794  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1795  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1796  },
1797 };
1798 
1801 
1802 static uword
1804  vlib_node_runtime_t * node,
1805  vlib_frame_t * frame)
1806 {
1807  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
1808 }
1809 
1811  .function = snat_in2out_output_slow_path_fn,
1812  .name = "nat44-in2out-output-slowpath",
1813  .vector_size = sizeof (u32),
1814  .format_trace = format_snat_in2out_trace,
1815  .type = VLIB_NODE_TYPE_INTERNAL,
1816 
1817  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1818  .error_strings = snat_in2out_error_strings,
1819 
1820  .runtime_data_bytes = sizeof (snat_runtime_t),
1821 
1822  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1823 
1824  /* edit / add dispositions here */
1825  .next_nodes = {
1826  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1827  [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1828  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1829  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1830  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1831  },
1832 };
1833 
1836 
1838 
1839 static inline uword
1841  vlib_node_runtime_t * node,
1842  vlib_frame_t * frame,
1843  int is_ed)
1844 {
1845  u32 n_left_from, * from, * to_next, stats_node_index;
1846  snat_in2out_next_t next_index;
1847  u32 pkts_processed = 0;
1848  snat_main_t * sm = &snat_main;
1850  u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1851  vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
1852 
1853  stats_node_index = is_ed ? nat44_ed_hairpinning_node.index :
1854  nat44_hairpinning_node.index;
1855  from = vlib_frame_vector_args (frame);
1856  n_left_from = frame->n_vectors;
1857  next_index = node->cached_next_index;
1858 
1859  while (n_left_from > 0)
1860  {
1861  u32 n_left_to_next;
1862 
1863  vlib_get_next_frame (vm, node, next_index,
1864  to_next, n_left_to_next);
1865 
1866  while (n_left_from > 0 && n_left_to_next > 0)
1867  {
1868  u32 bi0;
1869  vlib_buffer_t * b0;
1870  u32 next0;
1871  ip4_header_t * ip0;
1872  u32 proto0;
1873  udp_header_t * udp0;
1874  tcp_header_t * tcp0;
1875 
1876  /* speculatively enqueue b0 to the current next frame */
1877  bi0 = from[0];
1878  to_next[0] = bi0;
1879  from += 1;
1880  to_next += 1;
1881  n_left_from -= 1;
1882  n_left_to_next -= 1;
1883 
1884  b0 = vlib_get_buffer (vm, bi0);
1885  ip0 = vlib_buffer_get_current (b0);
1886  udp0 = ip4_next_header (ip0);
1887  tcp0 = (tcp_header_t *) udp0;
1888 
1889  proto0 = ip_proto_to_snat_proto (ip0->protocol);
1890 
1892  &next0, 0);
1893 
1894  if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
1895  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1896 
1897  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1898 
1899  /* verify speculative enqueue, maybe switch current next frame */
1900  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1901  to_next, n_left_to_next,
1902  bi0, next0);
1903  }
1904 
1905  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1906  }
1907 
1908  vlib_node_increment_counter (vm, stats_node_index,
1909  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1910  pkts_processed);
1911  return frame->n_vectors;
1912 }
1913 
1914 static uword
1916  vlib_node_runtime_t * node,
1917  vlib_frame_t * frame)
1918 {
1919  return nat44_hairpinning_fn_inline (vm, node, frame, 0);
1920 }
1921 
1923  .function = nat44_hairpinning_fn,
1924  .name = "nat44-hairpinning",
1925  .vector_size = sizeof (u32),
1926  .type = VLIB_NODE_TYPE_INTERNAL,
1927  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1928  .error_strings = snat_in2out_error_strings,
1929  .n_next_nodes = 2,
1930  .next_nodes = {
1931  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1932  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1933  },
1934 };
1935 
1938 
1939 static uword
1941  vlib_node_runtime_t * node,
1942  vlib_frame_t * frame)
1943 {
1944  return nat44_hairpinning_fn_inline (vm, node, frame, 1);
1945 }
1946 
1948  .function = nat44_ed_hairpinning_fn,
1949  .name = "nat44-ed-hairpinning",
1950  .vector_size = sizeof (u32),
1951  .type = VLIB_NODE_TYPE_INTERNAL,
1952  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1953  .error_strings = snat_in2out_error_strings,
1954  .n_next_nodes = 2,
1955  .next_nodes = {
1956  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1957  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1958  },
1959 };
1960 
1963 
1964 static inline void
1966  vlib_buffer_t * b0,
1967  ip4_header_t * ip0,
1968  u16 sport,
1969  u16 dport,
1970  u32 proto0)
1971 {
1972  snat_session_key_t key0, sm0;
1973  snat_session_t * s0;
1974  clib_bihash_kv_8_8_t kv0, value0;
1975  ip_csum_t sum0;
1976  u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
1977  u16 new_dst_port0, old_dst_port0;
1978  udp_header_t * udp0;
1979  tcp_header_t * tcp0;
1980 
1981  key0.addr = ip0->dst_address;
1982  key0.port = dport;
1983  key0.protocol = proto0;
1984  key0.fib_index = sm->outside_fib_index;
1985  kv0.key = key0.as_u64;
1986 
1987  udp0 = ip4_next_header (ip0);
1988 
1989  /* Check if destination is static mappings */
1990  if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1991  {
1992  new_dst_addr0 = sm0.addr.as_u32;
1993  new_dst_port0 = sm0.port;
1994  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1995  }
1996  /* or active sessions */
1997  else
1998  {
1999  if (sm->num_workers > 1)
2000  ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2001  else
2002  ti = sm->num_workers;
2003 
2004  if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2005  {
2006  si = value0.value;
2007  s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2008  new_dst_addr0 = s0->in2out.addr.as_u32;
2009  new_dst_port0 = s0->in2out.port;
2010  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2011  }
2012  }
2013 
2014  /* Destination is behind the same NAT, use internal address and port */
2015  if (new_dst_addr0)
2016  {
2017  old_dst_addr0 = ip0->dst_address.as_u32;
2018  ip0->dst_address.as_u32 = new_dst_addr0;
2019  sum0 = ip0->checksum;
2020  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2021  ip4_header_t, dst_address);
2022  ip0->checksum = ip_csum_fold (sum0);
2023 
2024  old_dst_port0 = dport;
2025  if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2026  ip4_is_first_fragment (ip0)))
2027  {
2028  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2029  {
2030  tcp0 = ip4_next_header (ip0);
2031  tcp0->dst = new_dst_port0;
2032  sum0 = tcp0->checksum;
2033  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2034  ip4_header_t, dst_address);
2035  sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2036  ip4_header_t /* cheat */, length);
2037  tcp0->checksum = ip_csum_fold(sum0);
2038  }
2039  else
2040  {
2041  udp0->dst_port = new_dst_port0;
2042  udp0->checksum = 0;
2043  }
2044  }
2045  else
2046  {
2047  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2048  {
2049  tcp0 = ip4_next_header (ip0);
2050  sum0 = tcp0->checksum;
2051  sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2052  ip4_header_t, dst_address);
2053  tcp0->checksum = ip_csum_fold(sum0);
2054  }
2055  }
2056  }
2057 }
2058 
2059 static uword
2061  vlib_node_runtime_t * node,
2062  vlib_frame_t * frame)
2063 {
2064  u32 n_left_from, *from, *to_next;
2065  snat_in2out_next_t next_index;
2066  u32 pkts_processed = 0;
2067  snat_main_t *sm = &snat_main;
2068  f64 now = vlib_time_now (vm);
2069  u32 thread_index = vm->thread_index;
2070  snat_main_per_thread_data_t *per_thread_data =
2071  &sm->per_thread_data[thread_index];
2072  u32 *fragments_to_drop = 0;
2073  u32 *fragments_to_loopback = 0;
2074 
2075  from = vlib_frame_vector_args (frame);
2076  n_left_from = frame->n_vectors;
2077  next_index = node->cached_next_index;
2078 
2079  while (n_left_from > 0)
2080  {
2081  u32 n_left_to_next;
2082 
2083  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2084 
2085  while (n_left_from > 0 && n_left_to_next > 0)
2086  {
2087  u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2088  vlib_buffer_t *b0;
2089  u32 next0;
2090  u8 cached0 = 0;
2091  ip4_header_t *ip0;
2092  nat_reass_ip4_t *reass0;
2093  udp_header_t * udp0;
2094  tcp_header_t * tcp0;
2095  snat_session_key_t key0;
2096  clib_bihash_kv_8_8_t kv0, value0;
2097  snat_session_t * s0 = 0;
2098  u16 old_port0, new_port0;
2099  ip_csum_t sum0;
2100 
2101  /* speculatively enqueue b0 to the current next frame */
2102  bi0 = from[0];
2103  to_next[0] = bi0;
2104  from += 1;
2105  to_next += 1;
2106  n_left_from -= 1;
2107  n_left_to_next -= 1;
2108 
2109  b0 = vlib_get_buffer (vm, bi0);
2110  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2111 
2112  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2114  sw_if_index0);
2115 
2117  {
2118  next0 = SNAT_IN2OUT_NEXT_DROP;
2119  b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2120  goto trace0;
2121  }
2122 
2123  ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2124  udp0 = ip4_next_header (ip0);
2125  tcp0 = (tcp_header_t *) udp0;
2126  proto0 = ip_proto_to_snat_proto (ip0->protocol);
2127 
2129  ip0->dst_address,
2130  ip0->fragment_id,
2131  ip0->protocol,
2132  1,
2133  &fragments_to_drop);
2134 
2135  if (PREDICT_FALSE (!reass0))
2136  {
2137  next0 = SNAT_IN2OUT_NEXT_DROP;
2138  b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2139  nat_log_notice ("maximum reassemblies exceeded");
2140  goto trace0;
2141  }
2142 
2144  {
2145  key0.addr = ip0->src_address;
2146  key0.port = udp0->src_port;
2147  key0.protocol = proto0;
2148  key0.fib_index = rx_fib_index0;
2149  kv0.key = key0.as_u64;
2150 
2151  if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2152  {
2153  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2154  ip0, proto0, rx_fib_index0, thread_index)))
2155  goto trace0;
2156 
2157  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2158  &s0, node, next0, thread_index);
2159 
2160  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2161  goto trace0;
2162 
2163  reass0->sess_index = s0 - per_thread_data->sessions;
2164  }
2165  else
2166  {
2167  s0 = pool_elt_at_index (per_thread_data->sessions,
2168  value0.value);
2169  reass0->sess_index = value0.value;
2170  }
2171  nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2172  }
2173  else
2174  {
2175  if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2176  {
2177  if (nat_ip4_reass_add_fragment (reass0, bi0, &fragments_to_drop))
2178  {
2179  b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2180  nat_log_notice ("maximum fragments per reassembly exceeded");
2181  next0 = SNAT_IN2OUT_NEXT_DROP;
2182  goto trace0;
2183  }
2184  cached0 = 1;
2185  goto trace0;
2186  }
2187  s0 = pool_elt_at_index (per_thread_data->sessions,
2188  reass0->sess_index);
2189  }
2190 
2191  old_addr0 = ip0->src_address.as_u32;
2192  ip0->src_address = s0->out2in.addr;
2193  new_addr0 = ip0->src_address.as_u32;
2194  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2195 
2196  sum0 = ip0->checksum;
2197  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2198  ip4_header_t,
2199  src_address /* changed member */);
2200  ip0->checksum = ip_csum_fold (sum0);
2201 
2203  {
2204  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2205  {
2206  old_port0 = tcp0->src_port;
2207  tcp0->src_port = s0->out2in.port;
2208  new_port0 = tcp0->src_port;
2209 
2210  sum0 = tcp0->checksum;
2211  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2212  ip4_header_t,
2213  dst_address /* changed member */);
2214  sum0 = ip_csum_update (sum0, old_port0, new_port0,
2215  ip4_header_t /* cheat */,
2216  length /* changed member */);
2217  tcp0->checksum = ip_csum_fold(sum0);
2218  }
2219  else
2220  {
2221  old_port0 = udp0->src_port;
2222  udp0->src_port = s0->out2in.port;
2223  udp0->checksum = 0;
2224  }
2225  }
2226 
2227  /* Hairpinning */
2228  nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2229  s0->ext_host_port, proto0);
2230 
2231  /* Accounting */
2233  vlib_buffer_length_in_chain (vm, b0));
2234  /* Per-user LRU list maintenance */
2235  nat44_session_update_lru (sm, s0, thread_index);
2236 
2237  trace0:
2239  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2240  {
2242  vlib_add_trace (vm, node, b0, sizeof (*t));
2243  t->cached = cached0;
2244  t->sw_if_index = sw_if_index0;
2245  t->next_index = next0;
2246  }
2247 
2248  if (cached0)
2249  {
2250  n_left_to_next++;
2251  to_next--;
2252  }
2253  else
2254  {
2255  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2256 
2257  /* verify speculative enqueue, maybe switch current next frame */
2258  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2259  to_next, n_left_to_next,
2260  bi0, next0);
2261  }
2262 
2263  if (n_left_from == 0 && vec_len (fragments_to_loopback))
2264  {
2265  from = vlib_frame_vector_args (frame);
2266  u32 len = vec_len (fragments_to_loopback);
2267  if (len <= VLIB_FRAME_SIZE)
2268  {
2269  clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2270  n_left_from = len;
2271  vec_reset_length (fragments_to_loopback);
2272  }
2273  else
2274  {
2275  clib_memcpy (from,
2276  fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2277  sizeof (u32) * VLIB_FRAME_SIZE);
2278  n_left_from = VLIB_FRAME_SIZE;
2279  _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2280  }
2281  }
2282  }
2283 
2284  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2285  }
2286 
2288  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2289  pkts_processed);
2290 
2291  nat_send_all_to_node (vm, fragments_to_drop, node,
2292  &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2294 
2295  vec_free (fragments_to_drop);
2296  vec_free (fragments_to_loopback);
2297  return frame->n_vectors;
2298 }
2299 
2301  .function = nat44_in2out_reass_node_fn,
2302  .name = "nat44-in2out-reass",
2303  .vector_size = sizeof (u32),
2304  .format_trace = format_nat44_in2out_reass_trace,
2305  .type = VLIB_NODE_TYPE_INTERNAL,
2306 
2307  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2308  .error_strings = snat_in2out_error_strings,
2309 
2310  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2311  .next_nodes = {
2312  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2313  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2314  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2315  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2316  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2317  },
2318 };
2319 
2322 
2323 /*******************************/
2324 /*** endpoint-dependent mode ***/
2325 /*******************************/
2326 
2329 {
2330  icmp46_header_t *icmp0;
2331  nat_ed_ses_key_t key0;
2332  icmp_echo_header_t *echo0, *inner_echo0 = 0;
2333  ip4_header_t *inner_ip0 = 0;
2334  void *l4_header = 0;
2335  icmp46_header_t *inner_icmp0;
2336 
2337  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2338  echo0 = (icmp_echo_header_t *)(icmp0+1);
2339 
2340  if (!icmp_is_error_message (icmp0))
2341  {
2342  key0.proto = IP_PROTOCOL_ICMP;
2343  key0.l_addr = ip0->src_address;
2344  key0.r_addr = ip0->dst_address;
2345  key0.l_port = echo0->identifier;
2346  key0.r_port = 0;
2347  }
2348  else
2349  {
2350  inner_ip0 = (ip4_header_t *)(echo0+1);
2351  l4_header = ip4_next_header (inner_ip0);
2352  key0.proto = inner_ip0->protocol;
2353  key0.r_addr = inner_ip0->src_address;
2354  key0.l_addr = inner_ip0->dst_address;
2355  switch (ip_proto_to_snat_proto (inner_ip0->protocol))
2356  {
2357  case SNAT_PROTOCOL_ICMP:
2358  inner_icmp0 = (icmp46_header_t*)l4_header;
2359  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2360  key0.r_port = 0;
2361  key0.l_port = inner_echo0->identifier;
2362  break;
2363  case SNAT_PROTOCOL_UDP:
2364  case SNAT_PROTOCOL_TCP:
2365  key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2366  key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
2367  break;
2368  default:
2369  return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
2370  }
2371  }
2372  *p_key0 = key0;
2373  return 0;
2374 }
2375 
2376 static u32
2378  vlib_buffer_t *b,
2379  u32 rx_fib_index,
2381  snat_session_t ** sessionp,
2382  vlib_node_runtime_t * node,
2383  u32 next,
2384  u32 thread_index)
2385 {
2386  snat_session_t *s;
2387  snat_user_t *u;
2388  snat_session_key_t key0, key1;
2389  u8 lb = 0, is_sm = 0;
2390  u32 address_index = ~0;
2391  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2392  nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
2393  u32 proto = ip_proto_to_snat_proto (key->proto);
2394  nat_outside_fib_t *outside_fib;
2396  fib_prefix_t pfx = {
2398  .fp_len = 32,
2399  .fp_addr = {
2400  .ip4.as_u32 = key->r_addr.as_u32,
2401  },
2402  };
2403 
2404  if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
2405  {
2406  b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2408  nat_log_notice ("maximum sessions exceeded");
2409  return SNAT_IN2OUT_NEXT_DROP;
2410  }
2411 
2412  key0.addr = key->l_addr;
2413  key0.port = key->l_port;
2414  key1.protocol = key0.protocol = proto;
2415  key0.fib_index = rx_fib_index;
2416  key1.fib_index = sm->outside_fib_index;
2417  /* First try to match static mapping by local address and port */
2418  if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb))
2419  {
2420  /* Try to create dynamic translation */
2421  if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
2422  thread_index, &key1,
2423  &address_index,
2424  sm->port_per_thread,
2425  tsm->snat_thread_index))
2426  {
2427  nat_log_notice ("addresses exhausted");
2428  b->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2429  return SNAT_IN2OUT_NEXT_DROP;
2430  }
2431  }
2432  else
2433  is_sm = 1;
2434 
2435  u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
2436  if (!u)
2437  {
2438  nat_log_warn ("create NAT user failed");
2439  return SNAT_IN2OUT_NEXT_DROP;
2440  }
2441 
2442  s = nat_session_alloc_or_recycle (sm, u, thread_index);
2443  if (!s)
2444  {
2445  nat_log_warn ("create NAT session failed");
2446  return SNAT_IN2OUT_NEXT_DROP;
2447  }
2448 
2449  user_session_increment (sm, u, is_sm);
2450  if (is_sm)
2452  if (lb)
2455  s->outside_address_index = address_index;
2456  s->ext_host_addr = key->r_addr;
2457  s->ext_host_port = key->r_port;
2458  s->in2out = key0;
2459  s->out2in = key1;
2460  s->out2in.protocol = key0.protocol;
2461 
2462  switch (vec_len (sm->outside_fibs))
2463  {
2464  case 0:
2465  s->out2in.fib_index = sm->outside_fib_index;
2466  break;
2467  case 1:
2468  s->out2in.fib_index = sm->outside_fibs[0].fib_index;
2469  break;
2470  default:
2471  vec_foreach (outside_fib, sm->outside_fibs)
2472  {
2473  fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2474  if (FIB_NODE_INDEX_INVALID != fei)
2475  {
2476  if (fib_entry_get_resolving_interface (fei) != ~0)
2477  {
2478  s->out2in.fib_index = outside_fib->fib_index;
2479  break;
2480  }
2481  }
2482  }
2483  break;
2484  }
2485 
2486  /* Add to lookup tables */
2487  kv->value = s - tsm->sessions;
2488  if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, kv, 1))
2489  nat_log_notice ("in2out-ed key add failed");
2490 
2491  make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
2492  key1.port, key->r_port);
2493  kv->value = s - tsm->sessions;
2494  if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, kv, 1))
2495  nat_log_notice ("out2in-ed key add failed");
2496 
2497  *sessionp = s;
2498 
2499  /* log NAT event */
2500  snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
2501  s->out2in.addr.as_u32,
2502  s->in2out.protocol,
2503  s->in2out.port,
2504  s->out2in.port,
2505  s->in2out.fib_index);
2506  return next;
2507 }
2508 
2511  u32 sw_if_index, ip4_header_t * ip, u32 proto,
2512  u32 rx_fib_index, u32 thread_index)
2513 {
2514  udp_header_t *udp = ip4_next_header (ip);
2515  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2516  clib_bihash_kv_16_8_t kv, value;
2517  snat_session_key_t key0, key1;
2518 
2519  make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
2520  sm->outside_fib_index, udp->dst_port, udp->src_port);
2521 
2522  /* NAT packet aimed at external address if */
2523  /* has active sessions */
2524  if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2525  {
2526  key0.addr = ip->dst_address;
2527  key0.port = udp->dst_port;
2528  key0.protocol = proto;
2529  key0.fib_index = sm->outside_fib_index;
2530  /* or is static mappings */
2531  if (!snat_static_mapping_match(sm, key0, &key1, 1, 0, 0, 0))
2532  return 0;
2533  }
2534  else
2535  return 0;
2536 
2537  if (sm->forwarding_enabled)
2538  return 1;
2539 
2540  return snat_not_translate_fast(sm, node, sw_if_index, ip, proto, rx_fib_index);
2541 }
2542 
2545  u32 thread_index, f64 now,
2546  vlib_main_t * vm, vlib_buffer_t * b)
2547 {
2548  nat_ed_ses_key_t key;
2549  clib_bihash_kv_16_8_t kv, value;
2550  udp_header_t *udp;
2551  snat_session_t *s = 0;
2552  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2553 
2554  if (!sm->forwarding_enabled)
2555  return 0;
2556 
2557  if (ip->protocol == IP_PROTOCOL_ICMP)
2558  {
2559  key.as_u64[0] = key.as_u64[1] = 0;
2560  if (icmp_get_ed_key (ip, &key))
2561  return 0;
2562  key.fib_index = 0;
2563  kv.key[0] = key.as_u64[0];
2564  kv.key[1] = key.as_u64[1];
2565  }
2566  else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
2567  {
2568  udp = ip4_next_header(ip);
2569  make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
2570  udp->src_port, udp->dst_port);
2571  }
2572  else
2573  {
2574  make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
2575  0);
2576  }
2577 
2578  if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2579  {
2580  s = pool_elt_at_index (tsm->sessions, value.value);
2581  if (is_fwd_bypass_session (s))
2582  {
2583  if (ip->protocol == IP_PROTOCOL_TCP)
2584  {
2585  tcp_header_t *tcp = ip4_next_header(ip);
2586  if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
2587  return 1;
2588  }
2589  /* Per-user LRU list maintenance */
2590  nat44_session_update_lru (sm, s, thread_index);
2591  /* Accounting */
2593  vlib_buffer_length_in_chain (vm, b));
2594  return 1;
2595  }
2596  else
2597  return 0;
2598  }
2599 
2600  return 0;
2601 }
2602 
2605  u8 proto, u16 src_port, u16 dst_port,
2606  u32 thread_index, u32 rx_sw_if_index,
2607  u32 tx_sw_if_index)
2608 {
2609  clib_bihash_kv_16_8_t kv, value;
2610  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2612  snat_session_t *s;
2613  u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
2614  u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
2615 
2616  /* src NAT check */
2617  make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
2618  src_port, dst_port);
2619  if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2620  return 1;
2621 
2622  /* dst NAT check */
2623  make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
2624  dst_port, src_port);
2625  if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2626  {
2627  s = pool_elt_at_index (tsm->sessions, value.value);
2628  if (is_fwd_bypass_session (s))
2629  return 0;
2630 
2631  /* hairpinning */
2633  ({
2634  if ((nat_interface_is_inside(i)) && (rx_sw_if_index == i->sw_if_index))
2635  return 0;
2636  }));
2637  return 1;
2638  }
2639 
2640  return 0;
2641 }
2642 
2643 u32
2645  u32 thread_index, vlib_buffer_t *b, ip4_header_t *ip,
2646  u8 *p_proto, snat_session_key_t *p_value,
2647  u8 *p_dont_translate, void *d, void *e)
2648 {
2649  icmp46_header_t *icmp;
2650  u32 sw_if_index;
2651  u32 rx_fib_index;
2652  nat_ed_ses_key_t key;
2653  snat_session_t *s = 0;
2654  u8 dont_translate = 0;
2655  clib_bihash_kv_16_8_t kv, value;
2656  u32 next = ~0;
2657  int err;
2658  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2659 
2660  icmp = (icmp46_header_t *) ip4_next_header (ip);
2661  sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
2662  rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2663 
2664  key.as_u64[0] = key.as_u64[1] = 0;
2665  err = icmp_get_ed_key (ip, &key);
2666  if (err != 0)
2667  {
2668  b->error = node->errors[err];
2669  next = SNAT_IN2OUT_NEXT_DROP;
2670  goto out;
2671  }
2672  key.fib_index = rx_fib_index;
2673 
2674  kv.key[0] = key.as_u64[0];
2675  kv.key[1] = key.as_u64[1];
2676 
2677  if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2678  {
2679  if (vnet_buffer(b)->sw_if_index[VLIB_TX] != ~0)
2680  {
2682  key.proto, key.l_port, key.r_port, thread_index, sw_if_index,
2683  vnet_buffer(b)->sw_if_index[VLIB_TX])))
2684  {
2685  dont_translate = 1;
2686  goto out;
2687  }
2688  }
2689  else
2690  {
2691  if (PREDICT_FALSE(nat44_ed_not_translate(sm, node, sw_if_index,
2692  ip, SNAT_PROTOCOL_ICMP, rx_fib_index, thread_index)))
2693  {
2694  dont_translate = 1;
2695  goto out;
2696  }
2697  }
2698 
2700  {
2701  b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2702  next = SNAT_IN2OUT_NEXT_DROP;
2703  goto out;
2704  }
2705 
2706  next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
2707  thread_index);
2708 
2709  if (PREDICT_FALSE (next == SNAT_IN2OUT_NEXT_DROP))
2710  goto out;
2711  }
2712  else
2713  {
2714  if (PREDICT_FALSE(icmp->type != ICMP4_echo_request &&
2715  icmp->type != ICMP4_echo_reply &&
2716  !icmp_is_error_message (icmp)))
2717  {
2718  b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2719  next = SNAT_IN2OUT_NEXT_DROP;
2720  goto out;
2721  }
2722 
2723  s = pool_elt_at_index (tsm->sessions, value.value);
2724  }
2725 
2726  *p_proto = ip_proto_to_snat_proto (key.proto);
2727 out:
2728  if (s)
2729  *p_value = s->out2in;
2730  *p_dont_translate = dont_translate;
2731  if (d)
2732  *(snat_session_t**)d = s;
2733  return next;
2734 }
2735 
2736 static inline void
2738  vlib_buffer_t * b,
2739  ip4_header_t * ip)
2740 {
2741  u32 old_addr, new_addr = 0, ti = 0;
2742  clib_bihash_kv_8_8_t kv, value;
2743  clib_bihash_kv_16_8_t s_kv, s_value;
2745  ip_csum_t sum;
2746  snat_session_t *s;
2748 
2749  if (sm->num_workers > 1)
2750  ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
2751  else
2752  ti = sm->num_workers;
2753  tsm = &sm->per_thread_data[ti];
2754 
2755  old_addr = ip->dst_address.as_u32;
2756  make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2757  sm->outside_fib_index, 0, 0);
2758  if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2759  {
2760  make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
2761  if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2762  return;
2763 
2764  m = pool_elt_at_index (sm->static_mappings, value.value);
2765  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2766  vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
2767  new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
2768  }
2769  else
2770  {
2771  s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
2772  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2773  vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2774  new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
2775  }
2776  sum = ip->checksum;
2777  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2778  ip->checksum = ip_csum_fold (sum);
2779 }
2780 
2781 static snat_session_t *
2783  vlib_buffer_t * b,
2784  ip4_header_t * ip,
2785  u32 rx_fib_index,
2786  u32 thread_index,
2787  f64 now,
2788  vlib_main_t * vm,
2789  vlib_node_runtime_t * node)
2790 {
2791  clib_bihash_kv_8_8_t kv, value;
2792  clib_bihash_kv_16_8_t s_kv, s_value;
2794  u32 old_addr, new_addr = 0;
2795  ip_csum_t sum;
2796  snat_user_t *u;
2797  dlist_elt_t *head, *elt;
2798  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2799  u32 elt_index, head_index, ses_index;
2800  snat_session_t * s;
2801  u32 address_index = ~0, outside_fib_index = sm->outside_fib_index;
2802  int i;
2803  u8 is_sm = 0;
2804  nat_outside_fib_t *outside_fib;
2806  fib_prefix_t pfx = {
2808  .fp_len = 32,
2809  .fp_addr = {
2810  .ip4.as_u32 = ip->dst_address.as_u32,
2811  },
2812  };
2813 
2814  switch (vec_len (sm->outside_fibs))
2815  {
2816  case 0:
2817  outside_fib_index = sm->outside_fib_index;
2818  break;
2819  case 1:
2820  outside_fib_index = sm->outside_fibs[0].fib_index;
2821  break;
2822  default:
2823  vec_foreach (outside_fib, sm->outside_fibs)
2824  {
2825  fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2826  if (FIB_NODE_INDEX_INVALID != fei)
2827  {
2828  if (fib_entry_get_resolving_interface (fei) != ~0)
2829  {
2830  outside_fib_index = outside_fib->fib_index;
2831  break;
2832  }
2833  }
2834  }
2835  break;
2836  }
2837  old_addr = ip->src_address.as_u32;
2838 
2839  make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
2840  rx_fib_index, 0, 0);
2841 
2842  if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
2843  {
2844  s = pool_elt_at_index (tsm->sessions, s_value.value);
2845  new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
2846  }
2847  else
2848  {
2849  if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
2850  {
2851  b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2853  nat_log_notice ("maximum sessions exceeded");
2854  return 0;
2855  }
2856 
2857  u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
2858  thread_index);
2859  if (!u)
2860  {
2861  nat_log_warn ("create NAT user failed");
2862  return 0;
2863  }
2864 
2865  make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
2866 
2867  /* Try to find static mapping first */
2868  if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
2869  {
2870  m = pool_elt_at_index (sm->static_mappings, value.value);
2871  new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
2872  is_sm = 1;
2873  goto create_ses;
2874  }
2875  /* Fallback to 3-tuple key */
2876  else
2877  {
2878  /* Choose same out address as for TCP/UDP session to same destination */
2879  head_index = u->sessions_per_user_list_head_index;
2880  head = pool_elt_at_index (tsm->list_pool, head_index);
2881  elt_index = head->next;
2882  if (PREDICT_FALSE (elt_index == ~0))
2883  ses_index = ~0;
2884  else
2885  {
2886  elt = pool_elt_at_index (tsm->list_pool, elt_index);
2887  ses_index = elt->value;
2888  }
2889 
2890  while (ses_index != ~0)
2891  {
2892  s = pool_elt_at_index (tsm->sessions, ses_index);
2893  elt_index = elt->next;
2894  elt = pool_elt_at_index (tsm->list_pool, elt_index);
2895  ses_index = elt->value;
2896 
2897  if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
2898  {
2899  new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
2900  address_index = s->outside_address_index;
2901 
2902  make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
2903  ip->protocol, outside_fib_index, 0, 0);
2904  if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2905  goto create_ses;
2906 
2907  break;
2908  }
2909  }
2910 
2911  for (i = 0; i < vec_len (sm->addresses); i++)
2912  {
2913  make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
2914  ip->protocol, outside_fib_index, 0, 0);
2915  if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2916  {
2917  new_addr = ip->src_address.as_u32 =
2918  sm->addresses[i].addr.as_u32;
2919  address_index = i;
2920  goto create_ses;
2921  }
2922  }
2923  return 0;
2924  }
2925 
2926 create_ses:
2927  s = nat_session_alloc_or_recycle (sm, u, thread_index);
2928  if (!s)
2929  {
2930  nat_log_warn ("create NAT session failed");
2931  return 0;
2932  }
2933 
2934  s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
2935  s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
2937  s->outside_address_index = address_index;
2938  s->out2in.addr.as_u32 = new_addr;
2939  s->out2in.fib_index = outside_fib_index;
2940  s->in2out.addr.as_u32 = old_addr;
2941  s->in2out.fib_index = rx_fib_index;
2942  s->in2out.port = s->out2in.port = ip->protocol;
2943  if (is_sm)
2945  user_session_increment (sm, u, is_sm);
2946 
2947  /* Add to lookup tables */
2948  make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
2949  rx_fib_index, 0, 0);
2950  s_kv.value = s - tsm->sessions;
2951  if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2952  nat_log_notice ("in2out key add failed");
2953 
2954  make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
2955  outside_fib_index, 0, 0);
2956  s_kv.value = s - tsm->sessions;
2957  if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2958  nat_log_notice ("out2in key add failed");
2959  }
2960 
2961  /* Update IP checksum */
2962  sum = ip->checksum;
2963  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
2964  ip->checksum = ip_csum_fold (sum);
2965 
2966  /* Accounting */
2968  /* Per-user LRU list maintenance */
2969  nat44_session_update_lru (sm, s, thread_index);
2970 
2971  /* Hairpinning */
2972  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2974 
2975  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2976  vnet_buffer(b)->sw_if_index[VLIB_TX] = outside_fib_index;
2977 
2978  return s;
2979 }
2980 
2981 static inline uword
2983  vlib_node_runtime_t * node,
2984  vlib_frame_t * frame, int is_slow_path,
2985  int is_output_feature)
2986 {
2987  u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2988  snat_in2out_next_t next_index;
2989  snat_main_t *sm = &snat_main;
2990  f64 now = vlib_time_now (vm);
2991  u32 thread_index = vm->thread_index;
2992  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2993 
2994  stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
2995  nat44_ed_in2out_node.index;
2996 
2997  from = vlib_frame_vector_args (frame);
2998  n_left_from = frame->n_vectors;
2999  next_index = node->cached_next_index;
3000 
3001  while (n_left_from > 0)
3002  {
3003  u32 n_left_to_next;
3004 
3005  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3006 
3007  while (n_left_from >= 4 && n_left_to_next >= 2)
3008  {
3009  u32 bi0, bi1;
3010  vlib_buffer_t *b0, *b1;
3011  u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3012  new_addr0, old_addr0;
3013  u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
3014  new_addr1, old_addr1;
3015  u16 old_port0, new_port0, old_port1, new_port1;
3016  ip4_header_t *ip0, *ip1;
3017  udp_header_t *udp0, *udp1;
3018  tcp_header_t *tcp0, *tcp1;
3019  icmp46_header_t *icmp0, *icmp1;
3020  snat_session_t *s0 = 0, *s1 = 0;
3021  clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
3022  ip_csum_t sum0, sum1;
3023 
3024  /* Prefetch next iteration. */
3025  {
3026  vlib_buffer_t * p2, * p3;
3027 
3028  p2 = vlib_get_buffer (vm, from[2]);
3029  p3 = vlib_get_buffer (vm, from[3]);
3030 
3031  vlib_prefetch_buffer_header (p2, LOAD);
3032  vlib_prefetch_buffer_header (p3, LOAD);
3033 
3036  }
3037 
3038  /* speculatively enqueue b0 and b1 to the current next frame */
3039  to_next[0] = bi0 = from[0];
3040  to_next[1] = bi1 = from[1];
3041  from += 2;
3042  to_next += 2;
3043  n_left_from -= 2;
3044  n_left_to_next -= 2;
3045 
3046  b0 = vlib_get_buffer (vm, bi0);
3047  b1 = vlib_get_buffer (vm, bi1);
3048 
3049  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3050 
3051  if (is_output_feature)
3052  iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3053 
3054  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3055  iph_offset0);
3056 
3057  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3059  sw_if_index0);
3060 
3061  if (PREDICT_FALSE(ip0->ttl == 1))
3062  {
3063  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3064  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3065  ICMP4_time_exceeded_ttl_exceeded_in_transit,
3066  0);
3068  goto trace00;
3069  }
3070 
3071  udp0 = ip4_next_header (ip0);
3072  tcp0 = (tcp_header_t *) udp0;
3073  icmp0 = (icmp46_header_t *) udp0;
3074  proto0 = ip_proto_to_snat_proto (ip0->protocol);
3075 
3076  if (is_slow_path)
3077  {
3078  if (PREDICT_FALSE (proto0 == ~0))
3079  {
3080  s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3081  rx_fib_index0,
3082  thread_index, now, vm,
3083  node);
3084  if (!s0)
3085  next0 = SNAT_IN2OUT_NEXT_DROP;
3086  goto trace00;
3087  }
3088 
3089  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3090  {
3091  next0 = icmp_in2out_slow_path
3092  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3093  next0, now, thread_index, &s0);
3094  goto trace00;
3095  }
3096  }
3097  else
3098  {
3099  if (is_output_feature)
3100  {
3102  sm, ip0, thread_index, now, vm, b0)))
3103  goto trace00;
3104  }
3105 
3106  if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3107  {
3109  goto trace00;
3110  }
3111 
3112  if (ip4_is_fragment (ip0))
3113  {
3114  b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3115  next0 = SNAT_IN2OUT_NEXT_DROP;
3116  goto trace00;
3117  }
3118  }
3119 
3120  make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3121  rx_fib_index0, udp0->src_port, udp0->dst_port);
3122 
3123  if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3124  {
3125  if (is_slow_path)
3126  {
3127  if (is_output_feature)
3128  {
3130  sm, ip0, ip0->protocol, udp0->src_port,
3131  udp0->dst_port, thread_index, sw_if_index0,
3132  vnet_buffer(b0)->sw_if_index[VLIB_TX])))
3133  goto trace00;
3134  }
3135  else
3136  {
3138  sw_if_index0, ip0, proto0, rx_fib_index0,
3139  thread_index)))
3140  goto trace00;
3141  }
3142 
3143  next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3144  next0, thread_index);
3145 
3146  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3147  goto trace00;
3148  }
3149  else
3150  {
3152  goto trace00;
3153  }
3154  }
3155  else
3156  {
3157  s0 = pool_elt_at_index (tsm->sessions, value0.value);
3158  }
3159 
3160  b0->flags |= VNET_BUFFER_F_IS_NATED;
3161 
3162  if (!is_output_feature)
3163  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3164 
3165  old_addr0 = ip0->src_address.as_u32;
3166  new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3167  sum0 = ip0->checksum;
3168  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3169  src_address);
3171  sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3172  s0->ext_host_addr.as_u32, ip4_header_t,
3173  dst_address);
3174  ip0->checksum = ip_csum_fold (sum0);
3175 
3176  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3177  {
3178  old_port0 = tcp0->src_port;
3179  new_port0 = tcp0->src_port = s0->out2in.port;
3180 
3181  sum0 = tcp0->checksum;
3182  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3183  dst_address);
3184  sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3185  length);
3187  {
3188  sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3189  s0->ext_host_addr.as_u32,
3190  ip4_header_t, dst_address);
3191  sum0 = ip_csum_update (sum0, tcp0->dst_port,
3192  s0->ext_host_port, ip4_header_t,
3193  length);
3194  tcp0->dst_port = s0->ext_host_port;
3195  ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3196  }
3197  tcp0->checksum = ip_csum_fold(sum0);
3198  if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3199  goto trace00;
3200  }
3201  else
3202  {
3203  udp0->src_port = s0->out2in.port;
3204  udp0->checksum = 0;
3206  {
3207  udp0->dst_port = s0->ext_host_port;
3208  ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3209  }
3210  }
3211 
3212  /* Accounting */
3214  vlib_buffer_length_in_chain (vm, b0));
3215  /* Per-user LRU list maintenance */
3216  nat44_session_update_lru (sm, s0, thread_index);
3217 
3218  trace00:
3220  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3221  {
3222  snat_in2out_trace_t *t =
3223  vlib_add_trace (vm, node, b0, sizeof (*t));
3224  t->is_slow_path = is_slow_path;
3225  t->sw_if_index = sw_if_index0;
3226  t->next_index = next0;
3227  t->session_index = ~0;
3228  if (s0)
3229  t->session_index = s0 - tsm->sessions;
3230  }
3231 
3232  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3233 
3234 
3235  next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3236 
3237  if (is_output_feature)
3238  iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
3239 
3240  ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
3241  iph_offset1);
3242 
3243  sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3245  sw_if_index1);
3246 
3247  if (PREDICT_FALSE(ip1->ttl == 1))
3248  {
3249  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3250  icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3251  ICMP4_time_exceeded_ttl_exceeded_in_transit,
3252  0);
3254  goto trace01;
3255  }
3256 
3257  udp1 = ip4_next_header (ip1);
3258  tcp1 = (tcp_header_t *) udp1;
3259  icmp1 = (icmp46_header_t *) udp1;
3260  proto1 = ip_proto_to_snat_proto (ip1->protocol);
3261 
3262  if (is_slow_path)
3263  {
3264  if (PREDICT_FALSE (proto1 == ~0))
3265  {
3266  s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
3267  rx_fib_index1,
3268  thread_index, now, vm,
3269  node);
3270  if (!s1)
3271  next1 = SNAT_IN2OUT_NEXT_DROP;
3272  goto trace01;
3273  }
3274 
3275  if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
3276  {
3277  next1 = icmp_in2out_slow_path
3278  (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
3279  next1, now, thread_index, &s1);
3280  goto trace01;
3281  }
3282  }
3283  else
3284  {
3285  if (is_output_feature)
3286  {
3288  sm, ip1, thread_index, now, vm, b1)))
3289  goto trace01;
3290  }
3291 
3292  if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
3293  {
3295  goto trace01;
3296  }
3297 
3298  if (ip4_is_fragment (ip1))
3299  {
3300  b1->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3301  next1 = SNAT_IN2OUT_NEXT_DROP;
3302  goto trace01;
3303  }
3304  }
3305 
3306  make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address, ip1->protocol,
3307  rx_fib_index1, udp1->src_port, udp1->dst_port);
3308 
3309  if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
3310  {
3311  if (is_slow_path)
3312  {
3313  if (is_output_feature)
3314  {
3316  sm, ip1, ip1->protocol, udp1->src_port,
3317  udp1->dst_port, thread_index, sw_if_index1,
3318  vnet_buffer(b1)->sw_if_index[VLIB_TX])))
3319  goto trace01;
3320  }
3321  else
3322  {
3324  sw_if_index1, ip1, proto1, rx_fib_index1,
3325  thread_index)))
3326  goto trace01;
3327  }
3328 
3329  next1 = slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
3330  next1, thread_index);
3331 
3332  if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
3333  goto trace01;
3334  }
3335  else
3336  {
3338  goto trace01;
3339  }
3340  }
3341  else
3342  {
3343  s1 = pool_elt_at_index (tsm->sessions, value1.value);
3344  }
3345 
3346  b1->flags |= VNET_BUFFER_F_IS_NATED;
3347 
3348  if (!is_output_feature)
3349  vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
3350 
3351  old_addr1 = ip1->src_address.as_u32;
3352  new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
3353  sum1 = ip1->checksum;
3354  sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3355  src_address);
3357  sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3358  s1->ext_host_addr.as_u32, ip4_header_t,
3359  dst_address);
3360  ip1->checksum = ip_csum_fold (sum1);
3361 
3362  if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
3363  {
3364  old_port1 = tcp1->src_port;
3365  new_port1 = tcp1->src_port = s1->out2in.port;
3366 
3367  sum1 = tcp1->checksum;
3368  sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3369  dst_address);
3370  sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
3371  length);
3373  {
3374  sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3375  s1->ext_host_addr.as_u32,
3376  ip4_header_t, dst_address);
3377  sum1 = ip_csum_update (sum1, tcp1->dst_port,
3378  s1->ext_host_port, ip4_header_t,
3379  length);
3380  tcp1->dst_port = s1->ext_host_port;
3381  ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3382  }
3383  tcp1->checksum = ip_csum_fold(sum1);
3384  if (nat44_set_tcp_session_state_i2o (sm, s1, tcp1, thread_index))
3385  goto trace01;
3386  }
3387  else
3388  {
3389  udp1->src_port = s1->out2in.port;
3390  udp1->checksum = 0;
3392  {
3393  udp1->dst_port = s1->ext_host_port;
3394  ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3395  }
3396  }
3397 
3398  /* Accounting */
3400  vlib_buffer_length_in_chain (vm, b1));
3401  /* Per-user LRU list maintenance */
3402  nat44_session_update_lru (sm, s1, thread_index);
3403 
3404  trace01:
3406  && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3407  {
3408  snat_in2out_trace_t *t =
3409  vlib_add_trace (vm, node, b1, sizeof (*t));
3410  t->is_slow_path = is_slow_path;
3411  t->sw_if_index = sw_if_index1;
3412  t->next_index = next1;
3413  t->session_index = ~0;
3414  if (s1)
3415  t->session_index = s1 - tsm->sessions;
3416  }
3417 
3418  pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3419 
3420  /* verify speculative enqueues, maybe switch current next frame */
3421  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3422  to_next, n_left_to_next,
3423  bi0, bi1, next0, next1);
3424  }
3425 
3426  while (n_left_from > 0 && n_left_to_next > 0)
3427  {
3428  u32 bi0;
3429  vlib_buffer_t *b0;
3430  u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3431  new_addr0, old_addr0;
3432  u16 old_port0, new_port0;
3433  ip4_header_t *ip0;
3434  udp_header_t *udp0;
3435  tcp_header_t *tcp0;
3436  icmp46_header_t * icmp0;
3437  snat_session_t *s0 = 0;
3438  clib_bihash_kv_16_8_t kv0, value0;
3439  ip_csum_t sum0;
3440 
3441  /* speculatively enqueue b0 to the current next frame */
3442  bi0 = from[0];
3443  to_next[0] = bi0;
3444  from += 1;
3445  to_next += 1;
3446  n_left_from -= 1;
3447  n_left_to_next -= 1;
3448 
3449  b0 = vlib_get_buffer (vm, bi0);
3450  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3451 
3452  if (is_output_feature)
3453  iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3454 
3455  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3456  iph_offset0);
3457 
3458  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3460  sw_if_index0);
3461 
3462  if (PREDICT_FALSE(ip0->ttl == 1))
3463  {
3464  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3465  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3466  ICMP4_time_exceeded_ttl_exceeded_in_transit,
3467  0);
3469  goto trace0;
3470  }
3471 
3472  udp0 = ip4_next_header (ip0);
3473  tcp0 = (tcp_header_t *) udp0;
3474  icmp0 = (icmp46_header_t *) udp0;
3475  proto0 = ip_proto_to_snat_proto (ip0->protocol);
3476 
3477  if (is_slow_path)
3478  {
3479  if (PREDICT_FALSE (proto0 == ~0))
3480  {
3481  s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3482  rx_fib_index0,
3483  thread_index, now, vm,
3484  node);
3485  if (!s0)
3486  next0 = SNAT_IN2OUT_NEXT_DROP;
3487  goto trace0;
3488  }
3489 
3490  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3491  {
3492  next0 = icmp_in2out_slow_path
3493  (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3494  next0, now, thread_index, &s0);
3495  goto trace0;
3496  }
3497  }
3498  else
3499  {
3500  if (is_output_feature)
3501  {
3503  sm, ip0, thread_index, now, vm, b0)))
3504  goto trace0;
3505  }
3506 
3507  if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3508  {
3510  goto trace0;
3511  }
3512 
3513  if (ip4_is_fragment (ip0))
3514  {
3515  b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3516  next0 = SNAT_IN2OUT_NEXT_DROP;
3517  goto trace0;
3518  }
3519  }
3520 
3521  make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3522  rx_fib_index0, udp0->src_port, udp0->dst_port);
3523 
3524  if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3525  {
3526  if (is_slow_path)
3527  {
3528  if (is_output_feature)
3529  {
3531  sm, ip0, ip0->protocol, udp0->src_port,
3532  udp0->dst_port, thread_index, sw_if_index0,
3533  vnet_buffer(b0)->sw_if_index[VLIB_TX])))
3534  goto trace0;
3535  }
3536  else
3537  {
3539  sw_if_index0, ip0, proto0, rx_fib_index0,
3540  thread_index)))
3541  goto trace0;
3542  }
3543 
3544  next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3545  next0, thread_index);
3546 
3547  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3548  goto trace0;
3549  }
3550  else
3551  {
3553  goto trace0;
3554  }
3555  }
3556  else
3557  {
3558  s0 = pool_elt_at_index (tsm->sessions, value0.value);
3559  }
3560 
3561  b0->flags |= VNET_BUFFER_F_IS_NATED;
3562 
3563  if (!is_output_feature)
3564  vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3565 
3566  old_addr0 = ip0->src_address.as_u32;
3567  new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3568  sum0 = ip0->checksum;
3569  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3570  src_address);
3572  sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3573  s0->ext_host_addr.as_u32, ip4_header_t,
3574  dst_address);
3575  ip0->checksum = ip_csum_fold (sum0);
3576 
3577  if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3578  {
3579  old_port0 = tcp0->src_port;
3580  new_port0 = tcp0->src_port = s0->out2in.port;
3581 
3582  sum0 = tcp0->checksum;
3583  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3584  dst_address);
3585  sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3586  length);
3588  {
3589  sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3590  s0->ext_host_addr.as_u32,
3591  ip4_header_t, dst_address);
3592  sum0 = ip_csum_update (sum0, tcp0->dst_port,
3593  s0->ext_host_port, ip4_header_t,
3594  length);
3595  tcp0->dst_port = s0->ext_host_port;
3596  ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3597  }
3598  tcp0->checksum = ip_csum_fold(sum0);
3599  if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3600  goto trace0;
3601  }
3602  else
3603  {
3604  udp0->src_port = s0->out2in.port;
3605  udp0->checksum = 0;
3607  {
3608  udp0->dst_port = s0->ext_host_port;
3609  ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3610  }
3611  }
3612 
3613  /* Accounting */
3615  vlib_buffer_length_in_chain (vm, b0));
3616  /* Per-user LRU list maintenance */
3617  nat44_session_update_lru (sm, s0, thread_index);
3618 
3619  trace0:
3621  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3622  {
3623  snat_in2out_trace_t *t =
3624  vlib_add_trace (vm, node, b0, sizeof (*t));
3625  t->is_slow_path = is_slow_path;
3626  t->sw_if_index = sw_if_index0;
3627  t->next_index = next0;
3628  t->session_index = ~0;
3629  if (s0)
3630  t->session_index = s0 - tsm->sessions;
3631  }
3632 
3633  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3634 
3635  /* verify speculative enqueue, maybe switch current next frame */
3636  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3637  to_next, n_left_to_next,
3638  bi0, next0);
3639  }
3640 
3641  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3642  }
3643 
3644  vlib_node_increment_counter (vm, stats_node_index,
3645  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3646  pkts_processed);
3647  return frame->n_vectors;
3648 }
3649 
3650 static uword
3652  vlib_node_runtime_t * node,
3653  vlib_frame_t * frame)
3654 {
3655  return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
3656 }
3657 
3659  .function = nat44_ed_in2out_fast_path_fn,
3660  .name = "nat44-ed-in2out",
3661  .vector_size = sizeof (u32),
3662  .format_trace = format_snat_in2out_trace,
3663  .type = VLIB_NODE_TYPE_INTERNAL,
3664 
3665  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3666  .error_strings = snat_in2out_error_strings,
3667 
3668  .runtime_data_bytes = sizeof (snat_runtime_t),
3669 
3670  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3671 
3672  /* edit / add dispositions here */
3673  .next_nodes = {
3674  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3675  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3676  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3677  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3678  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3679  },
3680 };
3681 
3683 
3684 static uword
3686  vlib_node_runtime_t * node,
3687  vlib_frame_t * frame)
3688 {
3689  return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
3690 }
3691 
3694  .name = "nat44-ed-in2out-output",
3695  .vector_size = sizeof (u32),
3696  .format_trace = format_snat_in2out_trace,
3697  .type = VLIB_NODE_TYPE_INTERNAL,
3698 
3699  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3700  .error_strings = snat_in2out_error_strings,
3701 
3702  .runtime_data_bytes = sizeof (snat_runtime_t),
3703 
3704  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3705 
3706  /* edit / add dispositions here */
3707  .next_nodes = {
3708  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3709  [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3710  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3711  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3712  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3713  },
3714 };
3715 
3718 
3719 static uword
3721  vlib_node_runtime_t * node,
3722  vlib_frame_t * frame)
3723 {
3724  return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
3725 }
3726 
3728  .function = nat44_ed_in2out_slow_path_fn,
3729  .name = "nat44-ed-in2out-slowpath",
3730  .vector_size = sizeof (u32),
3731  .format_trace = format_snat_in2out_trace,
3732  .type = VLIB_NODE_TYPE_INTERNAL,
3733 
3734  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3735  .error_strings = snat_in2out_error_strings,
3736 
3737  .runtime_data_bytes = sizeof (snat_runtime_t),
3738 
3739  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3740 
3741  /* edit / add dispositions here */
3742  .next_nodes = {
3743  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3744  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3745  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3746  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3747  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3748  },
3749 };
3750 
3753 
3754 static uword
3756  vlib_node_runtime_t * node,
3757  vlib_frame_t * frame)
3758 {
3759  return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
3760 }
3761 
3764  .name = "nat44-ed-in2out-output-slowpath",
3765  .vector_size = sizeof (u32),
3766  .format_trace = format_snat_in2out_trace,
3767  .type = VLIB_NODE_TYPE_INTERNAL,
3768 
3769  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3770  .error_strings = snat_in2out_error_strings,
3771 
3772  .runtime_data_bytes = sizeof (snat_runtime_t),
3773 
3774  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3775 
3776  /* edit / add dispositions here */
3777  .next_nodes = {
3778  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3779  [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3780  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3781  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3782  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3783  },
3784 };
3785 
3788 
3789 /**************************/
3790 /*** deterministic mode ***/
3791 /**************************/
3792 static uword
3794  vlib_node_runtime_t * node,
3795  vlib_frame_t * frame)
3796 {
3797  u32 n_left_from, * from, * to_next;
3798  snat_in2out_next_t next_index;
3799  u32 pkts_processed = 0;
3800  snat_main_t * sm = &snat_main;
3801  u32 now = (u32) vlib_time_now (vm);
3802  u32 thread_index = vm->thread_index;
3803 
3804  from = vlib_frame_vector_args (frame);
3805  n_left_from = frame->n_vectors;
3806  next_index = node->cached_next_index;
3807 
3808  while (n_left_from > 0)
3809  {
3810  u32 n_left_to_next;
3811 
3812  vlib_get_next_frame (vm, node, next_index,
3813  to_next, n_left_to_next);
3814 
3815  while (n_left_from >= 4 && n_left_to_next >= 2)
3816  {
3817  u32 bi0, bi1;
3818  vlib_buffer_t * b0, * b1;
3819  u32 next0, next1;
3820  u32 sw_if_index0, sw_if_index1;
3821  ip4_header_t * ip0, * ip1;
3822  ip_csum_t sum0, sum1;
3823  ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
3824  u16 old_port0, new_port0, lo_port0, i0;
3825  u16 old_port1, new_port1, lo_port1, i1;
3826  udp_header_t * udp0, * udp1;
3827  tcp_header_t * tcp0, * tcp1;
3828  u32 proto0, proto1;
3829  snat_det_out_key_t key0, key1;
3830  snat_det_map_t * dm0, * dm1;
3831  snat_det_session_t * ses0 = 0, * ses1 = 0;
3832  u32 rx_fib_index0, rx_fib_index1;
3833  icmp46_header_t * icmp0, * icmp1;
3834 
3835  /* Prefetch next iteration. */
3836  {
3837  vlib_buffer_t * p2, * p3;
3838 
3839  p2 = vlib_get_buffer (vm, from[2]);
3840  p3 = vlib_get_buffer (vm, from[3]);
3841 
3842  vlib_prefetch_buffer_header (p2, LOAD);
3843  vlib_prefetch_buffer_header (p3, LOAD);
3844 
3847  }
3848 
3849  /* speculatively enqueue b0 and b1 to the current next frame */
3850  to_next[0] = bi0 = from[0];
3851  to_next[1] = bi1 = from[1];
3852  from += 2;
3853  to_next += 2;
3854  n_left_from -= 2;
3855  n_left_to_next -= 2;
3856 
3857  b0 = vlib_get_buffer (vm, bi0);
3858  b1 = vlib_get_buffer (vm, bi1);
3859 
3860  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3861  next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3862 
3863  ip0 = vlib_buffer_get_current (b0);
3864  udp0 = ip4_next_header (ip0);
3865  tcp0 = (tcp_header_t *) udp0;
3866 
3867  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3868 
3869  if (PREDICT_FALSE(ip0->ttl == 1))
3870  {
3871  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3872  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3873  ICMP4_time_exceeded_ttl_exceeded_in_transit,
3874  0);
3876  goto trace0;
3877  }
3878 
3879  proto0 = ip_proto_to_snat_proto (ip0->protocol);
3880 
3881  if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3882  {
3883  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3884  icmp0 = (icmp46_header_t *) udp0;
3885 
3886  next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3887  rx_fib_index0, node, next0, thread_index,
3888  &ses0, &dm0);
3889  goto trace0;
3890  }
3891 
3892  dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3893  if (PREDICT_FALSE(!dm0))
3894  {
3895  nat_log_info ("no match for internal host %U",
3897  next0 = SNAT_IN2OUT_NEXT_DROP;
3898  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3899  goto trace0;
3900  }
3901 
3902  snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3903 
3904  key0.ext_host_addr = ip0->dst_address;
3905  key0.ext_host_port = tcp0->dst;
3906 
3907  ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3908  if (PREDICT_FALSE(!ses0))
3909  {
3910  for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3911  {
3912  key0.out_port = clib_host_to_net_u16 (lo_port0 +
3913  ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3914 
3915  if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3916  continue;
3917 
3918  ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3919  break;
3920  }
3921  if (PREDICT_FALSE(!ses0))
3922  {
3923  /* too many sessions for user, send ICMP error packet */
3924 
3925  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3926  icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3927  ICMP4_destination_unreachable_destination_unreachable_host,
3928  0);
3930  goto trace0;
3931  }
3932  }
3933 
3934  new_port0 = ses0->out.out_port;
3935 
3936  old_addr0.as_u32 = ip0->src_address.as_u32;
3937  ip0->src_address.as_u32 = new_addr0.as_u32;
3938  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3939 
3940  sum0 = ip0->checksum;
3941  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3942  ip4_header_t,
3943  src_address /* changed member */);
3944  ip0->checksum = ip_csum_fold (sum0);
3945 
3946  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3947  {
3948  if (tcp0->flags & TCP_FLAG_SYN)
3949  ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3950  else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3951  ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3952  else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3953  ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3954  else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3955  snat_det_ses_close(dm0, ses0);
3956  else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3957  ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3958  else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3959  ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3960 
3961  old_port0 = tcp0->src;
3962  tcp0->src = new_port0;
3963 
3964  sum0 = tcp0->checksum;
3965  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3966  ip4_header_t,
3967  dst_address /* changed member */);
3968  sum0 = ip_csum_update (sum0, old_port0, new_port0,
3969  ip4_header_t /* cheat */,
3970  length /* changed member */);
3971  tcp0->checksum = ip_csum_fold(sum0);
3972  }
3973  else
3974  {
3975  ses0->state = SNAT_SESSION_UDP_ACTIVE;
3976  old_port0 = udp0->src_port;
3977  udp0->src_port = new_port0;
3978  udp0->checksum = 0;
3979  }
3980 
3981  switch(ses0->state)
3982  {
3983  case SNAT_SESSION_UDP_ACTIVE:
3984  ses0->expire = now + sm->udp_timeout;
3985  break;
3986  case SNAT_SESSION_TCP_SYN_SENT:
3987  case SNAT_SESSION_TCP_FIN_WAIT:
3988  case SNAT_SESSION_TCP_CLOSE_WAIT:
3989  case SNAT_SESSION_TCP_LAST_ACK:
3990  ses0->expire = now + sm->tcp_transitory_timeout;
3991  break;
3992  case SNAT_SESSION_TCP_ESTABLISHED:
3993  ses0->expire = now + sm->tcp_established_timeout;
3994  break;
3995  }
3996 
3997  trace0:
3999  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4000  {
4001  snat_in2out_trace_t *t =
4002  vlib_add_trace (vm, node, b0, sizeof (*t));
4003  t->is_slow_path = 0;
4004  t->sw_if_index = sw_if_index0;
4005  t->next_index = next0;
4006  t->session_index = ~0;
4007  if (ses0)
4008  t->session_index = ses0 - dm0->sessions;
4009  }
4010 
4011  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4012 
4013  ip1 = vlib_buffer_get_current (b1);
4014  udp1 = ip4_next_header (ip1);
4015  tcp1 = (tcp_header_t *) udp1;
4016 
4017  sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
4018 
4019  if (PREDICT_FALSE(ip1->ttl == 1))
4020  {
4021  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4022  icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
4023  ICMP4_time_exceeded_ttl_exceeded_in_transit,
4024  0);
4026  goto trace1;
4027  }
4028 
4029  proto1 = ip_proto_to_snat_proto (ip1->protocol);
4030 
4031  if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
4032  {
4033  rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
4034  icmp1 = (icmp46_header_t *) udp1;
4035 
4036  next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
4037  rx_fib_index1, node, next1, thread_index,
4038  &ses1, &dm1);
4039  goto trace1;
4040  }
4041 
4042  dm1 = snat_det_map_by_user(sm, &ip1->src_address);
4043  if (PREDICT_FALSE(!dm1))
4044  {
4045  nat_log_info ("no match for internal host %U",
4047  next1 = SNAT_IN2OUT_NEXT_DROP;
4048  b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4049  goto trace1;
4050  }
4051 
4052  snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
4053 
4054  key1.ext_host_addr = ip1->dst_address;
4055  key1.ext_host_port = tcp1->dst;
4056 
4057  ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
4058  if (PREDICT_FALSE(!ses1))
4059  {
4060  for (i1 = 0; i1 < dm1->ports_per_host; i1++)
4061  {
4062  key1.out_port = clib_host_to_net_u16 (lo_port1 +
4063  ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
4064 
4065  if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
4066  continue;
4067 
4068  ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
4069  break;
4070  }
4071  if (PREDICT_FALSE(!ses1))
4072  {
4073  /* too many sessions for user, send ICMP error packet */
4074 
4075  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4076  icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
4077  ICMP4_destination_unreachable_destination_unreachable_host,
4078  0);
4080  goto trace1;
4081  }
4082  }
4083 
4084  new_port1 = ses1->out.out_port;
4085 
4086  old_addr1.as_u32 = ip1->src_address.as_u32;
4087  ip1->src_address.as_u32 = new_addr1.as_u32;
4088  vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4089 
4090  sum1 = ip1->checksum;
4091  sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4092  ip4_header_t,
4093  src_address /* changed member */);
4094  ip1->checksum = ip_csum_fold (sum1);
4095 
4096  if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
4097  {
4098  if (tcp1->flags & TCP_FLAG_SYN)
4099  ses1->state = SNAT_SESSION_TCP_SYN_SENT;
4100  else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
4101  ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4102  else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
4103  ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
4104  else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
4105  snat_det_ses_close(dm1, ses1);
4106  else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4107  ses1->state = SNAT_SESSION_TCP_LAST_ACK;
4108  else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
4109  ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4110 
4111  old_port1 = tcp1->src;
4112  tcp1->src = new_port1;
4113 
4114  sum1 = tcp1->checksum;
4115  sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4116  ip4_header_t,
4117  dst_address /* changed member */);
4118  sum1 = ip_csum_update (sum1, old_port1, new_port1,
4119  ip4_header_t /* cheat */,
4120  length /* changed member */);
4121  tcp1->checksum = ip_csum_fold(sum1);
4122  }
4123  else
4124  {
4125  ses1->state = SNAT_SESSION_UDP_ACTIVE;
4126  old_port1 = udp1->src_port;
4127  udp1->src_port = new_port1;
4128  udp1->checksum = 0;
4129  }
4130 
4131  switch(ses1->state)
4132  {
4133  case SNAT_SESSION_UDP_ACTIVE:
4134  ses1->expire = now + sm->udp_timeout;
4135  break;
4136  case SNAT_SESSION_TCP_SYN_SENT:
4137  case SNAT_SESSION_TCP_FIN_WAIT:
4138  case SNAT_SESSION_TCP_CLOSE_WAIT:
4139  case SNAT_SESSION_TCP_LAST_ACK:
4140  ses1->expire = now + sm->tcp_transitory_timeout;
4141  break;
4142  case SNAT_SESSION_TCP_ESTABLISHED:
4143  ses1->expire = now + sm->tcp_established_timeout;
4144  break;
4145  }
4146 
4147  trace1:
4149  && (b1->flags & VLIB_BUFFER_IS_TRACED)))
4150  {
4151  snat_in2out_trace_t *t =
4152  vlib_add_trace (vm, node, b1, sizeof (*t));
4153  t->is_slow_path = 0;
4154  t->sw_if_index = sw_if_index1;
4155  t->next_index = next1;
4156  t->session_index = ~0;
4157  if (ses1)
4158  t->session_index = ses1 - dm1->sessions;
4159  }
4160 
4161  pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
4162 
4163  /* verify speculative enqueues, maybe switch current next frame */
4164  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
4165  to_next, n_left_to_next,
4166  bi0, bi1, next0, next1);
4167  }
4168 
4169  while (n_left_from > 0 && n_left_to_next > 0)
4170  {
4171  u32 bi0;
4172  vlib_buffer_t * b0;
4173  u32 next0;
4174  u32 sw_if_index0;
4175  ip4_header_t * ip0;
4176  ip_csum_t sum0;
4177  ip4_address_t new_addr0, old_addr0;
4178  u16 old_port0, new_port0, lo_port0, i0;
4179  udp_header_t * udp0;
4180  tcp_header_t * tcp0;
4181  u32 proto0;
4182  snat_det_out_key_t key0;
4183  snat_det_map_t * dm0;
4184  snat_det_session_t * ses0 = 0;
4185  u32 rx_fib_index0;
4186  icmp46_header_t * icmp0;
4187 
4188  /* speculatively enqueue b0 to the current next frame */
4189  bi0 = from[0];
4190  to_next[0] = bi0;
4191  from += 1;
4192  to_next += 1;
4193  n_left_from -= 1;
4194  n_left_to_next -= 1;
4195 
4196  b0 = vlib_get_buffer (vm, bi0);
4197  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4198 
4199  ip0 = vlib_buffer_get_current (b0);
4200  udp0 = ip4_next_header (ip0);
4201  tcp0 = (tcp_header_t *) udp0;
4202 
4203  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4204 
4205  if (PREDICT_FALSE(ip0->ttl == 1))
4206  {
4207  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4208  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4209  ICMP4_time_exceeded_ttl_exceeded_in_transit,
4210  0);
4212  goto trace00;
4213  }
4214 
4215  proto0 = ip_proto_to_snat_proto (ip0->protocol);
4216 
4217  if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4218  {
4219  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4220  icmp0 = (icmp46_header_t *) udp0;
4221 
4222  next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4223  rx_fib_index0, node, next0, thread_index,
4224  &ses0, &dm0);
4225  goto trace00;
4226  }
4227 
4228  dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4229  if (PREDICT_FALSE(!dm0))
4230  {
4231  nat_log_info ("no match for internal host %U",
4233  next0 = SNAT_IN2OUT_NEXT_DROP;
4234  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4235  goto trace00;
4236  }
4237 
4238  snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4239 
4240  key0.ext_host_addr = ip0->dst_address;
4241  key0.ext_host_port = tcp0->dst;
4242 
4243  ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4244  if (PREDICT_FALSE(!ses0))
4245  {
4246  for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4247  {
4248  key0.out_port = clib_host_to_net_u16 (lo_port0 +
4249  ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4250 
4251  if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4252  continue;
4253 
4254  ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4255  break;
4256  }
4257  if (PREDICT_FALSE(!ses0))
4258  {
4259  /* too many sessions for user, send ICMP error packet */
4260 
4261  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4262  icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4263  ICMP4_destination_unreachable_destination_unreachable_host,
4264  0);
4266  goto trace00;
4267  }
4268  }
4269 
4270  new_port0 = ses0->out.out_port;
4271 
4272  old_addr0.as_u32 = ip0->src_address.as_u32;
4273  ip0->src_address.as_u32 = new_addr0.as_u32;
4274  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4275 
4276  sum0 = ip0->checksum;
4277  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4278  ip4_header_t,
4279  src_address /* changed member */);
4280  ip0->checksum = ip_csum_fold (sum0);
4281 
4282  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4283  {
4284  if (tcp0->flags & TCP_FLAG_SYN)
4285  ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4286  else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4287  ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4288  else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4289  ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4290  else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4291  snat_det_ses_close(dm0, ses0);
4292  else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4293  ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4294  else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4295  ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4296 
4297  old_port0 = tcp0->src;
4298  tcp0->src = new_port0;
4299 
4300  sum0 = tcp0->checksum;
4301  sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4302  ip4_header_t,
4303  dst_address /* changed member */);
4304  sum0 = ip_csum_update (sum0, old_port0, new_port0,
4305  ip4_header_t /* cheat */,
4306  length /* changed member */);
4307  tcp0->checksum = ip_csum_fold(sum0);
4308  }
4309  else
4310  {
4311  ses0->state = SNAT_SESSION_UDP_ACTIVE;
4312  old_port0 = udp0->src_port;
4313  udp0->src_port = new_port0;
4314  udp0->checksum = 0;
4315  }
4316 
4317  switch(ses0->state)
4318  {
4319  case SNAT_SESSION_UDP_ACTIVE:
4320  ses0->expire = now + sm->udp_timeout;
4321  break;
4322  case SNAT_SESSION_TCP_SYN_SENT:
4323  case SNAT_SESSION_TCP_FIN_WAIT:
4324  case SNAT_SESSION_TCP_CLOSE_WAIT:
4325  case SNAT_SESSION_TCP_LAST_ACK:
4326  ses0->expire = now + sm->tcp_transitory_timeout;
4327  break;
4328  case SNAT_SESSION_TCP_ESTABLISHED:
4329  ses0->expire = now + sm->tcp_established_timeout;
4330  break;
4331  }
4332 
4333  trace00:
4335  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4336  {
4337  snat_in2out_trace_t *t =
4338  vlib_add_trace (vm, node, b0, sizeof (*t));
4339  t->is_slow_path = 0;
4340  t->sw_if_index = sw_if_index0;
4341  t->next_index = next0;
4342  t->session_index = ~0;
4343  if (ses0)
4344  t->session_index = ses0 - dm0->sessions;
4345  }
4346 
4347  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4348 
4349  /* verify speculative enqueue, maybe switch current next frame */
4350  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4351  to_next, n_left_to_next,
4352  bi0, next0);
4353  }
4354 
4355  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4356  }
4357 
4359  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4360  pkts_processed);
4361  return frame->n_vectors;
4362 }
4363 
4365  .function = snat_det_in2out_node_fn,
4366  .name = "nat44-det-in2out",
4367  .vector_size = sizeof (u32),
4368  .format_trace = format_snat_in2out_trace,
4369  .type = VLIB_NODE_TYPE_INTERNAL,
4370 
4371  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4372  .error_strings = snat_in2out_error_strings,
4373 
4374  .runtime_data_bytes = sizeof (snat_runtime_t),
4375 
4376  .n_next_nodes = 3,
4377 
4378  /* edit / add dispositions here */
4379  .next_nodes = {
4380  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4381  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4382  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4383  },
4384 };
4385 
4387 
4388 /**
4389  * Get address and port values to be used for ICMP packet translation
4390  * and create session if needed
4391  *
4392  * @param[in,out] sm NAT main
4393  * @param[in,out] node NAT node runtime
4394  * @param[in] thread_index thread index
4395  * @param[in,out] b0 buffer containing packet to be translated
4396  * @param[out] p_proto protocol used for matching
4397  * @param[out] p_value address and port after NAT translation
4398  * @param[out] p_dont_translate if packet should not be translated
4399  * @param d optional parameter
4400  * @param e optional parameter
4401  */
4403  u32 thread_index, vlib_buffer_t *b0,
4404  ip4_header_t *ip0, u8 *p_proto,
4405  snat_session_key_t *p_value,
4406  u8 *p_dont_translate, void *d, void *e)
4407 {
4408  icmp46_header_t *icmp0;
4409  u32 sw_if_index0;
4410  u32 rx_fib_index0;
4411  u8 protocol;
4412  snat_det_out_key_t key0;
4413  u8 dont_translate = 0;
4414  u32 next0 = ~0;
4415  icmp_echo_header_t *echo0, *inner_echo0 = 0;
4416  ip4_header_t *inner_ip0;
4417  void *l4_header = 0;
4418  icmp46_header_t *inner_icmp0;
4419  snat_det_map_t * dm0 = 0;
4420  ip4_address_t new_addr0;
4421  u16 lo_port0, i0;
4422  snat_det_session_t * ses0 = 0;
4423  ip4_address_t in_addr;
4424  u16 in_port;
4425 
4426  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
4427  echo0 = (icmp_echo_header_t *)(icmp0+1);
4428  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4429  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
4430 
4431  if (!icmp_is_error_message (icmp0))
4432  {
4433  protocol = SNAT_PROTOCOL_ICMP;
4434  in_addr = ip0->src_address;
4435  in_port = echo0->identifier;
4436  }
4437  else
4438  {
4439  inner_ip0 = (ip4_header_t *)(echo0+1);
4440  l4_header = ip4_next_header (inner_ip0);
4441  protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
4442  in_addr = inner_ip0->dst_address;
4443  switch (protocol)
4444  {
4445  case SNAT_PROTOCOL_ICMP:
4446  inner_icmp0 = (icmp46_header_t*)l4_header;
4447  inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
4448  in_port = inner_echo0->identifier;
4449  break;
4450  case SNAT_PROTOCOL_UDP:
4451  case SNAT_PROTOCOL_TCP:
4452  in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
4453  break;
4454  default:
4455  b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
4456  next0 = SNAT_IN2OUT_NEXT_DROP;
4457  goto out;
4458  }
4459  }
4460 
4461  dm0 = snat_det_map_by_user(sm, &in_addr);
4462  if (PREDICT_FALSE(!dm0))
4463  {
4464  nat_log_info ("no match for internal host %U",
4465  format_ip4_address, &in_addr);
4466  if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4467  IP_PROTOCOL_ICMP, rx_fib_index0)))
4468  {
4469  dont_translate = 1;
4470  goto out;
4471  }
4472  next0 = SNAT_IN2OUT_NEXT_DROP;
4473  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4474  goto out;
4475  }
4476 
4477  snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
4478 
4479  key0.ext_host_addr = ip0->dst_address;
4480  key0.ext_host_port = 0;
4481 
4482  ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
4483  if (PREDICT_FALSE(!ses0))
4484  {
4485  if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4486  IP_PROTOCOL_ICMP, rx_fib_index0)))
4487  {
4488  dont_translate = 1;
4489  goto out;
4490  }
4491  if (icmp0->type != ICMP4_echo_request)
4492  {
4493  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4494  next0 = SNAT_IN2OUT_NEXT_DROP;
4495  goto out;
4496  }
4497  for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4498  {
4499  key0.out_port = clib_host_to_net_u16 (lo_port0 +
4500  ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
4501 
4502  if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
4503  continue;
4504 
4505  ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
4506  break;
4507  }
4508  if (PREDICT_FALSE(!ses0))
4509  {
4510  next0 = SNAT_IN2OUT_NEXT_DROP;
4511  b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
4512  goto out;
4513  }
4514  }
4515 
4516  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
4517  !icmp_is_error_message (icmp0)))
4518  {
4519  b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4520  next0 = SNAT_IN2OUT_NEXT_DROP;
4521  goto out;
4522  }
4523 
4524  u32 now = (u32) vlib_time_now (sm->vlib_main);
4525 
4526  ses0->state = SNAT_SESSION_ICMP_ACTIVE;
4527  ses0->expire = now + sm->icmp_timeout;
4528 
4529 out:
4530  *p_proto = protocol;
4531  if (ses0)
4532  {
4533  p_value->addr = new_addr0;
4534  p_value->fib_index = sm->outside_fib_index;
4535  p_value->port = ses0->out.out_port;
4536  }
4537  *p_dont_translate = dont_translate;
4538  if (d)
4539  *(snat_det_session_t**)d = ses0;
4540  if (e)
4541  *(snat_det_map_t**)e = dm0;
4542  return next0;
4543 }
4544 
4545 /**********************/
4546 /*** worker handoff ***/
4547 /**********************/
4548 static inline uword
4550  vlib_node_runtime_t * node,
4551  vlib_frame_t * frame,
4552  u8 is_output)
4553 {
4554  snat_main_t *sm = &snat_main;
4556  u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
4557  static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
4558  static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
4559  = 0;
4560  vlib_frame_queue_elt_t *hf = 0;
4561  vlib_frame_queue_t *fq;
4562  vlib_frame_t *f = 0;
4563  int i;
4564  u32 n_left_to_next_worker = 0, *to_next_worker = 0;
4565  u32 next_worker_index = 0;
4566  u32 current_worker_index = ~0;
4567  u32 thread_index = vm->thread_index;
4568  u32 fq_index;
4569  u32 to_node_index;
4570  vlib_frame_t *d = 0;
4571 
4572  ASSERT (vec_len (sm->workers));
4573 
4574  if (is_output)
4575  {
4576  fq_index = sm->fq_in2out_output_index;
4577  to_node_index = sm->in2out_output_node_index;
4578  }
4579  else
4580  {
4581  fq_index = sm->fq_in2out_index;
4582  to_node_index = sm->in2out_node_index;
4583  }
4584 
4585  if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
4586  {
4587  vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
4588 
4589  vec_validate_init_empty (congested_handoff_queue_by_worker_index,
4590  tm->n_vlib_mains - 1,
4591  (vlib_frame_queue_t *) (~0));
4592  }
4593 
4594  from = vlib_frame_vector_args (frame);
4595  n_left_from = frame->n_vectors;
4596 
4597  while (n_left_from > 0)
4598  {
4599  u32 bi0;
4600  vlib_buffer_t *b0;
4601  u32 sw_if_index0;
4602  u32 rx_fib_index0;
4603  ip4_header_t * ip0;
4604  u8 do_handoff;
4605 
4606  bi0 = from[0];
4607  from += 1;
4608  n_left_from -= 1;
4609 
4610  b0 = vlib_get_buffer (vm, bi0);
4611 
4612  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
4613  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4614 
4615  ip0 = vlib_buffer_get_current (b0);
4616 
4617  next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
4618 
4619  if (PREDICT_FALSE (next_worker_index != thread_index))
4620  {
4621  do_handoff = 1;
4622 
4623  if (next_worker_index != current_worker_index)
4624  {
4626  fq_index, next_worker_index, NAT_FQ_NELTS - 2,
4627  congested_handoff_queue_by_worker_index);
4628 
4629  if (fq)
4630  {
4631  /* if this is 1st frame */
4632  if (!d)
4633  {
4635  to_next_drop = vlib_frame_vector_args (d);
4636  }
4637 
4638  to_next_drop[0] = bi0;
4639  to_next_drop += 1;
4640  d->n_vectors++;
4641  b0->error = node->errors[SNAT_IN2OUT_ERROR_FQ_CONGESTED];
4642  goto trace0;
4643  }
4644 
4645  if (hf)
4646  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4647 
4648  hf = vlib_get_worker_handoff_queue_elt (fq_index,
4649  next_worker_index,
4650  handoff_queue_elt_by_worker_index);
4651 
4652  n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
4653  to_next_worker = &hf->buffer_index[hf->n_vectors];
4654  current_worker_index = next_worker_index;
4655  }
4656 
4657  /* enqueue to correct worker thread */
4658  to_next_worker[0] = bi0;
4659  to_next_worker++;
4660  n_left_to_next_worker--;
4661 
4662  if (n_left_to_next_worker == 0)
4663  {
4664  hf->n_vectors = VLIB_FRAME_SIZE;
4666  current_worker_index = ~0;
4667  handoff_queue_elt_by_worker_index[next_worker_index] = 0;
4668  hf = 0;
4669  }
4670  }
4671  else
4672  {
4673  do_handoff = 0;
4674  /* if this is 1st frame */
4675  if (!f)
4676  {
4677  f = vlib_get_frame_to_node (vm, to_node_index);
4678  to_next = vlib_frame_vector_args (f);
4679  }
4680 
4681  to_next[0] = bi0;
4682  to_next += 1;
4683  f->n_vectors++;
4684  }
4685 
4686 trace0:
4688  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4689  {
4691  vlib_add_trace (vm, node, b0, sizeof (*t));
4692  t->next_worker_index = next_worker_index;
4693  t->do_handoff = do_handoff;
4694  }
4695  }
4696 
4697  if (f)
4698  vlib_put_frame_to_node (vm, to_node_index, f);
4699 
4700  if (d)
4702 
4703  if (hf)
4704  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4705 
4706  /* Ship frames to the worker nodes */
4707  for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
4708  {
4709  if (handoff_queue_elt_by_worker_index[i])
4710  {
4711  hf = handoff_queue_elt_by_worker_index[i];
4712  /*
4713  * It works better to let the handoff node
4714  * rate-adapt, always ship the handoff queue element.
4715  */
4716  if (1 || hf->n_vectors == hf->last_n_vectors)
4717  {
4719  handoff_queue_elt_by_worker_index[i] = 0;
4720  }
4721  else
4722  hf->last_n_vectors = hf->n_vectors;
4723  }
4724  congested_handoff_queue_by_worker_index[i] =
4725  (vlib_frame_queue_t *) (~0);
4726  }
4727  hf = 0;
4728  current_worker_index = ~0;
4729  return frame->n_vectors;
4730 }
4731 
4732 static uword
4734  vlib_node_runtime_t * node,
4735  vlib_frame_t * frame)
4736 {
4737  return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
4738 }
4739 
4741  .function = snat_in2out_worker_handoff_fn,
4742  .name = "nat44-in2out-worker-handoff",
4743  .vector_size = sizeof (u32),
4745  .type = VLIB_NODE_TYPE_INTERNAL,
4746 
4747  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4748  .error_strings = snat_in2out_error_strings,
4749 
4750  .n_next_nodes = 1,
4751 
4752  .next_nodes = {
4753  [0] = "error-drop",
4754  },
4755 };
4756 
4759 
4760 static uword
4762  vlib_node_runtime_t * node,
4763  vlib_frame_t * frame)
4764 {
4765  return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
4766 }
4767 
4770  .name = "nat44-in2out-output-worker-handoff",
4771  .vector_size = sizeof (u32),
4773  .type = VLIB_NODE_TYPE_INTERNAL,
4774 
4775  .n_next_nodes = 1,
4776 
4777  .next_nodes = {
4778  [0] = "error-drop",
4779  },
4780 };
4781 
4784 
4787 {
4788  snat_address_t * ap;
4789  clib_bihash_kv_8_8_t kv, value;
4790  snat_session_key_t m_key;
4791 
4792  vec_foreach (ap, sm->addresses)
4793  {
4794  if (ap->addr.as_u32 == dst_addr->as_u32)
4795  return 1;
4796  }
4797 
4798  m_key.addr.as_u32 = dst_addr->as_u32;
4799  m_key.fib_index = 0;
4800  m_key.port = 0;
4801  m_key.protocol = 0;
4802  kv.key = m_key.as_u64;
4803  if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4804  return 1;
4805 
4806  return 0;
4807 }
4808 
4809 static inline uword
4811  vlib_node_runtime_t * node,
4812  vlib_frame_t * frame,
4813  int is_ed)
4814 {
4815  u32 n_left_from, * from, * to_next, stats_node_index;
4816  snat_in2out_next_t next_index;
4817  u32 pkts_processed = 0;
4818  snat_main_t * sm = &snat_main;
4819 
4820  stats_node_index = is_ed ? nat44_ed_hairpin_dst_node.index :
4821  snat_hairpin_dst_node.index;
4822 
4823  from = vlib_frame_vector_args (frame);
4824  n_left_from = frame->n_vectors;
4825  next_index = node->cached_next_index;
4826 
4827  while (n_left_from > 0)
4828  {
4829  u32 n_left_to_next;
4830 
4831  vlib_get_next_frame (vm, node, next_index,
4832  to_next, n_left_to_next);
4833 
4834  while (n_left_from > 0 && n_left_to_next > 0)
4835  {
4836  u32 bi0;
4837  vlib_buffer_t * b0;
4838  u32 next0;
4839  ip4_header_t * ip0;
4840  u32 proto0;
4841 
4842  /* speculatively enqueue b0 to the current next frame */
4843  bi0 = from[0];
4844  to_next[0] = bi0;
4845  from += 1;
4846  to_next += 1;
4847  n_left_from -= 1;
4848  n_left_to_next -= 1;
4849 
4850  b0 = vlib_get_buffer (vm, bi0);
4851  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4852  ip0 = vlib_buffer_get_current (b0);
4853 
4854  proto0 = ip_proto_to_snat_proto (ip0->protocol);
4855 
4856  vnet_buffer (b0)->snat.flags = 0;
4857  if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
4858  {
4859  if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
4860  {
4861  udp_header_t * udp0 = ip4_next_header (ip0);
4862  tcp_header_t * tcp0 = (tcp_header_t *) udp0;
4863 
4864  snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
4865  }
4866  else if (proto0 == SNAT_PROTOCOL_ICMP)
4867  {
4868  icmp46_header_t * icmp0 = ip4_next_header (ip0);
4869 
4870  snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
4871  }
4872  else
4873  {
4874  if (is_ed)
4875  nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
4876  else
4877  nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
4878  }
4879 
4880  vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
4881  }
4882 
4883  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4884 
4885  /* verify speculative enqueue, maybe switch current next frame */
4886  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4887  to_next, n_left_to_next,
4888  bi0, next0);
4889  }
4890 
4891  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4892  }
4893 
4894  vlib_node_increment_counter (vm, stats_node_index,
4895  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4896  pkts_processed);
4897  return frame->n_vectors;
4898 }
4899 
4900 static uword
4902  vlib_node_runtime_t * node,
4903  vlib_frame_t * frame)
4904 {
4905  return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
4906 }
4907 
4909  .function = snat_hairpin_dst_fn,
4910  .name = "nat44-hairpin-dst",
4911  .vector_size = sizeof (u32),
4912  .type = VLIB_NODE_TYPE_INTERNAL,
4913  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4914  .error_strings = snat_in2out_error_strings,
4915  .n_next_nodes = 2,
4916  .next_nodes = {
4917  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4918  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4919  },
4920 };
4921 
4924 
4925 static uword
4927  vlib_node_runtime_t * node,
4928  vlib_frame_t * frame)
4929 {
4930  return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
4931 }
4932 
4934  .function = nat44_ed_hairpin_dst_fn,
4935  .name = "nat44-ed-hairpin-dst",
4936  .vector_size = sizeof (u32),
4937  .type = VLIB_NODE_TYPE_INTERNAL,
4938  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4939  .error_strings = snat_in2out_error_strings,
4940  .n_next_nodes = 2,
4941  .next_nodes = {
4942  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4943  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4944  },
4945 };
4946 
4949 
4950 static inline uword
4952  vlib_node_runtime_t * node,
4953  vlib_frame_t * frame,
4954  int is_ed)
4955 {
4956  u32 n_left_from, * from, * to_next, stats_node_index;
4957  snat_in2out_next_t next_index;
4958  u32 pkts_processed = 0;
4959  snat_main_t *sm = &snat_main;
4960 
4961  stats_node_index = is_ed ? nat44_ed_hairpin_src_node.index :
4962  snat_hairpin_src_node.index;
4963 
4964  from = vlib_frame_vector_args (frame);
4965  n_left_from = frame->n_vectors;
4966  next_index = node->cached_next_index;
4967 
4968  while (n_left_from > 0)
4969  {
4970  u32 n_left_to_next;
4971 
4972  vlib_get_next_frame (vm, node, next_index,
4973  to_next, n_left_to_next);
4974 
4975  while (n_left_from > 0 && n_left_to_next > 0)
4976  {
4977  u32 bi0;
4978  vlib_buffer_t * b0;
4979  u32 next0;
4981  u32 sw_if_index0;
4982 
4983  /* speculatively enqueue b0 to the current next frame */
4984  bi0 = from[0];
4985  to_next[0] = bi0;
4986  from += 1;
4987  to_next += 1;
4988  n_left_from -= 1;
4989  n_left_to_next -= 1;
4990 
4991  b0 = vlib_get_buffer (vm, bi0);
4992  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4994 
4996  ({
4997  /* Only packets from NAT inside interface */
4998  if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
4999  {
5000  if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
5001  SNAT_FLAG_HAIRPINNING))
5002  {
5003  if (PREDICT_TRUE (sm->num_workers > 1))
5004  next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
5005  else
5006  next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
5007  }
5008  break;
5009  }
5010  }));
5011 
5012  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5013 
5014  /* verify speculative enqueue, maybe switch current next frame */
5015  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5016  to_next, n_left_to_next,
5017  bi0, next0);
5018  }
5019 
5020  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5021  }
5022 
5023  vlib_node_increment_counter (vm, stats_node_index,
5024  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5025  pkts_processed);
5026  return frame->n_vectors;
5027 }
5028 
5029 static uword
5031  vlib_node_runtime_t * node,
5032  vlib_frame_t * frame)
5033 {
5034  return snat_hairpin_src_fn_inline (vm, node, frame, 0);
5035 }
5036 
5038  .function = snat_hairpin_src_fn,
5039  .name = "nat44-hairpin-src",
5040  .vector_size = sizeof (u32),
5041  .type = VLIB_NODE_TYPE_INTERNAL,
5042  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5043  .error_strings = snat_in2out_error_strings,
5044  .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5045  .next_nodes = {
5046  [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5047  [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
5048  [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5049  [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5050  },
5051 };
5052 
5055 
5056 static uword
5058  vlib_node_runtime_t * node,
5059  vlib_frame_t * frame)
5060 {
5061  return snat_hairpin_src_fn_inline (vm, node, frame, 1);
5062 }
5063 
5065  .function = nat44_ed_hairpin_src_fn,
5066  .name = "nat44-ed-hairpin-src",
5067  .vector_size = sizeof (u32),
5068  .type = VLIB_NODE_TYPE_INTERNAL,
5069  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5070  .error_strings = snat_in2out_error_strings,
5071  .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5072  .next_nodes = {
5073  [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5074  [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
5075  [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5076  [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5077  },
5078 };
5079 
5082 
5083 static uword
5085  vlib_node_runtime_t * node,
5086  vlib_frame_t * frame)
5087 {
5088  u32 n_left_from, * from, * to_next;
5089  snat_in2out_next_t next_index;
5090  u32 pkts_processed = 0;
5091  snat_main_t * sm = &snat_main;
5092  u32 stats_node_index;
5093 
5094  stats_node_index = snat_in2out_fast_node.index;
5095 
5096  from = vlib_frame_vector_args (frame);
5097  n_left_from = frame->n_vectors;
5098  next_index = node->cached_next_index;
5099 
5100  while (n_left_from > 0)
5101  {
5102  u32 n_left_to_next;
5103 
5104  vlib_get_next_frame (vm, node, next_index,
5105  to_next, n_left_to_next);
5106 
5107  while (n_left_from > 0 && n_left_to_next > 0)
5108  {
5109  u32 bi0;
5110  vlib_buffer_t * b0;
5111  u32 next0;
5112  u32 sw_if_index0;
5113  ip4_header_t * ip0;
5114  ip_csum_t sum0;
5115  u32 new_addr0, old_addr0;
5116  u16 old_port0, new_port0;
5117  udp_header_t * udp0;
5118  tcp_header_t * tcp0;
5119  icmp46_header_t * icmp0;
5120  snat_session_key_t key0, sm0;
5121  u32 proto0;
5122  u32 rx_fib_index0;
5123 
5124  /* speculatively enqueue b0 to the current next frame */
5125  bi0 = from[0];
5126  to_next[0] = bi0;
5127  from += 1;
5128  to_next += 1;
5129  n_left_from -= 1;
5130  n_left_to_next -= 1;
5131 
5132  b0 = vlib_get_buffer (vm, bi0);
5133  next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5134 
5135  ip0 = vlib_buffer_get_current (b0);
5136  udp0 = ip4_next_header (ip0);
5137  tcp0 = (tcp_header_t *) udp0;
5138  icmp0 = (icmp46_header_t *) udp0;
5139 
5140  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5141  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
5142 
5143  if (PREDICT_FALSE(ip0->ttl == 1))
5144  {
5145  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
5146  icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
5147  ICMP4_time_exceeded_ttl_exceeded_in_transit,
5148  0);
5150  goto trace0;
5151  }
5152 
5153  proto0 = ip_proto_to_snat_proto (ip0->protocol);
5154 
5155  if (PREDICT_FALSE (proto0 == ~0))
5156  goto trace0;
5157 
5158  if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
5159  {
5160  next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
5161  rx_fib_index0, node, next0, ~0, 0, 0);
5162  goto trace0;
5163  }
5164 
5165  key0.addr = ip0->src_address;
5166  key0.protocol = proto0;
5167  key0.port = udp0->src_port;
5168  key0.fib_index = rx_fib_index0;
5169 
5170  if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
5171  {
5172  b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
5173  next0= SNAT_IN2OUT_NEXT_DROP;
5174  goto trace0;
5175  }
5176 
5177  new_addr0 = sm0.addr.as_u32;
5178  new_port0 = sm0.port;
5179  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
5180  old_addr0 = ip0->src_address.as_u32;
5181  ip0->src_address.as_u32 = new_addr0;
5182 
5183  sum0 = ip0->checksum;
5184  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5185  ip4_header_t,
5186  src_address /* changed member */);
5187  ip0->checksum = ip_csum_fold (sum0);
5188 
5189  if (PREDICT_FALSE(new_port0 != udp0->dst_port))
5190  {
5191  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5192  {
5193  old_port0 = tcp0->src_port;
5194  tcp0->src_port = new_port0;
5195 
5196  sum0 = tcp0->checksum;
5197  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5198  ip4_header_t,
5199  dst_address /* changed member */);
5200  sum0 = ip_csum_update (sum0, old_port0, new_port0,
5201  ip4_header_t /* cheat */,
5202  length /* changed member */);
5203  tcp0->checksum = ip_csum_fold(sum0);
5204  }
5205  else
5206  {
5207  old_port0 = udp0->src_port;
5208  udp0->src_port = new_port0;
5209  udp0->checksum = 0;
5210  }
5211  }
5212  else
5213  {
5214  if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5215  {
5216  sum0 = tcp0->checksum;
5217  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5218  ip4_header_t,
5219  dst_address /* changed member */);
5220  tcp0->checksum = ip_csum_fold(sum0);
5221  }
5222  }
5223 
5224  /* Hairpinning */
5225  snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
5226 
5227  trace0:
5229  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
5230  {
5231  snat_in2out_trace_t *t =
5232  vlib_add_trace (vm, node, b0, sizeof (*t));
5233  t->sw_if_index = sw_if_index0;
5234  t->next_index = next0;
5235  }
5236 
5237  pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5238 
5239  /* verify speculative enqueue, maybe switch current next frame */
5240  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5241  to_next, n_left_to_next,
5242  bi0, next0);
5243  }
5244 
5245  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5246  }
5247 
5248  vlib_node_increment_counter (vm, stats_node_index,
5249  SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5250  pkts_processed);
5251  return frame->n_vectors;
5252 }
5253 
5254 
5256  .function = snat_in2out_fast_static_map_fn,
5257  .name = "nat44-in2out-fast",
5258  .vector_size = sizeof (u32),
5259  .format_trace = format_snat_in2out_fast_trace,
5260  .type = VLIB_NODE_TYPE_INTERNAL,
5261 
5262  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5263  .error_strings = snat_in2out_error_strings,
5264 
5265  .runtime_data_bytes = sizeof (snat_runtime_t),
5266 
5267  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
5268 
5269  /* edit / add dispositions here */
5270  .next_nodes = {
5271  [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5272  [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5273  [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
5274  [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
5275  [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
5276  },
5277 };
5278 
ip4_address_t external_addr
Definition: nat.h:244
void nat_ipfix_logging_max_sessions(u32 limit)
Generate maximum session entries exceeded event.
vnet_config_main_t config_main
Definition: feature.h:65
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:437
u32 next
Definition: dlist.h:30
fib_protocol_t fp_proto
protocol type
Definition: fib_types.h:202
nat_outside_fib_t * outside_fibs
Definition: nat.h:359
#define nat_log_info(...)
Definition: nat.h:531
static uword nat44_ed_hairpinning_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:1940
u32 sessions_per_user_list_head_index
Definition: nat.h:189
static u8 * format_snat_in2out_trace(u8 *s, va_list *args)
Definition: in2out.c:47
static u32 slow_path_ed(snat_main_t *sm, vlib_buffer_t *b, u32 rx_fib_index, clib_bihash_kv_16_8_t *kv, snat_session_t **sessionp, vlib_node_runtime_t *node, u32 next, u32 thread_index)
Definition: in2out.c:2377
#define CLIB_UNUSED(x)
Definition: clib.h:79
snat_in2out_error_t
Definition: in2out.c:140
u16 ext_host_port
Definition: nat.h:82
static uword snat_in2out_slow_path_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:1768
u16 out_port
Definition: nat.h:83
u32 icmp_timeout
Definition: nat.h:412
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:132
ip4_address_t src_address
Definition: ip4_packet.h:169
static int nat_not_translate_output_feature(snat_main_t *sm, ip4_header_t *ip0, u32 proto0, u16 src_port, u16 dst_port, u32 thread_index, u32 sw_if_index)
Definition: in2out.c:275
u32 fq_in2out_output_index
Definition: nat.h:376
static u8 * format_snat_in2out_fast_trace(u8 *s, va_list *args)
Definition: in2out.c:62
#define TCP_FLAG_SYN
Definition: fa_node.h:13
static uword snat_in2out_node_fn_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_slow_path, int is_output_feature)
Definition: in2out.c:1092
vlib_node_registration_t nat44_ed_in2out_node
(constructor) VLIB_REGISTER_NODE (nat44_ed_in2out_node)
Definition: in2out.c:119
#define PREDICT_TRUE(x)
Definition: clib.h:106
static_always_inline u8 icmp_is_error_message(icmp46_header_t *icmp)
Definition: nat_inlines.h:52
static int ip4_header_bytes(ip4_header_t *i)
Definition: ip4_packet.h:234
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:225
static char * snat_in2out_error_strings[]
Definition: in2out.c:147
u32 fib_table_get_index_for_sw_if_index(fib_protocol_t proto, u32 sw_if_index)
Get the index of the FIB bound to the interface.
Definition: fib_table.c:948
static void make_sm_kv(clib_bihash_kv_8_8_t *kv, ip4_address_t *addr, u8 proto, u32 fib_index, u16 port)
Definition: nat_inlines.h:260
u16 port_per_thread
Definition: nat.h:332
static uword snat_hairpin_dst_fn_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_ed)
Definition: in2out.c:4810
u32 thread_index
Definition: main.h:176
static uword snat_in2out_worker_handoff_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:4733
#define nat_log_warn(...)
Definition: nat.h:527
static void snat_det_ses_close(snat_det_map_t *dm, snat_det_session_t *ses)
Definition: nat_det.h:180
int i
static snat_det_session_t * snat_det_find_ses_by_in(snat_det_map_t *dm, ip4_address_t *in_addr, u16 in_port, snat_det_out_key_t out_key)
Definition: nat_det.h:129
static void snat_det_forward(snat_det_map_t *dm, ip4_address_t *in_addr, ip4_address_t *out_addr, u16 *lo_port)
Definition: nat_det.h:75
uword ip_csum_t
Definition: ip_packet.h:181
vlib_node_registration_t nat44_ed_in2out_output_node
(constructor) VLIB_REGISTER_NODE (nat44_ed_in2out_output_node)
Definition: in2out.c:121
snat_in2out_next_t
Definition: in2out.c:153
u32 * fib_index_by_sw_if_index
Table index indexed by software interface.
Definition: ip4.h:111
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
static uword nat44_ed_hairpin_src_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: in2out.c:5057
u32 buffer_index[VLIB_FRAME_SIZE]
Definition: threads.h:98
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:451
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:250
dlist_elt_t * list_pool
Definition: nat.h:295
struct _tcp_header tcp_header_t