FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
lock.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef included_clib_lock_h
17 #define included_clib_lock_h
18 
19 #include <vppinfra/clib.h>
20 #include <vppinfra/atomics.h>
21 
22 #if __x86_64__
23 #define CLIB_PAUSE() __builtin_ia32_pause ()
24 #elif defined (__aarch64__) || defined (__arm__)
25 #define CLIB_PAUSE() __asm__ ("yield")
26 #else
27 #define CLIB_PAUSE()
28 #endif
29 
30 #if CLIB_DEBUG > 1
31 #define CLIB_LOCK_DBG(_p) \
32 do { \
33  (*_p)->frame_address = __builtin_frame_address (0); \
34  (*_p)->pid = getpid (); \
35  (*_p)->thread_index = os_get_thread_index (); \
36 } while (0)
37 #define CLIB_LOCK_DBG_CLEAR(_p) \
38 do { \
39  (*_p)->frame_address = 0; \
40  (*_p)->pid = 0; \
41  (*_p)->thread_index = 0; \
42 } while (0)
43 #else
44 #define CLIB_LOCK_DBG(_p)
45 #define CLIB_LOCK_DBG_CLEAR(_p)
46 #endif
47 
48 #define CLIB_SPINLOCK_IS_LOCKED(_p) (*(_p))->lock
49 #define CLIB_SPINLOCK_ASSERT_LOCKED(_p) ASSERT(CLIB_SPINLOCK_IS_LOCKED((_p)))
50 
51 typedef struct
52 {
53  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
55 #if CLIB_DEBUG > 0
56  pid_t pid;
59 #endif
61 
62 static inline void
64 {
66  clib_memset ((void *) *p, 0, CLIB_CACHE_LINE_BYTES);
67 }
68 
69 static inline void
71 {
72  if (*p)
73  {
74  clib_mem_free ((void *) *p);
75  *p = 0;
76  }
77 }
78 
81 {
82  u32 free = 0;
83  while (!clib_atomic_cmp_and_swap_acq_relax_n (&(*p)->lock, &free, 1, 0))
84  {
85  /* atomic load limits number of compare_exchange executions */
86  while (clib_atomic_load_relax_n (&(*p)->lock))
87  CLIB_PAUSE ();
88  /* on failure, compare_exchange writes (*p)->lock into free */
89  free = 0;
90  }
91  CLIB_LOCK_DBG (p);
92 }
93 
96 {
98  return 0;
100  return 1;
101 }
102 
105 {
106  if (PREDICT_FALSE (*p != 0))
107  clib_spinlock_lock (p);
108 }
109 
112 {
113  if (PREDICT_FALSE (*p != 0))
114  return clib_spinlock_trylock (p);
115  return 1;
116 }
117 
120 {
122  /* Make sure all reads/writes are complete before releasing the lock */
123  clib_atomic_release (&(*p)->lock);
124 }
125 
128 {
129  if (PREDICT_FALSE (*p != 0))
131 }
132 
133 /*
134  * Readers-Writer Lock
135  */
136 
137 typedef struct clib_rw_lock_
138 {
139  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
140  /* -1 when W lock held, > 0 when R lock held */
141  volatile i32 rw_cnt;
142 #if CLIB_DEBUG > 0
143  pid_t pid;
146 #endif
147 } *clib_rwlock_t;
148 
149 always_inline void
151 {
153  clib_memset ((void *) *p, 0, CLIB_CACHE_LINE_BYTES);
154 }
155 
156 always_inline void
158 {
159  if (*p)
160  {
161  clib_mem_free ((void *) *p);
162  *p = 0;
163  }
164 }
165 
166 always_inline void
168 {
169  i32 cnt;
170  do
171  {
172  /* rwlock held by a writer */
173  while ((cnt = clib_atomic_load_relax_n (&(*p)->rw_cnt)) < 0)
174  CLIB_PAUSE ();
175  }
177  (&(*p)->rw_cnt, &cnt, cnt + 1, 1));
178  CLIB_LOCK_DBG (p);
179 }
180 
181 always_inline void
183 {
184  ASSERT ((*p)->rw_cnt > 0);
186  clib_atomic_fetch_sub_rel (&(*p)->rw_cnt, 1);
187 }
188 
189 always_inline void
191 {
192  i32 cnt = 0;
193  do
194  {
195  /* rwlock held by writer or reader(s) */
196  while ((cnt = clib_atomic_load_relax_n (&(*p)->rw_cnt)) != 0)
197  CLIB_PAUSE ();
198  }
199  while (!clib_atomic_cmp_and_swap_acq_relax_n (&(*p)->rw_cnt, &cnt, -1, 1));
200  CLIB_LOCK_DBG (p);
201 }
202 
203 always_inline void
205 {
207  clib_atomic_release (&(*p)->rw_cnt);
208 }
209 
210 #endif
211 
212 /*
213  * fd.io coding-style-patch-verification: ON
214  *
215  * Local Variables:
216  * eval: (c-set-style "gnu")
217  * End:
218  */
static void clib_rwlock_reader_lock(clib_rwlock_t *p)
Definition: lock.h:167
uword thread_index
Definition: lock.h:57
#define CLIB_PAUSE()
Definition: lock.h:23
static_always_inline int clib_spinlock_trylock_if_init(clib_spinlock_t *p)
Definition: lock.h:111
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:119
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:80
static void clib_rwlock_writer_lock(clib_rwlock_t *p)
Definition: lock.h:190
CLIB_CACHE_LINE_ALIGN_MARK(cacheline0)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static void clib_rwlock_free(clib_rwlock_t *p)
Definition: lock.h:157
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:127
pid_t pid
Definition: lock.h:143
#define CLIB_SPINLOCK_IS_LOCKED(_p)
Definition: lock.h:48
void * frame_address
Definition: lock.h:145
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:70
#define CLIB_LOCK_DBG_CLEAR(_p)
Definition: lock.h:45
#define static_always_inline
Definition: clib.h:108
unsigned int u32
Definition: types.h:88
uword thread_index
Definition: lock.h:144
u32 lock
Definition: lock.h:54
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:63
static void clib_rwlock_init(clib_rwlock_t *p)
Definition: lock.h:150
#define clib_atomic_release(a)
Definition: atomics.h:43
static void clib_rwlock_reader_unlock(clib_rwlock_t *p)
Definition: lock.h:182
#define PREDICT_FALSE(x)
Definition: clib.h:120
#define always_inline
Definition: ipsec.h:28
static void clib_rwlock_writer_unlock(clib_rwlock_t *p)
Definition: lock.h:204
pid_t pid
Definition: lock.h:56
struct clib_rw_lock_ * clib_rwlock_t
#define clib_atomic_load_relax_n(a)
Definition: atomics.h:47
signed int i32
Definition: types.h:77
#define ASSERT(truth)
static void clib_mem_free(void *p)
Definition: mem.h:215
static_always_inline int clib_spinlock_trylock(clib_spinlock_t *p)
Definition: lock.h:95
volatile i32 rw_cnt
Definition: lock.h:141
#define clib_atomic_fetch_sub_rel(a, b)
Definition: atomics.h:55
u64 uword
Definition: types.h:112
#define clib_atomic_cmp_and_swap_acq_relax_n(addr, exp, new, weak)
Definition: atomics.h:40
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:165
void * frame_address
Definition: lock.h:58
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:104
#define CLIB_LOCK_DBG(_p)
Definition: lock.h:44