44 #ifndef included_pool_h 45 #define included_pool_h 71 #define pool_aligned_header_bytes \ 72 vec_aligned_header_bytes (sizeof (pool_header_t), sizeof (void *)) 81 extern void _pool_init_fixed (
void **,
u32,
u32);
85 #define pool_init_fixed(pool,max_elts) \ 87 _pool_init_fixed((void **)&(pool),sizeof(pool[0]),max_elts); \ 115 #define pool_validate_index(v,i) \ 117 uword __pool_validate_index = (i); \ 118 vec_validate_ha ((v), __pool_validate_index, \ 119 pool_aligned_header_bytes, 0); \ 120 pool_header_validate_index ((v), __pool_validate_index); \ 139 #define pool_len(p) vec_len(p) 145 #define _pool_len(p) _vec_len(p) 160 #define pool_bytes(P) (vec_bytes (P) + pool_header_bytes (P)) 163 #define _pool_var(v) _pool_##v 191 #define _pool_get_aligned_internal_numa(P,E,A,Z,N) \ 193 pool_header_t * _pool_var (p) = pool_header (P); \ 194 uword _pool_var (l); \ 196 STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) \ 197 || ((sizeof(P[0]) % A) == 0), \ 198 "Pool aligned alloc of incorrectly sized object"); \ 201 _pool_var (l) = vec_len (_pool_var (p)->free_indices); \ 203 if (_pool_var (l) > 0) \ 206 uword _pool_var (i) = \ 207 _pool_var (p)->free_indices[_pool_var (l) - 1]; \ 208 (E) = (P) + _pool_var (i); \ 209 _pool_var (p)->free_bitmap = \ 210 clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \ 212 _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \ 213 CLIB_MEM_UNPOISON((E), sizeof((E)[0])); \ 218 if ((P) && _pool_var(p)->max_elts) \ 220 clib_warning ("can't expand fixed-size pool"); \ 221 os_out_of_memory(); \ 224 P = _vec_resize_numa (P, \ 226 (vec_len (P) + 1) * sizeof (P[0]), \ 227 pool_aligned_header_bytes, \ 230 E = vec_end (P) - 1; \ 233 memset(E, 0, sizeof(*E)); \ 236 #define pool_get_aligned_zero_numa(P,E,A,Z,S) \ 237 _pool_get_aligned_internal_numa(P,E,A,Z,S) 239 #define pool_get_aligned_numa(P,E,A,S) \ 240 _pool_get_aligned_internal_numa(P,E,A,0,S) 242 #define pool_get_numa(P,E,S) \ 243 _pool_get_aligned_internal_numa(P,E,0,0,S) 245 #define _pool_get_aligned_internal(P,E,A,Z) \ 246 _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED) 249 #define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0) 252 #define pool_get_aligned_zero(P,E,A) _pool_get_aligned_internal(P,E,A,1) 255 #define pool_get(P,E) pool_get_aligned(P,E,0) 258 #define pool_get_zero(P,E) pool_get_aligned_zero(P,E,0) 261 #define pool_get_aligned_will_expand(P,YESNO,A) \ 263 pool_header_t * _pool_var (p) = pool_header (P); \ 264 uword _pool_var (l); \ 269 if (_pool_var (p)->max_elts) \ 270 _pool_var (l) = _pool_var (p)->max_elts; \ 272 _pool_var (l) = vec_len (_pool_var (p)->free_indices); \ 276 if (_pool_var (l) > 0) \ 281 YESNO = _vec_resize_will_expand \ 284 (vec_len (P) + 1) * sizeof (P[0]), \ 285 pool_aligned_header_bytes, \ 291 #define pool_get_will_expand(P,YESNO) pool_get_aligned_will_expand(P,YESNO,0) 294 #define pool_is_free(P,E) \ 296 pool_header_t * _pool_var (p) = pool_header (P); \ 297 uword _pool_var (i) = (E) - (P); \ 298 (_pool_var (i) < vec_len (P)) ? clib_bitmap_get (_pool_var (p)->free_bitmap, _pool_i) : 1; \ 302 #define pool_is_free_index(P,I) pool_is_free((P),(P)+(I)) 305 #define pool_put(P, E) \ 308 typeof (P) _pool_var (p__) = (P); \ 309 typeof (E) _pool_var (e__) = (E); \ 310 pool_header_t *_pool_var (p) = pool_header (_pool_var (p__)); \ 311 uword _pool_var (l) = _pool_var (e__) - _pool_var (p__); \ 312 if (_pool_var (p)->max_elts == 0) \ 313 ASSERT (vec_is_member (_pool_var (p__), _pool_var (e__))); \ 314 ASSERT (!pool_is_free (_pool_var (p__), _pool_var (e__))); \ 317 _pool_var (p)->free_bitmap = \ 318 clib_bitmap_ori_notrim (_pool_var (p)->free_bitmap, _pool_var (l)); \ 321 if (_pool_var (p)->max_elts) \ 323 ASSERT (_pool_var (l) < _pool_var (p)->max_elts); \ 325 ->free_indices[_vec_len (_pool_var (p)->free_indices)] = \ 327 _vec_len (_pool_var (p)->free_indices) += 1; \ 330 vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \ 332 CLIB_MEM_POISON (_pool_var (e__), sizeof (_pool_var (e__)[0])); \ 337 #define pool_put_index(p,i) \ 339 typeof (p) _e = (p) + (i); \ 344 #define pool_alloc_aligned(P,N,A) \ 346 pool_header_t * _p; \ 350 _p = pool_header (P); \ 353 clib_warning ("Can't expand fixed-size pool"); \ 354 os_out_of_memory(); \ 358 (P) = _vec_resize ((P), 0, (vec_len (P) + (N)) * sizeof (P[0]), \ 359 pool_aligned_header_bytes, \ 361 _p = pool_header (P); \ 362 vec_resize (_p->free_indices, (N)); \ 363 _vec_len (_p->free_indices) -= (N); \ 367 #define pool_alloc(P,N) pool_alloc_aligned(P,N,0) 376 #define pool_dup_aligned(P,A) \ 378 typeof (P) _pool_var (new) = 0; \ 379 pool_header_t * _pool_var (ph), * _pool_var (new_ph); \ 380 u32 _pool_var (n) = pool_len (P); \ 383 _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n), \ 384 _pool_var (n) * sizeof ((P)[0]), \ 385 pool_aligned_header_bytes, (A)); \ 386 clib_memcpy_fast (_pool_var (new), (P), \ 387 _pool_var (n) * sizeof ((P)[0])); \ 388 _pool_var (ph) = pool_header (P); \ 389 _pool_var (new_ph) = pool_header (_pool_var (new)); \ 390 _pool_var (new_ph)->free_bitmap = \ 391 clib_bitmap_dup (_pool_var (ph)->free_bitmap); \ 392 _pool_var (new_ph)->free_indices = \ 393 vec_dup (_pool_var (ph)->free_indices); \ 394 _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts; \ 405 #define pool_dup(P) pool_dup_aligned(P,0) 443 return clib_bitmap_next_clear (h->
free_bitmap, last + 1);
447 #define pool_free(p) (p) = _pool_free(p) 459 #define pool_foreach_region(LO,HI,POOL,BODY) \ 461 uword _pool_var (i), _pool_var (lo), _pool_var (hi), _pool_var (len); \ 462 uword _pool_var (bl), * _pool_var (b); \ 463 pool_header_t * _pool_var (p); \ 465 _pool_var (p) = pool_header (POOL); \ 466 _pool_var (b) = (POOL) ? _pool_var (p)->free_bitmap : 0; \ 467 _pool_var (bl) = vec_len (_pool_var (b)); \ 468 _pool_var (len) = vec_len (POOL); \ 469 _pool_var (lo) = 0; \ 471 for (_pool_var (i) = 0; \ 472 _pool_var (i) <= _pool_var (bl); \ 475 uword _pool_var (m), _pool_var (f); \ 476 _pool_var (m) = (_pool_var (i) < _pool_var (bl) \ 477 ? _pool_var (b) [_pool_var (i)] \ 479 while (_pool_var (m) != 0) \ 481 _pool_var (f) = first_set (_pool_var (m)); \ 482 _pool_var (hi) = (_pool_var (i) * BITS (_pool_var (b)[0]) \ 483 + min_log2 (_pool_var (f))); \ 484 _pool_var (hi) = (_pool_var (i) < _pool_var (bl) \ 485 ? _pool_var (hi) : _pool_var (len)); \ 486 _pool_var (m) ^= _pool_var (f); \ 487 if (_pool_var (hi) > _pool_var (lo)) \ 489 (LO) = _pool_var (lo); \ 490 (HI) = _pool_var (hi); \ 491 do { BODY; } while (0); \ 493 _pool_var (lo) = _pool_var (hi) + 1; \ 534 #define pool_foreach(VAR,POOL) \ 536 for (VAR = POOL + pool_get_first_index (POOL); \ 537 VAR < vec_end (POOL); \ 538 VAR = POOL + pool_get_next_index (POOL, VAR - POOL)) 540 #define pool_foreach_old(VAR,POOL,BODY) \ 541 pool_foreach(VAR,POOL) \ 553 #define pool_elt_at_index(p,i) \ 555 typeof (p) _e = (p) + (i); \ 556 ASSERT (! pool_is_free (p, _e)); \ 561 #define pool_next_index(P,I) \ 563 pool_header_t * _pool_var (p) = pool_header (P); \ 564 uword _pool_var (rv) = (I) + 1; \ 567 (_pool_var (rv) < vec_len (P) ? \ 568 clib_bitmap_next_clear (_pool_var (p)->free_bitmap, _pool_var(rv)) \ 571 (_pool_var (rv) < vec_len (P) ? \ 572 _pool_var (rv) : ~0); \ 576 #define pool_foreach_index(i,v) \ 578 for (i = pool_get_first_index (v); \ 580 i = pool_get_next_index (v, i)) \ 583 #define pool_foreach_index_old(i,v,body) \ 584 pool_foreach_index (i,v) \ 595 #define pool_flush(VAR, POOL, BODY) \ 597 uword *_pool_var(ii), *_pool_var(dv) = NULL; \ 599 pool_foreach((VAR), (POOL)) \ 601 vec_add1(_pool_var(dv), (VAR) - (POOL)); \ 603 vec_foreach(_pool_var(ii), _pool_var(dv)) \ 605 (VAR) = pool_elt_at_index((POOL), *_pool_var(ii)); \ 606 do { BODY; } while (0); \ 607 pool_put((POOL), (VAR)); \ 609 vec_free(_pool_var(dv)); \ #define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
static_always_inline uword pool_get_first_index(void *pool)
static void pool_header_validate_index(void *v, uword index)
static heap_elt_t * last(heap_header_t *h)
#define vec_bytes(v)
Number of data bytes in vector.
#define static_always_inline
static pool_header_t * pool_header(void *v)
Get pool header from user pool pointer.
static_always_inline uword pool_get_next_index(void *pool, uword last)
#define pool_aligned_header_bytes
Align pool header so that pointers are naturally aligned.
static uword pool_header_bytes(void *v)
Memory usage of pool header.
sll srl srl sll sra u16x4 i
static void pool_validate(void *v)
Validate a pool.
#define vec_free(V)
Free vector's memory (no header).
static void * vec_aligned_header(void *v, uword header_bytes, uword align)
#define vec_capacity(v, b)
Total number of bytes that can fit in vector with current allocation.
static uword clib_bitmap_get(uword *ai, uword i)
Gets the ith bit value from a bitmap.
Bitmaps built as vectors of machine words.
#define clib_bitmap_free(v)
Free a bitmap.
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
#define vec_free_h(V, H)
Free vector's memory (general version)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define clib_unix_warning(format, args...)
static uword clib_bitmap_first_clear(uword *ai)
Return the lowest numbered clear bit in a bitmap.
static uword pool_free_elts(void *v)
Queries whether pool has at least N_FREE free elements.
static uword pool_elts(void *v)
Number of active elements in a pool.