Auto-generate files after cl/590419408
diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c
index 31487a5..faf65cb 100644
--- a/php/ext/google/protobuf/php-upb.c
+++ b/php/ext/google/protobuf/php-upb.c
@@ -5521,6 +5521,9 @@
upb_alloc upb_alloc_global = {&upb_global_allocfunc};
+#include <stddef.h>
+#include <stdint.h>
+
// Must be last.
@@ -5531,15 +5534,59 @@
// Data follows.
};
-static const size_t memblock_reserve =
+static const size_t kUpb_MemblockReserve =
UPB_ALIGN_UP(sizeof(_upb_MemBlock), UPB_MALLOC_ALIGN);
-typedef struct _upb_ArenaRoot {
+typedef struct {
upb_Arena* root;
uintptr_t tagged_count;
-} _upb_ArenaRoot;
+} upb_ArenaRoot;
-static _upb_ArenaRoot _upb_Arena_FindRoot(upb_Arena* a) {
+static bool _upb_Arena_IsTaggedRefcount(uintptr_t parent_or_count) {
+ return (parent_or_count & 1) == 1;
+}
+
+static bool _upb_Arena_IsTaggedPointer(uintptr_t parent_or_count) {
+ return (parent_or_count & 1) == 0;
+}
+
+static uintptr_t _upb_Arena_RefCountFromTagged(uintptr_t parent_or_count) {
+ UPB_ASSERT(_upb_Arena_IsTaggedRefcount(parent_or_count));
+ return parent_or_count >> 1;
+}
+
+static uintptr_t _upb_Arena_TaggedFromRefcount(uintptr_t refcount) {
+ uintptr_t parent_or_count = (refcount << 1) | 1;
+ UPB_ASSERT(_upb_Arena_IsTaggedRefcount(parent_or_count));
+ return parent_or_count;
+}
+
+static upb_Arena* _upb_Arena_PointerFromTagged(uintptr_t parent_or_count) {
+ UPB_ASSERT(_upb_Arena_IsTaggedPointer(parent_or_count));
+ return (upb_Arena*)parent_or_count;
+}
+
+static uintptr_t _upb_Arena_TaggedFromPointer(upb_Arena* a) {
+ uintptr_t parent_or_count = (uintptr_t)a;
+ UPB_ASSERT(_upb_Arena_IsTaggedPointer(parent_or_count));
+ return parent_or_count;
+}
+
+static upb_alloc* _upb_Arena_BlockAlloc(upb_Arena* arena) {
+ return (upb_alloc*)(arena->block_alloc & ~0x1);
+}
+
+static uintptr_t _upb_Arena_MakeBlockAlloc(upb_alloc* alloc, bool has_initial) {
+ uintptr_t alloc_uint = (uintptr_t)alloc;
+ UPB_ASSERT((alloc_uint & 1) == 0);
+ return alloc_uint | (has_initial ? 1 : 0);
+}
+
+static bool _upb_Arena_HasInitialBlock(upb_Arena* arena) {
+ return arena->block_alloc & 0x1;
+}
+
+static upb_ArenaRoot _upb_Arena_FindRoot(upb_Arena* a) {
uintptr_t poc = upb_Atomic_Load(&a->parent_or_count, memory_order_acquire);
while (_upb_Arena_IsTaggedPointer(poc)) {
upb_Arena* next = _upb_Arena_PointerFromTagged(poc);
@@ -5573,7 +5620,7 @@
a = next;
poc = next_poc;
}
- return (_upb_ArenaRoot){.root = a, .tagged_count = poc};
+ return (upb_ArenaRoot){.root = a, .tagged_count = poc};
}
size_t upb_Arena_SpaceAllocated(upb_Arena* arena) {
@@ -5604,7 +5651,7 @@
return _upb_Arena_RefCountFromTagged(poc);
}
-static void upb_Arena_AddBlock(upb_Arena* a, void* ptr, size_t size) {
+static void _upb_Arena_AddBlock(upb_Arena* a, void* ptr, size_t size) {
_upb_MemBlock* block = ptr;
// Insert into linked list.
@@ -5612,37 +5659,36 @@
upb_Atomic_Init(&block->next, a->blocks);
upb_Atomic_Store(&a->blocks, block, memory_order_release);
- a->head.ptr = UPB_PTR_AT(block, memblock_reserve, char);
- a->head.end = UPB_PTR_AT(block, size, char);
+ a->head.UPB_PRIVATE(ptr) = UPB_PTR_AT(block, kUpb_MemblockReserve, char);
+ a->head.UPB_PRIVATE(end) = UPB_PTR_AT(block, size, char);
- UPB_POISON_MEMORY_REGION(a->head.ptr, a->head.end - a->head.ptr);
+ UPB_POISON_MEMORY_REGION(a->head.UPB_PRIVATE(ptr),
+ a->head.UPB_PRIVATE(end) - a->head.UPB_PRIVATE(ptr));
}
-static bool upb_Arena_AllocBlock(upb_Arena* a, size_t size) {
+static bool _upb_Arena_AllocBlock(upb_Arena* a, size_t size) {
if (!a->block_alloc) return false;
_upb_MemBlock* last_block = upb_Atomic_Load(&a->blocks, memory_order_acquire);
size_t last_size = last_block != NULL ? last_block->size : 128;
- size_t block_size = UPB_MAX(size, last_size * 2) + memblock_reserve;
- _upb_MemBlock* block = upb_malloc(upb_Arena_BlockAlloc(a), block_size);
+ size_t block_size = UPB_MAX(size, last_size * 2) + kUpb_MemblockReserve;
+ _upb_MemBlock* block = upb_malloc(_upb_Arena_BlockAlloc(a), block_size);
if (!block) return false;
- upb_Arena_AddBlock(a, block, block_size);
+ _upb_Arena_AddBlock(a, block, block_size);
+ UPB_ASSERT(UPB_PRIVATE(_upb_ArenaHas)(a) >= size);
return true;
}
-void* _upb_Arena_SlowMalloc(upb_Arena* a, size_t size) {
- if (!upb_Arena_AllocBlock(a, size)) return NULL; /* Out of memory. */
- UPB_ASSERT(_upb_ArenaHas(a) >= size);
- return upb_Arena_Malloc(a, size);
+void* UPB_PRIVATE(_upb_Arena_SlowMalloc)(upb_Arena* a, size_t size) {
+ if (!_upb_Arena_AllocBlock(a, size)) return NULL; // OOM
+ return upb_Arena_Malloc(a, size - UPB_ASAN_GUARD_SIZE);
}
-/* Public Arena API ***********************************************************/
-
-static upb_Arena* upb_Arena_InitSlow(upb_alloc* alloc) {
- const size_t first_block_overhead = sizeof(upb_Arena) + memblock_reserve;
+static upb_Arena* _upb_Arena_InitSlow(upb_alloc* alloc) {
+ const size_t first_block_overhead = sizeof(upb_Arena) + kUpb_MemblockReserve;
upb_Arena* a;
- /* We need to malloc the initial block. */
+ // We need to malloc the initial block.
char* mem;
size_t n = first_block_overhead + 256;
if (!alloc || !(mem = upb_malloc(alloc, n))) {
@@ -5652,13 +5698,13 @@
a = UPB_PTR_AT(mem, n - sizeof(*a), upb_Arena);
n -= sizeof(*a);
- a->block_alloc = upb_Arena_MakeBlockAlloc(alloc, 0);
+ a->block_alloc = _upb_Arena_MakeBlockAlloc(alloc, 0);
upb_Atomic_Init(&a->parent_or_count, _upb_Arena_TaggedFromRefcount(1));
upb_Atomic_Init(&a->next, NULL);
upb_Atomic_Init(&a->tail, a);
upb_Atomic_Init(&a->blocks, NULL);
- upb_Arena_AddBlock(a, mem, n);
+ _upb_Arena_AddBlock(a, mem, n);
return a;
}
@@ -5679,7 +5725,7 @@
n = UPB_ALIGN_DOWN(n, UPB_ALIGN_OF(upb_Arena));
if (UPB_UNLIKELY(n < sizeof(upb_Arena))) {
- return upb_Arena_InitSlow(alloc);
+ return _upb_Arena_InitSlow(alloc);
}
a = UPB_PTR_AT(mem, n - sizeof(*a), upb_Arena);
@@ -5688,21 +5734,21 @@
upb_Atomic_Init(&a->next, NULL);
upb_Atomic_Init(&a->tail, a);
upb_Atomic_Init(&a->blocks, NULL);
- a->block_alloc = upb_Arena_MakeBlockAlloc(alloc, 1);
- a->head.ptr = mem;
- a->head.end = UPB_PTR_AT(mem, n - sizeof(*a), char);
+ a->block_alloc = _upb_Arena_MakeBlockAlloc(alloc, 1);
+ a->head.UPB_PRIVATE(ptr) = mem;
+ a->head.UPB_PRIVATE(end) = UPB_PTR_AT(mem, n - sizeof(*a), char);
return a;
}
-static void arena_dofree(upb_Arena* a) {
+static void _upb_Arena_DoFree(upb_Arena* a) {
UPB_ASSERT(_upb_Arena_RefCountFromTagged(a->parent_or_count) == 1);
while (a != NULL) {
// Load first since arena itself is likely from one of its blocks.
upb_Arena* next_arena =
(upb_Arena*)upb_Atomic_Load(&a->next, memory_order_acquire);
- upb_alloc* block_alloc = upb_Arena_BlockAlloc(a);
+ upb_alloc* block_alloc = _upb_Arena_BlockAlloc(a);
_upb_MemBlock* block = upb_Atomic_Load(&a->blocks, memory_order_acquire);
while (block != NULL) {
// Load first since we are deleting block.
@@ -5727,7 +5773,7 @@
// expensive then direct loads. As an optimization, we only do RMW ops
// when we need to update things for other threads to see.
if (poc == _upb_Arena_TaggedFromRefcount(1)) {
- arena_dofree(a);
+ _upb_Arena_DoFree(a);
return;
}
@@ -5778,14 +5824,14 @@
// In parent pointer mode, it may change what pointer it refers to in the
// tree, but it will always approach a root. Any operation that walks the
// tree to the root may collapse levels of the tree concurrently.
- _upb_ArenaRoot r1 = _upb_Arena_FindRoot(a1);
- _upb_ArenaRoot r2 = _upb_Arena_FindRoot(a2);
+ upb_ArenaRoot r1 = _upb_Arena_FindRoot(a1);
+ upb_ArenaRoot r2 = _upb_Arena_FindRoot(a2);
if (r1.root == r2.root) return r1.root; // Already fused.
// Avoid cycles by always fusing into the root with the lower address.
if ((uintptr_t)r1.root > (uintptr_t)r2.root) {
- _upb_ArenaRoot tmp = r1;
+ upb_ArenaRoot tmp = r1;
r1 = r2;
r2 = tmp;
}
@@ -5845,7 +5891,7 @@
// Do not fuse initial blocks since we cannot lifetime extend them.
// Any other fuse scenario is allowed.
- if (upb_Arena_HasInitialBlock(a1) || upb_Arena_HasInitialBlock(a2)) {
+ if (_upb_Arena_HasInitialBlock(a1) || _upb_Arena_HasInitialBlock(a2)) {
return false;
}
@@ -5860,8 +5906,8 @@
}
bool upb_Arena_IncRefFor(upb_Arena* arena, const void* owner) {
- _upb_ArenaRoot r;
- if (upb_Arena_HasInitialBlock(arena)) return false;
+ upb_ArenaRoot r;
+ if (_upb_Arena_HasInitialBlock(arena)) return false;
retry:
r = _upb_Arena_FindRoot(arena);
@@ -14639,7 +14685,7 @@
static void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size,
int copy, char* data, size_t data_offset,
upb_StringView* dst) {
- d->arena.head.ptr += copy;
+ d->arena.head.UPB_PRIVATE(ptr) += copy;
dst->data = data + data_offset;
UPB_UNPOISON_MEMORY_REGION(data, copy);
memcpy(data, ptr, copy);
@@ -14671,8 +14717,8 @@
ptr += tagbytes + 1; \
dst->size = size; \
\
- buf = d->arena.head.ptr; \
- arena_has = _upb_ArenaHas(&d->arena); \
+ buf = d->arena.head.UPB_PRIVATE(ptr); \
+ arena_has = UPB_PRIVATE(_upb_ArenaHas)(&d->arena); \
common_has = UPB_MIN(arena_has, \
upb_EpsCopyInputStream_BytesAvailable(&d->input, ptr)); \
\
@@ -14849,10 +14895,10 @@
size_t size = m->UPB_PRIVATE(size) + sizeof(upb_Message_Internal);
char* msg_data;
if (UPB_LIKELY(msg_ceil_bytes > 0 &&
- _upb_ArenaHas(&d->arena) >= msg_ceil_bytes)) {
+ UPB_PRIVATE(_upb_ArenaHas)(&d->arena) >= msg_ceil_bytes)) {
UPB_ASSERT(size <= (size_t)msg_ceil_bytes);
- msg_data = d->arena.head.ptr;
- d->arena.head.ptr += size;
+ msg_data = d->arena.head.UPB_PRIVATE(ptr);
+ d->arena.head.UPB_PRIVATE(ptr) += size;
UPB_UNPOISON_MEMORY_REGION(msg_data, msg_ceil_bytes);
memset(msg_data, 0, msg_ceil_bytes);
UPB_POISON_MEMORY_REGION(msg_data + size, msg_ceil_bytes - size);
diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h
index 3f6a270..f10d229 100644
--- a/php/ext/google/protobuf/php-upb.h
+++ b/php/ext/google/protobuf/php-upb.h
@@ -634,7 +634,8 @@
// LINT.IfChange(struct_definition)
typedef struct {
- char *ptr, *end;
+ char* UPB_ONLYBITS(ptr);
+ char* UPB_ONLYBITS(end);
} _upb_ArenaHead;
// LINT.ThenChange(//depot/google3/third_party/upb/bits/typescript/arena.ts)
@@ -650,33 +651,34 @@
UPB_API void upb_Arena_Free(upb_Arena* a);
UPB_API bool upb_Arena_Fuse(upb_Arena* a, upb_Arena* b);
-bool upb_Arena_IncRefFor(upb_Arena* arena, const void* owner);
-void upb_Arena_DecRefFor(upb_Arena* arena, const void* owner);
+bool upb_Arena_IncRefFor(upb_Arena* a, const void* owner);
+void upb_Arena_DecRefFor(upb_Arena* a, const void* owner);
-void* _upb_Arena_SlowMalloc(upb_Arena* a, size_t size);
-size_t upb_Arena_SpaceAllocated(upb_Arena* arena);
-uint32_t upb_Arena_DebugRefCount(upb_Arena* arena);
+void* UPB_PRIVATE(_upb_Arena_SlowMalloc)(upb_Arena* a, size_t size);
-UPB_INLINE size_t _upb_ArenaHas(upb_Arena* a) {
- _upb_ArenaHead* h = (_upb_ArenaHead*)a;
- return (size_t)(h->end - h->ptr);
+size_t upb_Arena_SpaceAllocated(upb_Arena* a);
+uint32_t upb_Arena_DebugRefCount(upb_Arena* a);
+
+UPB_INLINE size_t UPB_PRIVATE(_upb_ArenaHas)(upb_Arena* a) {
+ const _upb_ArenaHead* h = (_upb_ArenaHead*)a;
+ return (size_t)(h->UPB_ONLYBITS(end) - h->UPB_ONLYBITS(ptr));
}
UPB_API_INLINE void* upb_Arena_Malloc(upb_Arena* a, size_t size) {
size = UPB_ALIGN_MALLOC(size);
- size_t span = size + UPB_ASAN_GUARD_SIZE;
- if (UPB_UNLIKELY(_upb_ArenaHas(a) < span)) {
- return _upb_Arena_SlowMalloc(a, size);
+ const size_t span = size + UPB_ASAN_GUARD_SIZE;
+ if (UPB_UNLIKELY(UPB_PRIVATE(_upb_ArenaHas)(a) < span)) {
+ return UPB_PRIVATE(_upb_Arena_SlowMalloc)(a, span);
}
// We have enough space to do a fast malloc.
_upb_ArenaHead* h = (_upb_ArenaHead*)a;
- void* ret = h->ptr;
+ void* ret = h->UPB_ONLYBITS(ptr);
UPB_ASSERT(UPB_ALIGN_MALLOC((uintptr_t)ret) == (uintptr_t)ret);
UPB_ASSERT(UPB_ALIGN_MALLOC(size) == size);
UPB_UNPOISON_MEMORY_REGION(ret, size);
- h->ptr += span;
+ h->UPB_ONLYBITS(ptr) += span;
return ret;
}
@@ -691,9 +693,10 @@
oldsize = UPB_ALIGN_MALLOC(oldsize);
size = UPB_ALIGN_MALLOC(size);
// Must be the last alloc.
- UPB_ASSERT((char*)ptr + oldsize == h->ptr - UPB_ASAN_GUARD_SIZE);
+ UPB_ASSERT((char*)ptr + oldsize ==
+ h->UPB_ONLYBITS(ptr) - UPB_ASAN_GUARD_SIZE);
UPB_ASSERT(size <= oldsize);
- h->ptr = (char*)ptr + size;
+ h->UPB_ONLYBITS(ptr) = (char*)ptr + size;
}
UPB_API_INLINE void* upb_Arena_Realloc(upb_Arena* a, void* ptr, size_t oldsize,
@@ -701,12 +704,13 @@
_upb_ArenaHead* h = (_upb_ArenaHead*)a;
oldsize = UPB_ALIGN_MALLOC(oldsize);
size = UPB_ALIGN_MALLOC(size);
- bool is_most_recent_alloc = (uintptr_t)ptr + oldsize == (uintptr_t)h->ptr;
+ bool is_most_recent_alloc =
+ (uintptr_t)ptr + oldsize == (uintptr_t)h->UPB_ONLYBITS(ptr);
if (is_most_recent_alloc) {
ptrdiff_t diff = size - oldsize;
- if ((ptrdiff_t)_upb_ArenaHas(a) >= diff) {
- h->ptr += diff;
+ if ((ptrdiff_t)UPB_PRIVATE(_upb_ArenaHas)(a) >= diff) {
+ h->UPB_ONLYBITS(ptr) += diff;
return ptr;
}
} else if (size <= oldsize) {
@@ -12301,51 +12305,6 @@
};
// LINT.ThenChange(//depot/google3/third_party/upb/bits/typescript/arena.ts)
-UPB_INLINE bool _upb_Arena_IsTaggedRefcount(uintptr_t parent_or_count) {
- return (parent_or_count & 1) == 1;
-}
-
-UPB_INLINE bool _upb_Arena_IsTaggedPointer(uintptr_t parent_or_count) {
- return (parent_or_count & 1) == 0;
-}
-
-UPB_INLINE uintptr_t _upb_Arena_RefCountFromTagged(uintptr_t parent_or_count) {
- UPB_ASSERT(_upb_Arena_IsTaggedRefcount(parent_or_count));
- return parent_or_count >> 1;
-}
-
-UPB_INLINE uintptr_t _upb_Arena_TaggedFromRefcount(uintptr_t refcount) {
- uintptr_t parent_or_count = (refcount << 1) | 1;
- UPB_ASSERT(_upb_Arena_IsTaggedRefcount(parent_or_count));
- return parent_or_count;
-}
-
-UPB_INLINE upb_Arena* _upb_Arena_PointerFromTagged(uintptr_t parent_or_count) {
- UPB_ASSERT(_upb_Arena_IsTaggedPointer(parent_or_count));
- return (upb_Arena*)parent_or_count;
-}
-
-UPB_INLINE uintptr_t _upb_Arena_TaggedFromPointer(upb_Arena* a) {
- uintptr_t parent_or_count = (uintptr_t)a;
- UPB_ASSERT(_upb_Arena_IsTaggedPointer(parent_or_count));
- return parent_or_count;
-}
-
-UPB_INLINE upb_alloc* upb_Arena_BlockAlloc(upb_Arena* arena) {
- return (upb_alloc*)(arena->block_alloc & ~0x1);
-}
-
-UPB_INLINE uintptr_t upb_Arena_MakeBlockAlloc(upb_alloc* alloc,
- bool has_initial) {
- uintptr_t alloc_uint = (uintptr_t)alloc;
- UPB_ASSERT((alloc_uint & 1) == 0);
- return alloc_uint | (has_initial ? 1 : 0);
-}
-
-UPB_INLINE bool upb_Arena_HasInitialBlock(upb_Arena* arena) {
- return arena->block_alloc & 0x1;
-}
-
#endif /* UPB_MEM_INTERNAL_ARENA_H_ */
diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c
index 965581d..965e40a 100644
--- a/ruby/ext/google/protobuf_c/ruby-upb.c
+++ b/ruby/ext/google/protobuf_c/ruby-upb.c
@@ -5035,6 +5035,9 @@
upb_alloc upb_alloc_global = {&upb_global_allocfunc};
+#include <stddef.h>
+#include <stdint.h>
+
// Must be last.
@@ -5045,15 +5048,59 @@
// Data follows.
};
-static const size_t memblock_reserve =
+static const size_t kUpb_MemblockReserve =
UPB_ALIGN_UP(sizeof(_upb_MemBlock), UPB_MALLOC_ALIGN);
-typedef struct _upb_ArenaRoot {
+typedef struct {
upb_Arena* root;
uintptr_t tagged_count;
-} _upb_ArenaRoot;
+} upb_ArenaRoot;
-static _upb_ArenaRoot _upb_Arena_FindRoot(upb_Arena* a) {
+static bool _upb_Arena_IsTaggedRefcount(uintptr_t parent_or_count) {
+ return (parent_or_count & 1) == 1;
+}
+
+static bool _upb_Arena_IsTaggedPointer(uintptr_t parent_or_count) {
+ return (parent_or_count & 1) == 0;
+}
+
+static uintptr_t _upb_Arena_RefCountFromTagged(uintptr_t parent_or_count) {
+ UPB_ASSERT(_upb_Arena_IsTaggedRefcount(parent_or_count));
+ return parent_or_count >> 1;
+}
+
+static uintptr_t _upb_Arena_TaggedFromRefcount(uintptr_t refcount) {
+ uintptr_t parent_or_count = (refcount << 1) | 1;
+ UPB_ASSERT(_upb_Arena_IsTaggedRefcount(parent_or_count));
+ return parent_or_count;
+}
+
+static upb_Arena* _upb_Arena_PointerFromTagged(uintptr_t parent_or_count) {
+ UPB_ASSERT(_upb_Arena_IsTaggedPointer(parent_or_count));
+ return (upb_Arena*)parent_or_count;
+}
+
+static uintptr_t _upb_Arena_TaggedFromPointer(upb_Arena* a) {
+ uintptr_t parent_or_count = (uintptr_t)a;
+ UPB_ASSERT(_upb_Arena_IsTaggedPointer(parent_or_count));
+ return parent_or_count;
+}
+
+static upb_alloc* _upb_Arena_BlockAlloc(upb_Arena* arena) {
+ return (upb_alloc*)(arena->block_alloc & ~0x1);
+}
+
+static uintptr_t _upb_Arena_MakeBlockAlloc(upb_alloc* alloc, bool has_initial) {
+ uintptr_t alloc_uint = (uintptr_t)alloc;
+ UPB_ASSERT((alloc_uint & 1) == 0);
+ return alloc_uint | (has_initial ? 1 : 0);
+}
+
+static bool _upb_Arena_HasInitialBlock(upb_Arena* arena) {
+ return arena->block_alloc & 0x1;
+}
+
+static upb_ArenaRoot _upb_Arena_FindRoot(upb_Arena* a) {
uintptr_t poc = upb_Atomic_Load(&a->parent_or_count, memory_order_acquire);
while (_upb_Arena_IsTaggedPointer(poc)) {
upb_Arena* next = _upb_Arena_PointerFromTagged(poc);
@@ -5087,7 +5134,7 @@
a = next;
poc = next_poc;
}
- return (_upb_ArenaRoot){.root = a, .tagged_count = poc};
+ return (upb_ArenaRoot){.root = a, .tagged_count = poc};
}
size_t upb_Arena_SpaceAllocated(upb_Arena* arena) {
@@ -5118,7 +5165,7 @@
return _upb_Arena_RefCountFromTagged(poc);
}
-static void upb_Arena_AddBlock(upb_Arena* a, void* ptr, size_t size) {
+static void _upb_Arena_AddBlock(upb_Arena* a, void* ptr, size_t size) {
_upb_MemBlock* block = ptr;
// Insert into linked list.
@@ -5126,37 +5173,36 @@
upb_Atomic_Init(&block->next, a->blocks);
upb_Atomic_Store(&a->blocks, block, memory_order_release);
- a->head.ptr = UPB_PTR_AT(block, memblock_reserve, char);
- a->head.end = UPB_PTR_AT(block, size, char);
+ a->head.UPB_PRIVATE(ptr) = UPB_PTR_AT(block, kUpb_MemblockReserve, char);
+ a->head.UPB_PRIVATE(end) = UPB_PTR_AT(block, size, char);
- UPB_POISON_MEMORY_REGION(a->head.ptr, a->head.end - a->head.ptr);
+ UPB_POISON_MEMORY_REGION(a->head.UPB_PRIVATE(ptr),
+ a->head.UPB_PRIVATE(end) - a->head.UPB_PRIVATE(ptr));
}
-static bool upb_Arena_AllocBlock(upb_Arena* a, size_t size) {
+static bool _upb_Arena_AllocBlock(upb_Arena* a, size_t size) {
if (!a->block_alloc) return false;
_upb_MemBlock* last_block = upb_Atomic_Load(&a->blocks, memory_order_acquire);
size_t last_size = last_block != NULL ? last_block->size : 128;
- size_t block_size = UPB_MAX(size, last_size * 2) + memblock_reserve;
- _upb_MemBlock* block = upb_malloc(upb_Arena_BlockAlloc(a), block_size);
+ size_t block_size = UPB_MAX(size, last_size * 2) + kUpb_MemblockReserve;
+ _upb_MemBlock* block = upb_malloc(_upb_Arena_BlockAlloc(a), block_size);
if (!block) return false;
- upb_Arena_AddBlock(a, block, block_size);
+ _upb_Arena_AddBlock(a, block, block_size);
+ UPB_ASSERT(UPB_PRIVATE(_upb_ArenaHas)(a) >= size);
return true;
}
-void* _upb_Arena_SlowMalloc(upb_Arena* a, size_t size) {
- if (!upb_Arena_AllocBlock(a, size)) return NULL; /* Out of memory. */
- UPB_ASSERT(_upb_ArenaHas(a) >= size);
- return upb_Arena_Malloc(a, size);
+void* UPB_PRIVATE(_upb_Arena_SlowMalloc)(upb_Arena* a, size_t size) {
+ if (!_upb_Arena_AllocBlock(a, size)) return NULL; // OOM
+ return upb_Arena_Malloc(a, size - UPB_ASAN_GUARD_SIZE);
}
-/* Public Arena API ***********************************************************/
-
-static upb_Arena* upb_Arena_InitSlow(upb_alloc* alloc) {
- const size_t first_block_overhead = sizeof(upb_Arena) + memblock_reserve;
+static upb_Arena* _upb_Arena_InitSlow(upb_alloc* alloc) {
+ const size_t first_block_overhead = sizeof(upb_Arena) + kUpb_MemblockReserve;
upb_Arena* a;
- /* We need to malloc the initial block. */
+ // We need to malloc the initial block.
char* mem;
size_t n = first_block_overhead + 256;
if (!alloc || !(mem = upb_malloc(alloc, n))) {
@@ -5166,13 +5212,13 @@
a = UPB_PTR_AT(mem, n - sizeof(*a), upb_Arena);
n -= sizeof(*a);
- a->block_alloc = upb_Arena_MakeBlockAlloc(alloc, 0);
+ a->block_alloc = _upb_Arena_MakeBlockAlloc(alloc, 0);
upb_Atomic_Init(&a->parent_or_count, _upb_Arena_TaggedFromRefcount(1));
upb_Atomic_Init(&a->next, NULL);
upb_Atomic_Init(&a->tail, a);
upb_Atomic_Init(&a->blocks, NULL);
- upb_Arena_AddBlock(a, mem, n);
+ _upb_Arena_AddBlock(a, mem, n);
return a;
}
@@ -5193,7 +5239,7 @@
n = UPB_ALIGN_DOWN(n, UPB_ALIGN_OF(upb_Arena));
if (UPB_UNLIKELY(n < sizeof(upb_Arena))) {
- return upb_Arena_InitSlow(alloc);
+ return _upb_Arena_InitSlow(alloc);
}
a = UPB_PTR_AT(mem, n - sizeof(*a), upb_Arena);
@@ -5202,21 +5248,21 @@
upb_Atomic_Init(&a->next, NULL);
upb_Atomic_Init(&a->tail, a);
upb_Atomic_Init(&a->blocks, NULL);
- a->block_alloc = upb_Arena_MakeBlockAlloc(alloc, 1);
- a->head.ptr = mem;
- a->head.end = UPB_PTR_AT(mem, n - sizeof(*a), char);
+ a->block_alloc = _upb_Arena_MakeBlockAlloc(alloc, 1);
+ a->head.UPB_PRIVATE(ptr) = mem;
+ a->head.UPB_PRIVATE(end) = UPB_PTR_AT(mem, n - sizeof(*a), char);
return a;
}
-static void arena_dofree(upb_Arena* a) {
+static void _upb_Arena_DoFree(upb_Arena* a) {
UPB_ASSERT(_upb_Arena_RefCountFromTagged(a->parent_or_count) == 1);
while (a != NULL) {
// Load first since arena itself is likely from one of its blocks.
upb_Arena* next_arena =
(upb_Arena*)upb_Atomic_Load(&a->next, memory_order_acquire);
- upb_alloc* block_alloc = upb_Arena_BlockAlloc(a);
+ upb_alloc* block_alloc = _upb_Arena_BlockAlloc(a);
_upb_MemBlock* block = upb_Atomic_Load(&a->blocks, memory_order_acquire);
while (block != NULL) {
// Load first since we are deleting block.
@@ -5241,7 +5287,7 @@
// expensive then direct loads. As an optimization, we only do RMW ops
// when we need to update things for other threads to see.
if (poc == _upb_Arena_TaggedFromRefcount(1)) {
- arena_dofree(a);
+ _upb_Arena_DoFree(a);
return;
}
@@ -5292,14 +5338,14 @@
// In parent pointer mode, it may change what pointer it refers to in the
// tree, but it will always approach a root. Any operation that walks the
// tree to the root may collapse levels of the tree concurrently.
- _upb_ArenaRoot r1 = _upb_Arena_FindRoot(a1);
- _upb_ArenaRoot r2 = _upb_Arena_FindRoot(a2);
+ upb_ArenaRoot r1 = _upb_Arena_FindRoot(a1);
+ upb_ArenaRoot r2 = _upb_Arena_FindRoot(a2);
if (r1.root == r2.root) return r1.root; // Already fused.
// Avoid cycles by always fusing into the root with the lower address.
if ((uintptr_t)r1.root > (uintptr_t)r2.root) {
- _upb_ArenaRoot tmp = r1;
+ upb_ArenaRoot tmp = r1;
r1 = r2;
r2 = tmp;
}
@@ -5359,7 +5405,7 @@
// Do not fuse initial blocks since we cannot lifetime extend them.
// Any other fuse scenario is allowed.
- if (upb_Arena_HasInitialBlock(a1) || upb_Arena_HasInitialBlock(a2)) {
+ if (_upb_Arena_HasInitialBlock(a1) || _upb_Arena_HasInitialBlock(a2)) {
return false;
}
@@ -5374,8 +5420,8 @@
}
bool upb_Arena_IncRefFor(upb_Arena* arena, const void* owner) {
- _upb_ArenaRoot r;
- if (upb_Arena_HasInitialBlock(arena)) return false;
+ upb_ArenaRoot r;
+ if (_upb_Arena_HasInitialBlock(arena)) return false;
retry:
r = _upb_Arena_FindRoot(arena);
@@ -14153,7 +14199,7 @@
static void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size,
int copy, char* data, size_t data_offset,
upb_StringView* dst) {
- d->arena.head.ptr += copy;
+ d->arena.head.UPB_PRIVATE(ptr) += copy;
dst->data = data + data_offset;
UPB_UNPOISON_MEMORY_REGION(data, copy);
memcpy(data, ptr, copy);
@@ -14185,8 +14231,8 @@
ptr += tagbytes + 1; \
dst->size = size; \
\
- buf = d->arena.head.ptr; \
- arena_has = _upb_ArenaHas(&d->arena); \
+ buf = d->arena.head.UPB_PRIVATE(ptr); \
+ arena_has = UPB_PRIVATE(_upb_ArenaHas)(&d->arena); \
common_has = UPB_MIN(arena_has, \
upb_EpsCopyInputStream_BytesAvailable(&d->input, ptr)); \
\
@@ -14363,10 +14409,10 @@
size_t size = m->UPB_PRIVATE(size) + sizeof(upb_Message_Internal);
char* msg_data;
if (UPB_LIKELY(msg_ceil_bytes > 0 &&
- _upb_ArenaHas(&d->arena) >= msg_ceil_bytes)) {
+ UPB_PRIVATE(_upb_ArenaHas)(&d->arena) >= msg_ceil_bytes)) {
UPB_ASSERT(size <= (size_t)msg_ceil_bytes);
- msg_data = d->arena.head.ptr;
- d->arena.head.ptr += size;
+ msg_data = d->arena.head.UPB_PRIVATE(ptr);
+ d->arena.head.UPB_PRIVATE(ptr) += size;
UPB_UNPOISON_MEMORY_REGION(msg_data, msg_ceil_bytes);
memset(msg_data, 0, msg_ceil_bytes);
UPB_POISON_MEMORY_REGION(msg_data + size, msg_ceil_bytes - size);
diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h
index b339a03..aa54ab7 100755
--- a/ruby/ext/google/protobuf_c/ruby-upb.h
+++ b/ruby/ext/google/protobuf_c/ruby-upb.h
@@ -636,7 +636,8 @@
// LINT.IfChange(struct_definition)
typedef struct {
- char *ptr, *end;
+ char* UPB_ONLYBITS(ptr);
+ char* UPB_ONLYBITS(end);
} _upb_ArenaHead;
// LINT.ThenChange(//depot/google3/third_party/upb/bits/typescript/arena.ts)
@@ -652,33 +653,34 @@
UPB_API void upb_Arena_Free(upb_Arena* a);
UPB_API bool upb_Arena_Fuse(upb_Arena* a, upb_Arena* b);
-bool upb_Arena_IncRefFor(upb_Arena* arena, const void* owner);
-void upb_Arena_DecRefFor(upb_Arena* arena, const void* owner);
+bool upb_Arena_IncRefFor(upb_Arena* a, const void* owner);
+void upb_Arena_DecRefFor(upb_Arena* a, const void* owner);
-void* _upb_Arena_SlowMalloc(upb_Arena* a, size_t size);
-size_t upb_Arena_SpaceAllocated(upb_Arena* arena);
-uint32_t upb_Arena_DebugRefCount(upb_Arena* arena);
+void* UPB_PRIVATE(_upb_Arena_SlowMalloc)(upb_Arena* a, size_t size);
-UPB_INLINE size_t _upb_ArenaHas(upb_Arena* a) {
- _upb_ArenaHead* h = (_upb_ArenaHead*)a;
- return (size_t)(h->end - h->ptr);
+size_t upb_Arena_SpaceAllocated(upb_Arena* a);
+uint32_t upb_Arena_DebugRefCount(upb_Arena* a);
+
+UPB_INLINE size_t UPB_PRIVATE(_upb_ArenaHas)(upb_Arena* a) {
+ const _upb_ArenaHead* h = (_upb_ArenaHead*)a;
+ return (size_t)(h->UPB_ONLYBITS(end) - h->UPB_ONLYBITS(ptr));
}
UPB_API_INLINE void* upb_Arena_Malloc(upb_Arena* a, size_t size) {
size = UPB_ALIGN_MALLOC(size);
- size_t span = size + UPB_ASAN_GUARD_SIZE;
- if (UPB_UNLIKELY(_upb_ArenaHas(a) < span)) {
- return _upb_Arena_SlowMalloc(a, size);
+ const size_t span = size + UPB_ASAN_GUARD_SIZE;
+ if (UPB_UNLIKELY(UPB_PRIVATE(_upb_ArenaHas)(a) < span)) {
+ return UPB_PRIVATE(_upb_Arena_SlowMalloc)(a, span);
}
// We have enough space to do a fast malloc.
_upb_ArenaHead* h = (_upb_ArenaHead*)a;
- void* ret = h->ptr;
+ void* ret = h->UPB_ONLYBITS(ptr);
UPB_ASSERT(UPB_ALIGN_MALLOC((uintptr_t)ret) == (uintptr_t)ret);
UPB_ASSERT(UPB_ALIGN_MALLOC(size) == size);
UPB_UNPOISON_MEMORY_REGION(ret, size);
- h->ptr += span;
+ h->UPB_ONLYBITS(ptr) += span;
return ret;
}
@@ -693,9 +695,10 @@
oldsize = UPB_ALIGN_MALLOC(oldsize);
size = UPB_ALIGN_MALLOC(size);
// Must be the last alloc.
- UPB_ASSERT((char*)ptr + oldsize == h->ptr - UPB_ASAN_GUARD_SIZE);
+ UPB_ASSERT((char*)ptr + oldsize ==
+ h->UPB_ONLYBITS(ptr) - UPB_ASAN_GUARD_SIZE);
UPB_ASSERT(size <= oldsize);
- h->ptr = (char*)ptr + size;
+ h->UPB_ONLYBITS(ptr) = (char*)ptr + size;
}
UPB_API_INLINE void* upb_Arena_Realloc(upb_Arena* a, void* ptr, size_t oldsize,
@@ -703,12 +706,13 @@
_upb_ArenaHead* h = (_upb_ArenaHead*)a;
oldsize = UPB_ALIGN_MALLOC(oldsize);
size = UPB_ALIGN_MALLOC(size);
- bool is_most_recent_alloc = (uintptr_t)ptr + oldsize == (uintptr_t)h->ptr;
+ bool is_most_recent_alloc =
+ (uintptr_t)ptr + oldsize == (uintptr_t)h->UPB_ONLYBITS(ptr);
if (is_most_recent_alloc) {
ptrdiff_t diff = size - oldsize;
- if ((ptrdiff_t)_upb_ArenaHas(a) >= diff) {
- h->ptr += diff;
+ if ((ptrdiff_t)UPB_PRIVATE(_upb_ArenaHas)(a) >= diff) {
+ h->UPB_ONLYBITS(ptr) += diff;
return ptr;
}
} else if (size <= oldsize) {
@@ -12073,51 +12077,6 @@
};
// LINT.ThenChange(//depot/google3/third_party/upb/bits/typescript/arena.ts)
-UPB_INLINE bool _upb_Arena_IsTaggedRefcount(uintptr_t parent_or_count) {
- return (parent_or_count & 1) == 1;
-}
-
-UPB_INLINE bool _upb_Arena_IsTaggedPointer(uintptr_t parent_or_count) {
- return (parent_or_count & 1) == 0;
-}
-
-UPB_INLINE uintptr_t _upb_Arena_RefCountFromTagged(uintptr_t parent_or_count) {
- UPB_ASSERT(_upb_Arena_IsTaggedRefcount(parent_or_count));
- return parent_or_count >> 1;
-}
-
-UPB_INLINE uintptr_t _upb_Arena_TaggedFromRefcount(uintptr_t refcount) {
- uintptr_t parent_or_count = (refcount << 1) | 1;
- UPB_ASSERT(_upb_Arena_IsTaggedRefcount(parent_or_count));
- return parent_or_count;
-}
-
-UPB_INLINE upb_Arena* _upb_Arena_PointerFromTagged(uintptr_t parent_or_count) {
- UPB_ASSERT(_upb_Arena_IsTaggedPointer(parent_or_count));
- return (upb_Arena*)parent_or_count;
-}
-
-UPB_INLINE uintptr_t _upb_Arena_TaggedFromPointer(upb_Arena* a) {
- uintptr_t parent_or_count = (uintptr_t)a;
- UPB_ASSERT(_upb_Arena_IsTaggedPointer(parent_or_count));
- return parent_or_count;
-}
-
-UPB_INLINE upb_alloc* upb_Arena_BlockAlloc(upb_Arena* arena) {
- return (upb_alloc*)(arena->block_alloc & ~0x1);
-}
-
-UPB_INLINE uintptr_t upb_Arena_MakeBlockAlloc(upb_alloc* alloc,
- bool has_initial) {
- uintptr_t alloc_uint = (uintptr_t)alloc;
- UPB_ASSERT((alloc_uint & 1) == 0);
- return alloc_uint | (has_initial ? 1 : 0);
-}
-
-UPB_INLINE bool upb_Arena_HasInitialBlock(upb_Arena* arena) {
- return arena->block_alloc & 0x1;
-}
-
#endif /* UPB_MEM_INTERNAL_ARENA_H_ */
