Auto-generate files after cl/594992691
diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index e2dbc47..87f95d0 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c
@@ -1760,832 +1760,6 @@ e, ptr, overrun, _upb_EpsCopyInputStream_NoOpCallback); } -/* - * upb_table Implementation - * - * Implementation is heavily inspired by Lua's ltable.c. - */ - -#include <string.h> - - -// Must be last. - -#define UPB_MAXARRSIZE 16 // 2**16 = 64k. - -// From Chromium. -#define ARRAY_SIZE(x) \ - ((sizeof(x) / sizeof(0 [x])) / ((size_t)(!(sizeof(x) % sizeof(0 [x]))))) - -static const double MAX_LOAD = 0.85; - -/* The minimum utilization of the array part of a mixed hash/array table. This - * is a speed/memory-usage tradeoff (though it's not straightforward because of - * cache effects). The lower this is, the more memory we'll use. */ -static const double MIN_DENSITY = 0.1; - -static bool is_pow2(uint64_t v) { return v == 0 || (v & (v - 1)) == 0; } - -static upb_value _upb_value_val(uint64_t val) { - upb_value ret; - _upb_value_setval(&ret, val); - return ret; -} - -static int log2ceil(uint64_t v) { - int ret = 0; - bool pow2 = is_pow2(v); - while (v >>= 1) ret++; - ret = pow2 ? ret : ret + 1; // Ceiling. - return UPB_MIN(UPB_MAXARRSIZE, ret); -} - -/* A type to represent the lookup key of either a strtable or an inttable. */ -typedef union { - uintptr_t num; - struct { - const char* str; - size_t len; - } str; -} lookupkey_t; - -static lookupkey_t strkey2(const char* str, size_t len) { - lookupkey_t k; - k.str.str = str; - k.str.len = len; - return k; -} - -static lookupkey_t intkey(uintptr_t key) { - lookupkey_t k; - k.num = key; - return k; -} - -typedef uint32_t hashfunc_t(upb_tabkey key); -typedef bool eqlfunc_t(upb_tabkey k1, lookupkey_t k2); - -/* Base table (shared code) ***************************************************/ - -static uint32_t upb_inthash(uintptr_t key) { return (uint32_t)key; } - -static const upb_tabent* upb_getentry(const upb_table* t, uint32_t hash) { - return t->entries + (hash & t->mask); -} - -static bool upb_arrhas(upb_tabval key) { return key.val != (uint64_t)-1; } - -static bool isfull(upb_table* t) { return t->count == t->max_count; } - -static bool init(upb_table* t, uint8_t size_lg2, upb_Arena* a) { - size_t bytes; - - t->count = 0; - t->size_lg2 = size_lg2; - t->mask = upb_table_size(t) ? upb_table_size(t) - 1 : 0; - t->max_count = upb_table_size(t) * MAX_LOAD; - bytes = upb_table_size(t) * sizeof(upb_tabent); - if (bytes > 0) { - t->entries = upb_Arena_Malloc(a, bytes); - if (!t->entries) return false; - memset(t->entries, 0, bytes); - } else { - t->entries = NULL; - } - return true; -} - -static upb_tabent* emptyent(upb_table* t, upb_tabent* e) { - upb_tabent* begin = t->entries; - upb_tabent* end = begin + upb_table_size(t); - for (e = e + 1; e < end; e++) { - if (upb_tabent_isempty(e)) return e; - } - for (e = begin; e < end; e++) { - if (upb_tabent_isempty(e)) return e; - } - UPB_ASSERT(false); - return NULL; -} - -static upb_tabent* getentry_mutable(upb_table* t, uint32_t hash) { - return (upb_tabent*)upb_getentry(t, hash); -} - -static const upb_tabent* findentry(const upb_table* t, lookupkey_t key, - uint32_t hash, eqlfunc_t* eql) { - const upb_tabent* e; - - if (t->size_lg2 == 0) return NULL; - e = upb_getentry(t, hash); - if (upb_tabent_isempty(e)) return NULL; - while (1) { - if (eql(e->key, key)) return e; - if ((e = e->next) == NULL) return NULL; - } -} - -static upb_tabent* findentry_mutable(upb_table* t, lookupkey_t key, - uint32_t hash, eqlfunc_t* eql) { - return (upb_tabent*)findentry(t, key, hash, eql); -} - -static bool lookup(const upb_table* t, lookupkey_t key, upb_value* v, - uint32_t hash, eqlfunc_t* eql) { - const upb_tabent* e = findentry(t, key, hash, eql); - if (e) { - if (v) { - _upb_value_setval(v, e->val.val); - } - return true; - } else { - return false; - } -} - -/* The given key must not already exist in the table. */ -static void insert(upb_table* t, lookupkey_t key, upb_tabkey tabkey, - upb_value val, uint32_t hash, hashfunc_t* hashfunc, - eqlfunc_t* eql) { - upb_tabent* mainpos_e; - upb_tabent* our_e; - - UPB_ASSERT(findentry(t, key, hash, eql) == NULL); - - t->count++; - mainpos_e = getentry_mutable(t, hash); - our_e = mainpos_e; - - if (upb_tabent_isempty(mainpos_e)) { - /* Our main position is empty; use it. */ - our_e->next = NULL; - } else { - /* Collision. */ - upb_tabent* new_e = emptyent(t, mainpos_e); - /* Head of collider's chain. */ - upb_tabent* chain = getentry_mutable(t, hashfunc(mainpos_e->key)); - if (chain == mainpos_e) { - /* Existing ent is in its main position (it has the same hash as us, and - * is the head of our chain). Insert to new ent and append to this chain. - */ - new_e->next = mainpos_e->next; - mainpos_e->next = new_e; - our_e = new_e; - } else { - /* Existing ent is not in its main position (it is a node in some other - * chain). This implies that no existing ent in the table has our hash. - * Evict it (updating its chain) and use its ent for head of our chain. */ - *new_e = *mainpos_e; /* copies next. */ - while (chain->next != mainpos_e) { - chain = (upb_tabent*)chain->next; - UPB_ASSERT(chain); - } - chain->next = new_e; - our_e = mainpos_e; - our_e->next = NULL; - } - } - our_e->key = tabkey; - our_e->val.val = val.val; - UPB_ASSERT(findentry(t, key, hash, eql) == our_e); -} - -static bool rm(upb_table* t, lookupkey_t key, upb_value* val, - upb_tabkey* removed, uint32_t hash, eqlfunc_t* eql) { - upb_tabent* chain = getentry_mutable(t, hash); - if (upb_tabent_isempty(chain)) return false; - if (eql(chain->key, key)) { - /* Element to remove is at the head of its chain. */ - t->count--; - if (val) _upb_value_setval(val, chain->val.val); - if (removed) *removed = chain->key; - if (chain->next) { - upb_tabent* move = (upb_tabent*)chain->next; - *chain = *move; - move->key = 0; /* Make the slot empty. */ - } else { - chain->key = 0; /* Make the slot empty. */ - } - return true; - } else { - /* Element to remove is either in a non-head position or not in the - * table. */ - while (chain->next && !eql(chain->next->key, key)) { - chain = (upb_tabent*)chain->next; - } - if (chain->next) { - /* Found element to remove. */ - upb_tabent* rm = (upb_tabent*)chain->next; - t->count--; - if (val) _upb_value_setval(val, chain->next->val.val); - if (removed) *removed = rm->key; - rm->key = 0; /* Make the slot empty. */ - chain->next = rm->next; - return true; - } else { - /* Element to remove is not in the table. */ - return false; - } - } -} - -static size_t next(const upb_table* t, size_t i) { - do { - if (++i >= upb_table_size(t)) return SIZE_MAX - 1; /* Distinct from -1. */ - } while (upb_tabent_isempty(&t->entries[i])); - - return i; -} - -static size_t begin(const upb_table* t) { return next(t, -1); } - -/* upb_strtable ***************************************************************/ - -/* A simple "subclass" of upb_table that only adds a hash function for strings. - */ - -static upb_tabkey strcopy(lookupkey_t k2, upb_Arena* a) { - uint32_t len = (uint32_t)k2.str.len; - char* str = upb_Arena_Malloc(a, k2.str.len + sizeof(uint32_t) + 1); - if (str == NULL) return 0; - memcpy(str, &len, sizeof(uint32_t)); - if (k2.str.len) memcpy(str + sizeof(uint32_t), k2.str.str, k2.str.len); - str[sizeof(uint32_t) + k2.str.len] = '\0'; - return (uintptr_t)str; -} - -/* Adapted from ABSL's wyhash. */ - -static uint64_t UnalignedLoad64(const void* p) { - uint64_t val; - memcpy(&val, p, 8); - return val; -} - -static uint32_t UnalignedLoad32(const void* p) { - uint32_t val; - memcpy(&val, p, 4); - return val; -} - -#if defined(_MSC_VER) && defined(_M_X64) -#include <intrin.h> -#endif - -/* Computes a * b, returning the low 64 bits of the result and storing the high - * 64 bits in |*high|. */ -static uint64_t upb_umul128(uint64_t v0, uint64_t v1, uint64_t* out_high) { -#ifdef __SIZEOF_INT128__ - __uint128_t p = v0; - p *= v1; - *out_high = (uint64_t)(p >> 64); - return (uint64_t)p; -#elif defined(_MSC_VER) && defined(_M_X64) - return _umul128(v0, v1, out_high); -#else - uint64_t a32 = v0 >> 32; - uint64_t a00 = v0 & 0xffffffff; - uint64_t b32 = v1 >> 32; - uint64_t b00 = v1 & 0xffffffff; - uint64_t high = a32 * b32; - uint64_t low = a00 * b00; - uint64_t mid1 = a32 * b00; - uint64_t mid2 = a00 * b32; - low += (mid1 << 32) + (mid2 << 32); - // Omit carry bit, for mixing we do not care about exact numerical precision. - high += (mid1 >> 32) + (mid2 >> 32); - *out_high = high; - return low; -#endif -} - -static uint64_t WyhashMix(uint64_t v0, uint64_t v1) { - uint64_t high; - uint64_t low = upb_umul128(v0, v1, &high); - return low ^ high; -} - -static uint64_t Wyhash(const void* data, size_t len, uint64_t seed, - const uint64_t salt[]) { - const uint8_t* ptr = (const uint8_t*)data; - uint64_t starting_length = (uint64_t)len; - uint64_t current_state = seed ^ salt[0]; - - if (len > 64) { - // If we have more than 64 bytes, we're going to handle chunks of 64 - // bytes at a time. We're going to build up two separate hash states - // which we will then hash together. - uint64_t duplicated_state = current_state; - - do { - uint64_t a = UnalignedLoad64(ptr); - uint64_t b = UnalignedLoad64(ptr + 8); - uint64_t c = UnalignedLoad64(ptr + 16); - uint64_t d = UnalignedLoad64(ptr + 24); - uint64_t e = UnalignedLoad64(ptr + 32); - uint64_t f = UnalignedLoad64(ptr + 40); - uint64_t g = UnalignedLoad64(ptr + 48); - uint64_t h = UnalignedLoad64(ptr + 56); - - uint64_t cs0 = WyhashMix(a ^ salt[1], b ^ current_state); - uint64_t cs1 = WyhashMix(c ^ salt[2], d ^ current_state); - current_state = (cs0 ^ cs1); - - uint64_t ds0 = WyhashMix(e ^ salt[3], f ^ duplicated_state); - uint64_t ds1 = WyhashMix(g ^ salt[4], h ^ duplicated_state); - duplicated_state = (ds0 ^ ds1); - - ptr += 64; - len -= 64; - } while (len > 64); - - current_state = current_state ^ duplicated_state; - } - - // We now have a data `ptr` with at most 64 bytes and the current state - // of the hashing state machine stored in current_state. - while (len > 16) { - uint64_t a = UnalignedLoad64(ptr); - uint64_t b = UnalignedLoad64(ptr + 8); - - current_state = WyhashMix(a ^ salt[1], b ^ current_state); - - ptr += 16; - len -= 16; - } - - // We now have a data `ptr` with at most 16 bytes. - uint64_t a = 0; - uint64_t b = 0; - if (len > 8) { - // When we have at least 9 and at most 16 bytes, set A to the first 64 - // bits of the input and B to the last 64 bits of the input. Yes, they will - // overlap in the middle if we are working with less than the full 16 - // bytes. - a = UnalignedLoad64(ptr); - b = UnalignedLoad64(ptr + len - 8); - } else if (len > 3) { - // If we have at least 4 and at most 8 bytes, set A to the first 32 - // bits and B to the last 32 bits. - a = UnalignedLoad32(ptr); - b = UnalignedLoad32(ptr + len - 4); - } else if (len > 0) { - // If we have at least 1 and at most 3 bytes, read all of the provided - // bits into A, with some adjustments. - a = ((ptr[0] << 16) | (ptr[len >> 1] << 8) | ptr[len - 1]); - b = 0; - } else { - a = 0; - b = 0; - } - - uint64_t w = WyhashMix(a ^ salt[1], b ^ current_state); - uint64_t z = salt[1] ^ starting_length; - return WyhashMix(w, z); -} - -const uint64_t kWyhashSalt[5] = { - 0x243F6A8885A308D3ULL, 0x13198A2E03707344ULL, 0xA4093822299F31D0ULL, - 0x082EFA98EC4E6C89ULL, 0x452821E638D01377ULL, -}; - -uint32_t _upb_Hash(const void* p, size_t n, uint64_t seed) { - return Wyhash(p, n, seed, kWyhashSalt); -} - -static uint32_t _upb_Hash_NoSeed(const char* p, size_t n) { - return _upb_Hash(p, n, 0); -} - -static uint32_t strhash(upb_tabkey key) { - uint32_t len; - char* str = upb_tabstr(key, &len); - return _upb_Hash_NoSeed(str, len); -} - -static bool streql(upb_tabkey k1, lookupkey_t k2) { - uint32_t len; - char* str = upb_tabstr(k1, &len); - return len == k2.str.len && (len == 0 || memcmp(str, k2.str.str, len) == 0); -} - -bool upb_strtable_init(upb_strtable* t, size_t expected_size, upb_Arena* a) { - // Multiply by approximate reciprocal of MAX_LOAD (0.85), with pow2 - // denominator. - size_t need_entries = (expected_size + 1) * 1204 / 1024; - UPB_ASSERT(need_entries >= expected_size * 0.85); - int size_lg2 = upb_Log2Ceiling(need_entries); - return init(&t->t, size_lg2, a); -} - -void upb_strtable_clear(upb_strtable* t) { - size_t bytes = upb_table_size(&t->t) * sizeof(upb_tabent); - t->t.count = 0; - memset((char*)t->t.entries, 0, bytes); -} - -bool upb_strtable_resize(upb_strtable* t, size_t size_lg2, upb_Arena* a) { - upb_strtable new_table; - if (!init(&new_table.t, size_lg2, a)) return false; - - intptr_t iter = UPB_STRTABLE_BEGIN; - upb_StringView key; - upb_value val; - while (upb_strtable_next2(t, &key, &val, &iter)) { - upb_strtable_insert(&new_table, key.data, key.size, val, a); - } - *t = new_table; - return true; -} - -bool upb_strtable_insert(upb_strtable* t, const char* k, size_t len, - upb_value v, upb_Arena* a) { - lookupkey_t key; - upb_tabkey tabkey; - uint32_t hash; - - if (isfull(&t->t)) { - /* Need to resize. New table of double the size, add old elements to it. */ - if (!upb_strtable_resize(t, t->t.size_lg2 + 1, a)) { - return false; - } - } - - key = strkey2(k, len); - tabkey = strcopy(key, a); - if (tabkey == 0) return false; - - hash = _upb_Hash_NoSeed(key.str.str, key.str.len); - insert(&t->t, key, tabkey, v, hash, &strhash, &streql); - return true; -} - -bool upb_strtable_lookup2(const upb_strtable* t, const char* key, size_t len, - upb_value* v) { - uint32_t hash = _upb_Hash_NoSeed(key, len); - return lookup(&t->t, strkey2(key, len), v, hash, &streql); -} - -bool upb_strtable_remove2(upb_strtable* t, const char* key, size_t len, - upb_value* val) { - uint32_t hash = _upb_Hash_NoSeed(key, len); - upb_tabkey tabkey; - return rm(&t->t, strkey2(key, len), val, &tabkey, hash, &streql); -} - -/* Iteration */ - -void upb_strtable_begin(upb_strtable_iter* i, const upb_strtable* t) { - i->t = t; - i->index = begin(&t->t); -} - -void upb_strtable_next(upb_strtable_iter* i) { - i->index = next(&i->t->t, i->index); -} - -bool upb_strtable_done(const upb_strtable_iter* i) { - if (!i->t) return true; - return i->index >= upb_table_size(&i->t->t) || - upb_tabent_isempty(str_tabent(i)); -} - -upb_StringView upb_strtable_iter_key(const upb_strtable_iter* i) { - upb_StringView key; - uint32_t len; - UPB_ASSERT(!upb_strtable_done(i)); - key.data = upb_tabstr(str_tabent(i)->key, &len); - key.size = len; - return key; -} - -upb_value upb_strtable_iter_value(const upb_strtable_iter* i) { - UPB_ASSERT(!upb_strtable_done(i)); - return _upb_value_val(str_tabent(i)->val.val); -} - -void upb_strtable_iter_setdone(upb_strtable_iter* i) { - i->t = NULL; - i->index = SIZE_MAX; -} - -bool upb_strtable_iter_isequal(const upb_strtable_iter* i1, - const upb_strtable_iter* i2) { - if (upb_strtable_done(i1) && upb_strtable_done(i2)) return true; - return i1->t == i2->t && i1->index == i2->index; -} - -/* upb_inttable ***************************************************************/ - -/* For inttables we use a hybrid structure where small keys are kept in an - * array and large keys are put in the hash table. */ - -static uint32_t inthash(upb_tabkey key) { return upb_inthash(key); } - -static bool inteql(upb_tabkey k1, lookupkey_t k2) { return k1 == k2.num; } - -static upb_tabval* mutable_array(upb_inttable* t) { - return (upb_tabval*)t->array; -} - -static upb_tabval* inttable_val(upb_inttable* t, uintptr_t key) { - if (key < t->array_size) { - return upb_arrhas(t->array[key]) ? &(mutable_array(t)[key]) : NULL; - } else { - upb_tabent* e = - findentry_mutable(&t->t, intkey(key), upb_inthash(key), &inteql); - return e ? &e->val : NULL; - } -} - -static const upb_tabval* inttable_val_const(const upb_inttable* t, - uintptr_t key) { - return inttable_val((upb_inttable*)t, key); -} - -size_t upb_inttable_count(const upb_inttable* t) { - return t->t.count + t->array_count; -} - -static void check(upb_inttable* t) { - UPB_UNUSED(t); -#if defined(UPB_DEBUG_TABLE) && !defined(NDEBUG) - { - // This check is very expensive (makes inserts/deletes O(N)). - size_t count = 0; - intptr_t iter = UPB_INTTABLE_BEGIN; - uintptr_t key; - upb_value val; - while (upb_inttable_next(t, &key, &val, &iter)) { - UPB_ASSERT(upb_inttable_lookup(t, key, NULL)); - } - UPB_ASSERT(count == upb_inttable_count(t)); - } -#endif -} - -bool upb_inttable_sizedinit(upb_inttable* t, size_t asize, int hsize_lg2, - upb_Arena* a) { - size_t array_bytes; - - if (!init(&t->t, hsize_lg2, a)) return false; - /* Always make the array part at least 1 long, so that we know key 0 - * won't be in the hash part, which simplifies things. */ - t->array_size = UPB_MAX(1, asize); - t->array_count = 0; - array_bytes = t->array_size * sizeof(upb_value); - t->array = upb_Arena_Malloc(a, array_bytes); - if (!t->array) { - return false; - } - memset(mutable_array(t), 0xff, array_bytes); - check(t); - return true; -} - -bool upb_inttable_init(upb_inttable* t, upb_Arena* a) { - return upb_inttable_sizedinit(t, 0, 4, a); -} - -bool upb_inttable_insert(upb_inttable* t, uintptr_t key, upb_value val, - upb_Arena* a) { - upb_tabval tabval; - tabval.val = val.val; - UPB_ASSERT( - upb_arrhas(tabval)); /* This will reject (uint64_t)-1. Fix this. */ - - if (key < t->array_size) { - UPB_ASSERT(!upb_arrhas(t->array[key])); - t->array_count++; - mutable_array(t)[key].val = val.val; - } else { - if (isfull(&t->t)) { - /* Need to resize the hash part, but we re-use the array part. */ - size_t i; - upb_table new_table; - - if (!init(&new_table, t->t.size_lg2 + 1, a)) { - return false; - } - - for (i = begin(&t->t); i < upb_table_size(&t->t); i = next(&t->t, i)) { - const upb_tabent* e = &t->t.entries[i]; - uint32_t hash; - upb_value v; - - _upb_value_setval(&v, e->val.val); - hash = upb_inthash(e->key); - insert(&new_table, intkey(e->key), e->key, v, hash, &inthash, &inteql); - } - - UPB_ASSERT(t->t.count == new_table.count); - - t->t = new_table; - } - insert(&t->t, intkey(key), key, val, upb_inthash(key), &inthash, &inteql); - } - check(t); - return true; -} - -bool upb_inttable_lookup(const upb_inttable* t, uintptr_t key, upb_value* v) { - const upb_tabval* table_v = inttable_val_const(t, key); - if (!table_v) return false; - if (v) _upb_value_setval(v, table_v->val); - return true; -} - -bool upb_inttable_replace(upb_inttable* t, uintptr_t key, upb_value val) { - upb_tabval* table_v = inttable_val(t, key); - if (!table_v) return false; - table_v->val = val.val; - return true; -} - -bool upb_inttable_remove(upb_inttable* t, uintptr_t key, upb_value* val) { - bool success; - if (key < t->array_size) { - if (upb_arrhas(t->array[key])) { - upb_tabval empty = UPB_TABVALUE_EMPTY_INIT; - t->array_count--; - if (val) { - _upb_value_setval(val, t->array[key].val); - } - mutable_array(t)[key] = empty; - success = true; - } else { - success = false; - } - } else { - success = rm(&t->t, intkey(key), val, NULL, upb_inthash(key), &inteql); - } - check(t); - return success; -} - -void upb_inttable_compact(upb_inttable* t, upb_Arena* a) { - /* A power-of-two histogram of the table keys. */ - size_t counts[UPB_MAXARRSIZE + 1] = {0}; - - /* The max key in each bucket. */ - uintptr_t max[UPB_MAXARRSIZE + 1] = {0}; - - { - intptr_t iter = UPB_INTTABLE_BEGIN; - uintptr_t key; - upb_value val; - while (upb_inttable_next(t, &key, &val, &iter)) { - int bucket = log2ceil(key); - max[bucket] = UPB_MAX(max[bucket], key); - counts[bucket]++; - } - } - - /* Find the largest power of two that satisfies the MIN_DENSITY - * definition (while actually having some keys). */ - size_t arr_count = upb_inttable_count(t); - int size_lg2; - upb_inttable new_t; - - for (size_lg2 = ARRAY_SIZE(counts) - 1; size_lg2 > 0; size_lg2--) { - if (counts[size_lg2] == 0) { - /* We can halve again without losing any entries. */ - continue; - } else if (arr_count >= (1 << size_lg2) * MIN_DENSITY) { - break; - } - - arr_count -= counts[size_lg2]; - } - - UPB_ASSERT(arr_count <= upb_inttable_count(t)); - - { - /* Insert all elements into new, perfectly-sized table. */ - size_t arr_size = max[size_lg2] + 1; /* +1 so arr[max] will fit. */ - size_t hash_count = upb_inttable_count(t) - arr_count; - size_t hash_size = hash_count ? (hash_count / MAX_LOAD) + 1 : 0; - int hashsize_lg2 = log2ceil(hash_size); - - upb_inttable_sizedinit(&new_t, arr_size, hashsize_lg2, a); - - { - intptr_t iter = UPB_INTTABLE_BEGIN; - uintptr_t key; - upb_value val; - while (upb_inttable_next(t, &key, &val, &iter)) { - upb_inttable_insert(&new_t, key, val, a); - } - } - - UPB_ASSERT(new_t.array_size == arr_size); - UPB_ASSERT(new_t.t.size_lg2 == hashsize_lg2); - } - *t = new_t; -} - -// Iteration. - -bool upb_inttable_next(const upb_inttable* t, uintptr_t* key, upb_value* val, - intptr_t* iter) { - intptr_t i = *iter; - if ((size_t)(i + 1) <= t->array_size) { - while ((size_t)++i < t->array_size) { - upb_tabval ent = t->array[i]; - if (upb_arrhas(ent)) { - *key = i; - *val = _upb_value_val(ent.val); - *iter = i; - return true; - } - } - i--; // Back up to exactly one position before the start of the table. - } - - size_t tab_idx = next(&t->t, i - t->array_size); - if (tab_idx < upb_table_size(&t->t)) { - upb_tabent* ent = &t->t.entries[tab_idx]; - *key = ent->key; - *val = _upb_value_val(ent->val.val); - *iter = tab_idx + t->array_size; - return true; - } - - return false; -} - -void upb_inttable_removeiter(upb_inttable* t, intptr_t* iter) { - intptr_t i = *iter; - if ((size_t)i < t->array_size) { - t->array_count--; - mutable_array(t)[i].val = -1; - } else { - upb_tabent* ent = &t->t.entries[i - t->array_size]; - upb_tabent* prev = NULL; - - // Linear search, not great. - upb_tabent* end = &t->t.entries[upb_table_size(&t->t)]; - for (upb_tabent* e = t->t.entries; e != end; e++) { - if (e->next == ent) { - prev = e; - break; - } - } - - if (prev) { - prev->next = ent->next; - } - - t->t.count--; - ent->key = 0; - ent->next = NULL; - } -} - -bool upb_strtable_next2(const upb_strtable* t, upb_StringView* key, - upb_value* val, intptr_t* iter) { - size_t tab_idx = next(&t->t, *iter); - if (tab_idx < upb_table_size(&t->t)) { - upb_tabent* ent = &t->t.entries[tab_idx]; - uint32_t len; - key->data = upb_tabstr(ent->key, &len); - key->size = len; - *val = _upb_value_val(ent->val.val); - *iter = tab_idx; - return true; - } - - return false; -} - -void upb_strtable_removeiter(upb_strtable* t, intptr_t* iter) { - intptr_t i = *iter; - upb_tabent* ent = &t->t.entries[i]; - upb_tabent* prev = NULL; - - // Linear search, not great. - upb_tabent* end = &t->t.entries[upb_table_size(&t->t)]; - for (upb_tabent* e = t->t.entries; e != end; e++) { - if (e->next == ent) { - prev = e; - break; - } - } - - if (prev) { - prev->next = ent->next; - } - - t->t.count--; - ent->key = 0; - ent->next = NULL; -} - -void upb_strtable_setentryvalue(upb_strtable* t, intptr_t iter, upb_value v) { - upb_tabent* ent = &t->t.entries[iter]; - ent->val.val = v.val; -} - #include <errno.h> #include <float.h> @@ -4867,183 +4041,6 @@ } -// Must be last. - -const char* upb_BufToUint64(const char* ptr, const char* end, uint64_t* val) { - uint64_t u64 = 0; - while (ptr < end) { - unsigned ch = *ptr - '0'; - if (ch >= 10) break; - if (u64 > UINT64_MAX / 10 || u64 * 10 > UINT64_MAX - ch) { - return NULL; // integer overflow - } - u64 *= 10; - u64 += ch; - ptr++; - } - - *val = u64; - return ptr; -} - -const char* upb_BufToInt64(const char* ptr, const char* end, int64_t* val, - bool* is_neg) { - bool neg = false; - uint64_t u64; - - if (ptr != end && *ptr == '-') { - ptr++; - neg = true; - } - - ptr = upb_BufToUint64(ptr, end, &u64); - if (!ptr || u64 > (uint64_t)INT64_MAX + neg) { - return NULL; // integer overflow - } - - *val = neg ? -u64 : u64; - if (is_neg) *is_neg = neg; - return ptr; -} - - -#include <float.h> -#include <stdlib.h> - -// Must be last. - -/* Miscellaneous utilities ****************************************************/ - -static void upb_FixLocale(char* p) { - /* printf() is dependent on locales; sadly there is no easy and portable way - * to avoid this. This little post-processing step will translate 1,2 -> 1.2 - * since JSON needs the latter. Arguably a hack, but it is simple and the - * alternatives are far more complicated, platform-dependent, and/or larger - * in code size. */ - for (; *p; p++) { - if (*p == ',') *p = '.'; - } -} - -void _upb_EncodeRoundTripDouble(double val, char* buf, size_t size) { - assert(size >= kUpb_RoundTripBufferSize); - snprintf(buf, size, "%.*g", DBL_DIG, val); - if (strtod(buf, NULL) != val) { - snprintf(buf, size, "%.*g", DBL_DIG + 2, val); - assert(strtod(buf, NULL) == val); - } - upb_FixLocale(buf); -} - -void _upb_EncodeRoundTripFloat(float val, char* buf, size_t size) { - assert(size >= kUpb_RoundTripBufferSize); - snprintf(buf, size, "%.*g", FLT_DIG, val); - if (strtof(buf, NULL) != val) { - snprintf(buf, size, "%.*g", FLT_DIG + 3, val); - assert(strtof(buf, NULL) == val); - } - upb_FixLocale(buf); -} - - -#include <stdlib.h> -#include <string.h> - -// Must be last. - -// Determine the locale-specific radix character by calling sprintf() to print -// the number 1.5, then stripping off the digits. As far as I can tell, this -// is the only portable, thread-safe way to get the C library to divulge the -// locale's radix character. No, localeconv() is NOT thread-safe. - -static int GetLocaleRadix(char *data, size_t capacity) { - char temp[16]; - const int size = snprintf(temp, sizeof(temp), "%.1f", 1.5); - UPB_ASSERT(temp[0] == '1'); - UPB_ASSERT(temp[size - 1] == '5'); - UPB_ASSERT(size < capacity); - temp[size - 1] = '\0'; - strcpy(data, temp + 1); - return size - 2; -} - -// Populates a string identical to *input except that the character pointed to -// by pos (which should be '.') is replaced with the locale-specific radix. - -static void LocalizeRadix(const char *input, const char *pos, char *output) { - const int len1 = pos - input; - - char radix[8]; - const int len2 = GetLocaleRadix(radix, sizeof(radix)); - - memcpy(output, input, len1); - memcpy(output + len1, radix, len2); - strcpy(output + len1 + len2, input + len1 + 1); -} - -double _upb_NoLocaleStrtod(const char *str, char **endptr) { - // We cannot simply set the locale to "C" temporarily with setlocale() - // as this is not thread-safe. Instead, we try to parse in the current - // locale first. If parsing stops at a '.' character, then this is a - // pretty good hint that we're actually in some other locale in which - // '.' is not the radix character. - - char *temp_endptr; - double result = strtod(str, &temp_endptr); - if (endptr != NULL) *endptr = temp_endptr; - if (*temp_endptr != '.') return result; - - // Parsing halted on a '.'. Perhaps we're in a different locale? Let's - // try to replace the '.' with a locale-specific radix character and - // try again. - - char localized[80]; - LocalizeRadix(str, temp_endptr, localized); - char *localized_endptr; - result = strtod(localized, &localized_endptr); - if ((localized_endptr - &localized[0]) > (temp_endptr - str)) { - // This attempt got further, so replacing the decimal must have helped. - // Update endptr to point at the right location. - if (endptr != NULL) { - // size_diff is non-zero if the localized radix has multiple bytes. - int size_diff = strlen(localized) - strlen(str); - *endptr = (char *)str + (localized_endptr - &localized[0] - size_diff); - } - } - - return result; -} - - -// Must be last. - -int upb_Unicode_ToUTF8(uint32_t cp, char* out) { - if (cp <= 0x7f) { - out[0] = cp; - return 1; - } - if (cp <= 0x07ff) { - out[0] = (cp >> 6) | 0xc0; - out[1] = (cp & 0x3f) | 0x80; - return 2; - } - if (cp <= 0xffff) { - out[0] = (cp >> 12) | 0xe0; - out[1] = ((cp >> 6) & 0x3f) | 0x80; - out[2] = (cp & 0x3f) | 0x80; - return 3; - } - if (cp <= 0x10ffff) { - out[0] = (cp >> 18) | 0xf0; - out[1] = ((cp >> 12) & 0x3f) | 0x80; - out[2] = ((cp >> 6) & 0x3f) | 0x80; - out[3] = (cp & 0x3f) | 0x80; - return 4; - } - return 0; -} - - #include <stdlib.h> // Must be last. @@ -7710,6 +6707,4453 @@ } +#include <assert.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + + +// Must be last. + +// A few fake field types for our tables. +enum { + kUpb_FakeFieldType_FieldNotFound = 0, + kUpb_FakeFieldType_MessageSetItem = 19, +}; + +// DecodeOp: an action to be performed for a wire-type/field-type combination. +enum { + // Special ops: we don't write data to regular fields for these. + kUpb_DecodeOp_UnknownField = -1, + kUpb_DecodeOp_MessageSetItem = -2, + + // Scalar-only ops. + kUpb_DecodeOp_Scalar1Byte = 0, + kUpb_DecodeOp_Scalar4Byte = 2, + kUpb_DecodeOp_Scalar8Byte = 3, + kUpb_DecodeOp_Enum = 1, + + // Scalar/repeated ops. + kUpb_DecodeOp_String = 4, + kUpb_DecodeOp_Bytes = 5, + kUpb_DecodeOp_SubMessage = 6, + + // Repeated-only ops (also see macros below). + kUpb_DecodeOp_PackedEnum = 13, +}; + +// For packed fields it is helpful to be able to recover the lg2 of the data +// size from the op. +#define OP_FIXPCK_LG2(n) (n + 5) /* n in [2, 3] => op in [7, 8] */ +#define OP_VARPCK_LG2(n) (n + 9) /* n in [0, 2, 3] => op in [9, 11, 12] */ + +typedef union { + bool bool_val; + uint32_t uint32_val; + uint64_t uint64_val; + uint32_t size; +} wireval; + +// Ideally these two functions should take the owning MiniTable pointer as a +// first argument, then we could just put them in mini_table/message.h as nice +// clean getters. But we don't have that so instead we gotta write these +// Frankenfunctions that take an array of subtables. +// TODO: Move these to mini_table/ anyway since there are other places +// that could use them. + +// Returns the MiniTable corresponding to a given MiniTableField +// from an array of MiniTableSubs. +static const upb_MiniTable* _upb_MiniTableSubs_MessageByField( + const upb_MiniTableSub* subs, const upb_MiniTableField* field) { + return upb_MiniTableSub_Message(subs[field->UPB_PRIVATE(submsg_index)]); +} + +// Returns the MiniTableEnum corresponding to a given MiniTableField +// from an array of MiniTableSub. +static const upb_MiniTableEnum* _upb_MiniTableSubs_EnumByField( + const upb_MiniTableSub* subs, const upb_MiniTableField* field) { + return upb_MiniTableSub_Enum(subs[field->UPB_PRIVATE(submsg_index)]); +} + +static const char* _upb_Decoder_DecodeMessage(upb_Decoder* d, const char* ptr, + upb_Message* msg, + const upb_MiniTable* layout); + +UPB_NORETURN static void* _upb_Decoder_ErrorJmp(upb_Decoder* d, + upb_DecodeStatus status) { + UPB_ASSERT(status != kUpb_DecodeStatus_Ok); + d->status = status; + UPB_LONGJMP(d->err, 1); +} + +const char* _upb_FastDecoder_ErrorJmp(upb_Decoder* d, int status) { + UPB_ASSERT(status != kUpb_DecodeStatus_Ok); + d->status = status; + UPB_LONGJMP(d->err, 1); + return NULL; +} + +static void _upb_Decoder_VerifyUtf8(upb_Decoder* d, const char* buf, int len) { + if (!_upb_Decoder_VerifyUtf8Inline(buf, len)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); + } +} + +static bool _upb_Decoder_Reserve(upb_Decoder* d, upb_Array* arr, size_t elem) { + bool need_realloc = + arr->UPB_PRIVATE(capacity) - arr->UPB_PRIVATE(size) < elem; + if (need_realloc && !UPB_PRIVATE(_upb_Array_Realloc)( + arr, arr->UPB_PRIVATE(size) + elem, &d->arena)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + } + return need_realloc; +} + +typedef struct { + const char* ptr; + uint64_t val; +} _upb_DecodeLongVarintReturn; + +UPB_NOINLINE +static _upb_DecodeLongVarintReturn _upb_Decoder_DecodeLongVarint( + const char* ptr, uint64_t val) { + _upb_DecodeLongVarintReturn ret = {NULL, 0}; + uint64_t byte; + int i; + for (i = 1; i < 10; i++) { + byte = (uint8_t)ptr[i]; + val += (byte - 1) << (i * 7); + if (!(byte & 0x80)) { + ret.ptr = ptr + i + 1; + ret.val = val; + return ret; + } + } + return ret; +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_DecodeVarint(upb_Decoder* d, const char* ptr, + uint64_t* val) { + uint64_t byte = (uint8_t)*ptr; + if (UPB_LIKELY((byte & 0x80) == 0)) { + *val = byte; + return ptr + 1; + } else { + _upb_DecodeLongVarintReturn res = _upb_Decoder_DecodeLongVarint(ptr, byte); + if (!res.ptr) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + *val = res.val; + return res.ptr; + } +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_DecodeTag(upb_Decoder* d, const char* ptr, + uint32_t* val) { + uint64_t byte = (uint8_t)*ptr; + if (UPB_LIKELY((byte & 0x80) == 0)) { + *val = byte; + return ptr + 1; + } else { + const char* start = ptr; + _upb_DecodeLongVarintReturn res = _upb_Decoder_DecodeLongVarint(ptr, byte); + if (!res.ptr || res.ptr - start > 5 || res.val > UINT32_MAX) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + } + *val = res.val; + return res.ptr; + } +} + +UPB_FORCEINLINE +static const char* upb_Decoder_DecodeSize(upb_Decoder* d, const char* ptr, + uint32_t* size) { + uint64_t size64; + ptr = _upb_Decoder_DecodeVarint(d, ptr, &size64); + if (size64 >= INT32_MAX || + !upb_EpsCopyInputStream_CheckSize(&d->input, ptr, (int)size64)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + } + *size = size64; + return ptr; +} + +static void _upb_Decoder_MungeInt32(wireval* val) { + if (!UPB_PRIVATE(_upb_IsLittleEndian)()) { + /* The next stage will memcpy(dst, &val, 4) */ + val->uint32_val = val->uint64_val; + } +} + +static void _upb_Decoder_Munge(int type, wireval* val) { + switch (type) { + case kUpb_FieldType_Bool: + val->bool_val = val->uint64_val != 0; + break; + case kUpb_FieldType_SInt32: { + uint32_t n = val->uint64_val; + val->uint32_val = (n >> 1) ^ -(int32_t)(n & 1); + break; + } + case kUpb_FieldType_SInt64: { + uint64_t n = val->uint64_val; + val->uint64_val = (n >> 1) ^ -(int64_t)(n & 1); + break; + } + case kUpb_FieldType_Int32: + case kUpb_FieldType_UInt32: + case kUpb_FieldType_Enum: + _upb_Decoder_MungeInt32(val); + break; + } +} + +static upb_Message* _upb_Decoder_NewSubMessage(upb_Decoder* d, + const upb_MiniTableSub* subs, + const upb_MiniTableField* field, + upb_TaggedMessagePtr* target) { + const upb_MiniTable* subl = _upb_MiniTableSubs_MessageByField(subs, field); + UPB_ASSERT(subl); + upb_Message* msg = _upb_Message_New(subl, &d->arena); + if (!msg) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + + // Extensions should not be unlinked. A message extension should not be + // registered until its sub-message type is available to be linked. + bool is_empty = UPB_PRIVATE(_upb_MiniTable_IsEmpty)(subl); + bool is_extension = field->UPB_PRIVATE(mode) & kUpb_LabelFlags_IsExtension; + UPB_ASSERT(!(is_empty && is_extension)); + + if (is_empty && !(d->options & kUpb_DecodeOption_ExperimentalAllowUnlinked)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_UnlinkedSubMessage); + } + + upb_TaggedMessagePtr tagged = + UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(msg, is_empty); + memcpy(target, &tagged, sizeof(tagged)); + return msg; +} + +static upb_Message* _upb_Decoder_ReuseSubMessage( + upb_Decoder* d, const upb_MiniTableSub* subs, + const upb_MiniTableField* field, upb_TaggedMessagePtr* target) { + upb_TaggedMessagePtr tagged = *target; + const upb_MiniTable* subl = _upb_MiniTableSubs_MessageByField(subs, field); + UPB_ASSERT(subl); + if (!upb_TaggedMessagePtr_IsEmpty(tagged) || + UPB_PRIVATE(_upb_MiniTable_IsEmpty)(subl)) { + return UPB_PRIVATE(_upb_TaggedMessagePtr_GetMessage)(tagged); + } + + // We found an empty message from a previous parse that was performed before + // this field was linked. But it is linked now, so we want to allocate a new + // message of the correct type and promote data into it before continuing. + upb_Message* existing = + UPB_PRIVATE(_upb_TaggedMessagePtr_GetEmptyMessage)(tagged); + upb_Message* promoted = _upb_Decoder_NewSubMessage(d, subs, field, target); + size_t size; + const char* unknown = upb_Message_GetUnknown(existing, &size); + upb_DecodeStatus status = upb_Decode(unknown, size, promoted, subl, d->extreg, + d->options, &d->arena); + if (status != kUpb_DecodeStatus_Ok) _upb_Decoder_ErrorJmp(d, status); + return promoted; +} + +static const char* _upb_Decoder_ReadString(upb_Decoder* d, const char* ptr, + int size, upb_StringView* str) { + const char* str_ptr = ptr; + ptr = upb_EpsCopyInputStream_ReadString(&d->input, &str_ptr, size, &d->arena); + if (!ptr) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + str->data = str_ptr; + str->size = size; + return ptr; +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_RecurseSubMessage(upb_Decoder* d, + const char* ptr, + upb_Message* submsg, + const upb_MiniTable* subl, + uint32_t expected_end_group) { + if (--d->depth < 0) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_MaxDepthExceeded); + } + ptr = _upb_Decoder_DecodeMessage(d, ptr, submsg, subl); + d->depth++; + if (d->end_group != expected_end_group) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + } + return ptr; +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_DecodeSubMessage( + upb_Decoder* d, const char* ptr, upb_Message* submsg, + const upb_MiniTableSub* subs, const upb_MiniTableField* field, int size) { + int saved_delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, size); + const upb_MiniTable* subl = _upb_MiniTableSubs_MessageByField(subs, field); + UPB_ASSERT(subl); + ptr = _upb_Decoder_RecurseSubMessage(d, ptr, submsg, subl, DECODE_NOGROUP); + upb_EpsCopyInputStream_PopLimit(&d->input, ptr, saved_delta); + return ptr; +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_DecodeGroup(upb_Decoder* d, const char* ptr, + upb_Message* submsg, + const upb_MiniTable* subl, + uint32_t number) { + if (_upb_Decoder_IsDone(d, &ptr)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + } + ptr = _upb_Decoder_RecurseSubMessage(d, ptr, submsg, subl, number); + d->end_group = DECODE_NOGROUP; + return ptr; +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_DecodeUnknownGroup(upb_Decoder* d, + const char* ptr, + uint32_t number) { + return _upb_Decoder_DecodeGroup(d, ptr, NULL, NULL, number); +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_DecodeKnownGroup( + upb_Decoder* d, const char* ptr, upb_Message* submsg, + const upb_MiniTableSub* subs, const upb_MiniTableField* field) { + const upb_MiniTable* subl = _upb_MiniTableSubs_MessageByField(subs, field); + UPB_ASSERT(subl); + return _upb_Decoder_DecodeGroup(d, ptr, submsg, subl, + field->UPB_PRIVATE(number)); +} + +static char* upb_Decoder_EncodeVarint32(uint32_t val, char* ptr) { + do { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + *(ptr++) = byte; + } while (val); + return ptr; +} + +static void _upb_Decoder_AddUnknownVarints(upb_Decoder* d, upb_Message* msg, + uint32_t val1, uint32_t val2) { + char buf[20]; + char* end = buf; + end = upb_Decoder_EncodeVarint32(val1, end); + end = upb_Decoder_EncodeVarint32(val2, end); + + if (!UPB_PRIVATE(_upb_Message_AddUnknown)(msg, buf, end - buf, &d->arena)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + } +} + +UPB_FORCEINLINE +static bool _upb_Decoder_CheckEnum(upb_Decoder* d, const char* ptr, + upb_Message* msg, const upb_MiniTableEnum* e, + const upb_MiniTableField* field, + wireval* val) { + const uint32_t v = val->uint32_val; + + if (UPB_LIKELY(upb_MiniTableEnum_CheckValue(e, v))) return true; + + // Unrecognized enum goes into unknown fields. + // For packed fields the tag could be arbitrarily far in the past, + // so we just re-encode the tag and value here. + const uint32_t tag = + ((uint32_t)field->UPB_PRIVATE(number) << 3) | kUpb_WireType_Varint; + upb_Message* unknown_msg = + field->UPB_PRIVATE(mode) & kUpb_LabelFlags_IsExtension ? d->unknown_msg + : msg; + _upb_Decoder_AddUnknownVarints(d, unknown_msg, tag, v); + return false; +} + +UPB_NOINLINE +static const char* _upb_Decoder_DecodeEnumArray(upb_Decoder* d, const char* ptr, + upb_Message* msg, + upb_Array* arr, + const upb_MiniTableSub* subs, + const upb_MiniTableField* field, + wireval* val) { + const upb_MiniTableEnum* e = _upb_MiniTableSubs_EnumByField(subs, field); + if (!_upb_Decoder_CheckEnum(d, ptr, msg, e, field, val)) return ptr; + void* mem = UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) * 4, void); + arr->UPB_PRIVATE(size)++; + memcpy(mem, val, 4); + return ptr; +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_DecodeFixedPacked( + upb_Decoder* d, const char* ptr, upb_Array* arr, wireval* val, + const upb_MiniTableField* field, int lg2) { + int mask = (1 << lg2) - 1; + size_t count = val->size >> lg2; + if ((val->size & mask) != 0) { + // Length isn't a round multiple of elem size. + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + } + _upb_Decoder_Reserve(d, arr, count); + void* mem = + UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) << lg2, void); + arr->UPB_PRIVATE(size) += count; + // Note: if/when the decoder supports multi-buffer input, we will need to + // handle buffer seams here. + if (UPB_PRIVATE(_upb_IsLittleEndian)()) { + ptr = upb_EpsCopyInputStream_Copy(&d->input, ptr, mem, val->size); + } else { + int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, val->size); + char* dst = mem; + while (!_upb_Decoder_IsDone(d, &ptr)) { + if (lg2 == 2) { + ptr = upb_WireReader_ReadFixed32(ptr, dst); + dst += 4; + } else { + UPB_ASSERT(lg2 == 3); + ptr = upb_WireReader_ReadFixed64(ptr, dst); + dst += 8; + } + } + upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta); + } + + return ptr; +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_DecodeVarintPacked( + upb_Decoder* d, const char* ptr, upb_Array* arr, wireval* val, + const upb_MiniTableField* field, int lg2) { + int scale = 1 << lg2; + int saved_limit = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, val->size); + char* out = + UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) << lg2, void); + while (!_upb_Decoder_IsDone(d, &ptr)) { + wireval elem; + ptr = _upb_Decoder_DecodeVarint(d, ptr, &elem.uint64_val); + _upb_Decoder_Munge(field->UPB_PRIVATE(descriptortype), &elem); + if (_upb_Decoder_Reserve(d, arr, 1)) { + out = + UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) << lg2, void); + } + arr->UPB_PRIVATE(size)++; + memcpy(out, &elem, scale); + out += scale; + } + upb_EpsCopyInputStream_PopLimit(&d->input, ptr, saved_limit); + return ptr; +} + +UPB_NOINLINE +static const char* _upb_Decoder_DecodeEnumPacked( + upb_Decoder* d, const char* ptr, upb_Message* msg, upb_Array* arr, + const upb_MiniTableSub* subs, const upb_MiniTableField* field, + wireval* val) { + const upb_MiniTableEnum* e = _upb_MiniTableSubs_EnumByField(subs, field); + int saved_limit = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, val->size); + char* out = UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) * 4, void); + while (!_upb_Decoder_IsDone(d, &ptr)) { + wireval elem; + ptr = _upb_Decoder_DecodeVarint(d, ptr, &elem.uint64_val); + _upb_Decoder_MungeInt32(&elem); + if (!_upb_Decoder_CheckEnum(d, ptr, msg, e, field, &elem)) { + continue; + } + if (_upb_Decoder_Reserve(d, arr, 1)) { + out = UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) * 4, void); + } + arr->UPB_PRIVATE(size)++; + memcpy(out, &elem, 4); + out += 4; + } + upb_EpsCopyInputStream_PopLimit(&d->input, ptr, saved_limit); + return ptr; +} + +upb_Array* _upb_Decoder_CreateArray(upb_Decoder* d, + const upb_MiniTableField* field) { + const upb_FieldType field_type = field->UPB_PRIVATE(descriptortype); + const size_t lg2 = UPB_PRIVATE(_upb_FieldType_SizeLg2)(field_type); + upb_Array* ret = UPB_PRIVATE(_upb_Array_New)(&d->arena, 4, lg2); + if (!ret) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + return ret; +} + +static const char* _upb_Decoder_DecodeToArray(upb_Decoder* d, const char* ptr, + upb_Message* msg, + const upb_MiniTableSub* subs, + const upb_MiniTableField* field, + wireval* val, int op) { + upb_Array** arrp = UPB_PTR_AT(msg, field->UPB_PRIVATE(offset), void); + upb_Array* arr = *arrp; + void* mem; + + if (arr) { + _upb_Decoder_Reserve(d, arr, 1); + } else { + arr = _upb_Decoder_CreateArray(d, field); + *arrp = arr; + } + + switch (op) { + case kUpb_DecodeOp_Scalar1Byte: + case kUpb_DecodeOp_Scalar4Byte: + case kUpb_DecodeOp_Scalar8Byte: + /* Append scalar value. */ + mem = UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) << op, void); + arr->UPB_PRIVATE(size)++; + memcpy(mem, val, 1 << op); + return ptr; + case kUpb_DecodeOp_String: + _upb_Decoder_VerifyUtf8(d, ptr, val->size); + /* Fallthrough. */ + case kUpb_DecodeOp_Bytes: { + /* Append bytes. */ + upb_StringView* str = + (upb_StringView*)_upb_array_ptr(arr) + arr->UPB_PRIVATE(size); + arr->UPB_PRIVATE(size)++; + return _upb_Decoder_ReadString(d, ptr, val->size, str); + } + case kUpb_DecodeOp_SubMessage: { + /* Append submessage / group. */ + upb_TaggedMessagePtr* target = UPB_PTR_AT( + _upb_array_ptr(arr), arr->UPB_PRIVATE(size) * sizeof(void*), + upb_TaggedMessagePtr); + upb_Message* submsg = _upb_Decoder_NewSubMessage(d, subs, field, target); + arr->UPB_PRIVATE(size)++; + if (UPB_UNLIKELY(field->UPB_PRIVATE(descriptortype) == + kUpb_FieldType_Group)) { + return _upb_Decoder_DecodeKnownGroup(d, ptr, submsg, subs, field); + } else { + return _upb_Decoder_DecodeSubMessage(d, ptr, submsg, subs, field, + val->size); + } + } + case OP_FIXPCK_LG2(2): + case OP_FIXPCK_LG2(3): + return _upb_Decoder_DecodeFixedPacked(d, ptr, arr, val, field, + op - OP_FIXPCK_LG2(0)); + case OP_VARPCK_LG2(0): + case OP_VARPCK_LG2(2): + case OP_VARPCK_LG2(3): + return _upb_Decoder_DecodeVarintPacked(d, ptr, arr, val, field, + op - OP_VARPCK_LG2(0)); + case kUpb_DecodeOp_Enum: + return _upb_Decoder_DecodeEnumArray(d, ptr, msg, arr, subs, field, val); + case kUpb_DecodeOp_PackedEnum: + return _upb_Decoder_DecodeEnumPacked(d, ptr, msg, arr, subs, field, val); + default: + UPB_UNREACHABLE(); + } +} + +upb_Map* _upb_Decoder_CreateMap(upb_Decoder* d, const upb_MiniTable* entry) { + /* Maps descriptor type -> upb map size. */ + static const uint8_t kSizeInMap[] = { + [0] = -1, // invalid descriptor type */ + [kUpb_FieldType_Double] = 8, + [kUpb_FieldType_Float] = 4, + [kUpb_FieldType_Int64] = 8, + [kUpb_FieldType_UInt64] = 8, + [kUpb_FieldType_Int32] = 4, + [kUpb_FieldType_Fixed64] = 8, + [kUpb_FieldType_Fixed32] = 4, + [kUpb_FieldType_Bool] = 1, + [kUpb_FieldType_String] = UPB_MAPTYPE_STRING, + [kUpb_FieldType_Group] = sizeof(void*), + [kUpb_FieldType_Message] = sizeof(void*), + [kUpb_FieldType_Bytes] = UPB_MAPTYPE_STRING, + [kUpb_FieldType_UInt32] = 4, + [kUpb_FieldType_Enum] = 4, + [kUpb_FieldType_SFixed32] = 4, + [kUpb_FieldType_SFixed64] = 8, + [kUpb_FieldType_SInt32] = 4, + [kUpb_FieldType_SInt64] = 8, + }; + + const upb_MiniTableField* key_field = &entry->UPB_PRIVATE(fields)[0]; + const upb_MiniTableField* val_field = &entry->UPB_PRIVATE(fields)[1]; + char key_size = kSizeInMap[key_field->UPB_PRIVATE(descriptortype)]; + char val_size = kSizeInMap[val_field->UPB_PRIVATE(descriptortype)]; + UPB_ASSERT(key_field->UPB_PRIVATE(offset) == offsetof(upb_MapEntryData, k)); + UPB_ASSERT(val_field->UPB_PRIVATE(offset) == offsetof(upb_MapEntryData, v)); + upb_Map* ret = _upb_Map_New(&d->arena, key_size, val_size); + if (!ret) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + return ret; +} + +static const char* _upb_Decoder_DecodeToMap(upb_Decoder* d, const char* ptr, + upb_Message* msg, + const upb_MiniTableSub* subs, + const upb_MiniTableField* field, + wireval* val) { + upb_Map** map_p = UPB_PTR_AT(msg, field->UPB_PRIVATE(offset), upb_Map*); + upb_Map* map = *map_p; + upb_MapEntry ent; + UPB_ASSERT(upb_MiniTableField_Type(field) == kUpb_FieldType_Message); + const upb_MiniTable* entry = _upb_MiniTableSubs_MessageByField(subs, field); + + UPB_ASSERT(entry); + UPB_ASSERT(entry->UPB_PRIVATE(field_count) == 2); + UPB_ASSERT(upb_MiniTableField_IsScalar(&entry->UPB_PRIVATE(fields)[0])); + UPB_ASSERT(upb_MiniTableField_IsScalar(&entry->UPB_PRIVATE(fields)[1])); + + if (!map) { + map = _upb_Decoder_CreateMap(d, entry); + *map_p = map; + } + + // Parse map entry. + memset(&ent, 0, sizeof(ent)); + + if (entry->UPB_PRIVATE(fields)[1].UPB_PRIVATE(descriptortype) == + kUpb_FieldType_Message || + entry->UPB_PRIVATE(fields)[1].UPB_PRIVATE(descriptortype) == + kUpb_FieldType_Group) { + // Create proactively to handle the case where it doesn't appear. + upb_TaggedMessagePtr msg; + _upb_Decoder_NewSubMessage(d, entry->UPB_PRIVATE(subs), + &entry->UPB_PRIVATE(fields)[1], &msg); + ent.data.v.val = upb_value_uintptr(msg); + } + + ptr = _upb_Decoder_DecodeSubMessage(d, ptr, (upb_Message*)&ent.data, subs, + field, val->size); + // check if ent had any unknown fields + size_t size; + upb_Message_GetUnknown((upb_Message*)&ent.data, &size); + if (size != 0) { + char* buf; + size_t size; + uint32_t tag = + ((uint32_t)field->UPB_PRIVATE(number) << 3) | kUpb_WireType_Delimited; + upb_EncodeStatus status = + upb_Encode((upb_Message*)&ent.data, entry, 0, &d->arena, &buf, &size); + if (status != kUpb_EncodeStatus_Ok) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + } + _upb_Decoder_AddUnknownVarints(d, msg, tag, size); + if (!UPB_PRIVATE(_upb_Message_AddUnknown)(msg, buf, size, &d->arena)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + } + } else { + if (_upb_Map_Insert(map, &ent.data.k, map->key_size, &ent.data.v, + map->val_size, + &d->arena) == kUpb_MapInsertStatus_OutOfMemory) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + } + } + return ptr; +} + +static const char* _upb_Decoder_DecodeToSubMessage( + upb_Decoder* d, const char* ptr, upb_Message* msg, + const upb_MiniTableSub* subs, const upb_MiniTableField* field, wireval* val, + int op) { + void* mem = UPB_PTR_AT(msg, field->UPB_PRIVATE(offset), void); + int type = field->UPB_PRIVATE(descriptortype); + + if (UPB_UNLIKELY(op == kUpb_DecodeOp_Enum) && + !_upb_Decoder_CheckEnum(d, ptr, msg, + _upb_MiniTableSubs_EnumByField(subs, field), + field, val)) { + return ptr; + } + + /* Set presence if necessary. */ + if (field->presence > 0) { + UPB_PRIVATE(_upb_Message_SetHasbit)(msg, field); + } else if (field->presence < 0) { + /* Oneof case */ + uint32_t* oneof_case = UPB_PRIVATE(_upb_Message_OneofCasePtr)(msg, field); + if (op == kUpb_DecodeOp_SubMessage && + *oneof_case != field->UPB_PRIVATE(number)) { + memset(mem, 0, sizeof(void*)); + } + *oneof_case = field->UPB_PRIVATE(number); + } + + /* Store into message. */ + switch (op) { + case kUpb_DecodeOp_SubMessage: { + upb_TaggedMessagePtr* submsgp = mem; + upb_Message* submsg; + if (*submsgp) { + submsg = _upb_Decoder_ReuseSubMessage(d, subs, field, submsgp); + } else { + submsg = _upb_Decoder_NewSubMessage(d, subs, field, submsgp); + } + if (UPB_UNLIKELY(type == kUpb_FieldType_Group)) { + ptr = _upb_Decoder_DecodeKnownGroup(d, ptr, submsg, subs, field); + } else { + ptr = _upb_Decoder_DecodeSubMessage(d, ptr, submsg, subs, field, + val->size); + } + break; + } + case kUpb_DecodeOp_String: + _upb_Decoder_VerifyUtf8(d, ptr, val->size); + /* Fallthrough. */ + case kUpb_DecodeOp_Bytes: + return _upb_Decoder_ReadString(d, ptr, val->size, mem); + case kUpb_DecodeOp_Scalar8Byte: + memcpy(mem, val, 8); + break; + case kUpb_DecodeOp_Enum: + case kUpb_DecodeOp_Scalar4Byte: + memcpy(mem, val, 4); + break; + case kUpb_DecodeOp_Scalar1Byte: + memcpy(mem, val, 1); + break; + default: + UPB_UNREACHABLE(); + } + + return ptr; +} + +UPB_NOINLINE +const char* _upb_Decoder_CheckRequired(upb_Decoder* d, const char* ptr, + const upb_Message* msg, + const upb_MiniTable* m) { + UPB_ASSERT(m->UPB_PRIVATE(required_count)); + if (UPB_LIKELY((d->options & kUpb_DecodeOption_CheckRequired) == 0)) { + return ptr; + } + uint64_t msg_head; + memcpy(&msg_head, msg, 8); + msg_head = UPB_PRIVATE(_upb_BigEndian64)(msg_head); + if (UPB_PRIVATE(_upb_MiniTable_RequiredMask)(m) & ~msg_head) { + d->missing_required = true; + } + return ptr; +} + +UPB_FORCEINLINE +static bool _upb_Decoder_TryFastDispatch(upb_Decoder* d, const char** ptr, + upb_Message* msg, + const upb_MiniTable* m) { +#if UPB_FASTTABLE + if (m && m->UPB_PRIVATE(table_mask) != (unsigned char)-1) { + uint16_t tag = _upb_FastDecoder_LoadTag(*ptr); + intptr_t table = decode_totable(m); + *ptr = _upb_FastDecoder_TagDispatch(d, *ptr, msg, table, 0, tag); + return true; + } +#endif + return false; +} + +static const char* upb_Decoder_SkipField(upb_Decoder* d, const char* ptr, + uint32_t tag) { + int field_number = tag >> 3; + int wire_type = tag & 7; + switch (wire_type) { + case kUpb_WireType_Varint: { + uint64_t val; + return _upb_Decoder_DecodeVarint(d, ptr, &val); + } + case kUpb_WireType_64Bit: + return ptr + 8; + case kUpb_WireType_32Bit: + return ptr + 4; + case kUpb_WireType_Delimited: { + uint32_t size; + ptr = upb_Decoder_DecodeSize(d, ptr, &size); + return ptr + size; + } + case kUpb_WireType_StartGroup: + return _upb_Decoder_DecodeUnknownGroup(d, ptr, field_number); + default: + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + } +} + +enum { + kStartItemTag = ((kUpb_MsgSet_Item << 3) | kUpb_WireType_StartGroup), + kEndItemTag = ((kUpb_MsgSet_Item << 3) | kUpb_WireType_EndGroup), + kTypeIdTag = ((kUpb_MsgSet_TypeId << 3) | kUpb_WireType_Varint), + kMessageTag = ((kUpb_MsgSet_Message << 3) | kUpb_WireType_Delimited), +}; + +static void upb_Decoder_AddKnownMessageSetItem( + upb_Decoder* d, upb_Message* msg, const upb_MiniTableExtension* item_mt, + const char* data, uint32_t size) { + upb_Extension* ext = + _upb_Message_GetOrCreateExtension(msg, item_mt, &d->arena); + if (UPB_UNLIKELY(!ext)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + } + upb_Message* submsg = _upb_Decoder_NewSubMessage( + d, &ext->ext->UPB_PRIVATE(sub), upb_MiniTableExtension_AsField(ext->ext), + (upb_TaggedMessagePtr*)&ext->data); + upb_DecodeStatus status = upb_Decode( + data, size, submsg, upb_MiniTableExtension_GetSubMessage(item_mt), + d->extreg, d->options, &d->arena); + if (status != kUpb_DecodeStatus_Ok) _upb_Decoder_ErrorJmp(d, status); +} + +static void upb_Decoder_AddUnknownMessageSetItem(upb_Decoder* d, + upb_Message* msg, + uint32_t type_id, + const char* message_data, + uint32_t message_size) { + char buf[60]; + char* ptr = buf; + ptr = upb_Decoder_EncodeVarint32(kStartItemTag, ptr); + ptr = upb_Decoder_EncodeVarint32(kTypeIdTag, ptr); + ptr = upb_Decoder_EncodeVarint32(type_id, ptr); + ptr = upb_Decoder_EncodeVarint32(kMessageTag, ptr); + ptr = upb_Decoder_EncodeVarint32(message_size, ptr); + char* split = ptr; + + ptr = upb_Decoder_EncodeVarint32(kEndItemTag, ptr); + char* end = ptr; + + if (!UPB_PRIVATE(_upb_Message_AddUnknown)(msg, buf, split - buf, &d->arena) || + !UPB_PRIVATE(_upb_Message_AddUnknown)(msg, message_data, message_size, + &d->arena) || + !UPB_PRIVATE(_upb_Message_AddUnknown)(msg, split, end - split, + &d->arena)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + } +} + +static void upb_Decoder_AddMessageSetItem(upb_Decoder* d, upb_Message* msg, + const upb_MiniTable* t, + uint32_t type_id, const char* data, + uint32_t size) { + const upb_MiniTableExtension* item_mt = + upb_ExtensionRegistry_Lookup(d->extreg, t, type_id); + if (item_mt) { + upb_Decoder_AddKnownMessageSetItem(d, msg, item_mt, data, size); + } else { + upb_Decoder_AddUnknownMessageSetItem(d, msg, type_id, data, size); + } +} + +static const char* upb_Decoder_DecodeMessageSetItem( + upb_Decoder* d, const char* ptr, upb_Message* msg, + const upb_MiniTable* layout) { + uint32_t type_id = 0; + upb_StringView preserved = {NULL, 0}; + typedef enum { + kUpb_HaveId = 1 << 0, + kUpb_HavePayload = 1 << 1, + } StateMask; + StateMask state_mask = 0; + while (!_upb_Decoder_IsDone(d, &ptr)) { + uint32_t tag; + ptr = _upb_Decoder_DecodeTag(d, ptr, &tag); + switch (tag) { + case kEndItemTag: + return ptr; + case kTypeIdTag: { + uint64_t tmp; + ptr = _upb_Decoder_DecodeVarint(d, ptr, &tmp); + if (state_mask & kUpb_HaveId) break; // Ignore dup. + state_mask |= kUpb_HaveId; + type_id = tmp; + if (state_mask & kUpb_HavePayload) { + upb_Decoder_AddMessageSetItem(d, msg, layout, type_id, preserved.data, + preserved.size); + } + break; + } + case kMessageTag: { + uint32_t size; + ptr = upb_Decoder_DecodeSize(d, ptr, &size); + const char* data = ptr; + ptr += size; + if (state_mask & kUpb_HavePayload) break; // Ignore dup. + state_mask |= kUpb_HavePayload; + if (state_mask & kUpb_HaveId) { + upb_Decoder_AddMessageSetItem(d, msg, layout, type_id, data, size); + } else { + // Out of order, we must preserve the payload. + preserved.data = data; + preserved.size = size; + } + break; + } + default: + // We do not preserve unexpected fields inside a message set item. + ptr = upb_Decoder_SkipField(d, ptr, tag); + break; + } + } + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); +} + +static const upb_MiniTableField* _upb_Decoder_FindField(upb_Decoder* d, + const upb_MiniTable* t, + uint32_t field_number, + int* last_field_index) { + static upb_MiniTableField none = { + 0, 0, 0, 0, kUpb_FakeFieldType_FieldNotFound, 0}; + if (t == NULL) return &none; + + size_t idx = ((size_t)field_number) - 1; // 0 wraps to SIZE_MAX + if (idx < t->UPB_PRIVATE(dense_below)) { + /* Fastest case: index into dense fields. */ + goto found; + } + + if (t->UPB_PRIVATE(dense_below) < t->UPB_PRIVATE(field_count)) { + /* Linear search non-dense fields. Resume scanning from last_field_index + * since fields are usually in order. */ + size_t last = *last_field_index; + for (idx = last; idx < t->UPB_PRIVATE(field_count); idx++) { + if (t->UPB_PRIVATE(fields)[idx].UPB_PRIVATE(number) == field_number) { + goto found; + } + } + + for (idx = t->UPB_PRIVATE(dense_below); idx < last; idx++) { + if (t->UPB_PRIVATE(fields)[idx].UPB_PRIVATE(number) == field_number) { + goto found; + } + } + } + + if (d->extreg) { + switch (t->UPB_PRIVATE(ext)) { + case kUpb_ExtMode_Extendable: { + const upb_MiniTableExtension* ext = + upb_ExtensionRegistry_Lookup(d->extreg, t, field_number); + if (ext) return upb_MiniTableExtension_AsField(ext); + break; + } + case kUpb_ExtMode_IsMessageSet: + if (field_number == kUpb_MsgSet_Item) { + static upb_MiniTableField item = { + 0, 0, 0, 0, kUpb_FakeFieldType_MessageSetItem, 0}; + return &item; + } + break; + } + } + + return &none; /* Unknown field. */ + +found: + UPB_ASSERT(t->UPB_PRIVATE(fields)[idx].UPB_PRIVATE(number) == field_number); + *last_field_index = idx; + return &t->UPB_PRIVATE(fields)[idx]; +} + +int _upb_Decoder_GetVarintOp(const upb_MiniTableField* field) { + static const int8_t kVarintOps[] = { + [kUpb_FakeFieldType_FieldNotFound] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Double] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Float] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Int64] = kUpb_DecodeOp_Scalar8Byte, + [kUpb_FieldType_UInt64] = kUpb_DecodeOp_Scalar8Byte, + [kUpb_FieldType_Int32] = kUpb_DecodeOp_Scalar4Byte, + [kUpb_FieldType_Fixed64] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Fixed32] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Bool] = kUpb_DecodeOp_Scalar1Byte, + [kUpb_FieldType_String] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Group] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Message] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Bytes] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_UInt32] = kUpb_DecodeOp_Scalar4Byte, + [kUpb_FieldType_Enum] = kUpb_DecodeOp_Enum, + [kUpb_FieldType_SFixed32] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_SFixed64] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_SInt32] = kUpb_DecodeOp_Scalar4Byte, + [kUpb_FieldType_SInt64] = kUpb_DecodeOp_Scalar8Byte, + [kUpb_FakeFieldType_MessageSetItem] = kUpb_DecodeOp_UnknownField, + }; + + return kVarintOps[field->UPB_PRIVATE(descriptortype)]; +} + +UPB_FORCEINLINE +static void _upb_Decoder_CheckUnlinked(upb_Decoder* d, const upb_MiniTable* mt, + const upb_MiniTableField* field, + int* op) { + // If sub-message is not linked, treat as unknown. + if (field->UPB_PRIVATE(mode) & kUpb_LabelFlags_IsExtension) return; + const upb_MiniTable* mt_sub = + _upb_MiniTableSubs_MessageByField(mt->UPB_PRIVATE(subs), field); + if ((d->options & kUpb_DecodeOption_ExperimentalAllowUnlinked) || + !UPB_PRIVATE(_upb_MiniTable_IsEmpty)(mt_sub)) { + return; + } +#ifndef NDEBUG + const upb_MiniTableField* oneof = upb_MiniTable_GetOneof(mt, field); + if (oneof) { + // All other members of the oneof must be message fields that are also + // unlinked. + do { + UPB_ASSERT(upb_MiniTableField_CType(oneof) == kUpb_CType_Message); + const upb_MiniTableSub* oneof_sub = + &mt->UPB_PRIVATE(subs)[oneof->UPB_PRIVATE(submsg_index)]; + UPB_ASSERT(!oneof_sub); + } while (upb_MiniTable_NextOneofField(mt, &oneof)); + } +#endif // NDEBUG + *op = kUpb_DecodeOp_UnknownField; +} + +int _upb_Decoder_GetDelimitedOp(upb_Decoder* d, const upb_MiniTable* mt, + const upb_MiniTableField* field) { + enum { kRepeatedBase = 19 }; + + static const int8_t kDelimitedOps[] = { + /* For non-repeated field type. */ + [kUpb_FakeFieldType_FieldNotFound] = + kUpb_DecodeOp_UnknownField, // Field not found. + [kUpb_FieldType_Double] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Float] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Int64] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_UInt64] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Int32] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Fixed64] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Fixed32] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Bool] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_String] = kUpb_DecodeOp_String, + [kUpb_FieldType_Group] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Message] = kUpb_DecodeOp_SubMessage, + [kUpb_FieldType_Bytes] = kUpb_DecodeOp_Bytes, + [kUpb_FieldType_UInt32] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_Enum] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_SFixed32] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_SFixed64] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_SInt32] = kUpb_DecodeOp_UnknownField, + [kUpb_FieldType_SInt64] = kUpb_DecodeOp_UnknownField, + [kUpb_FakeFieldType_MessageSetItem] = kUpb_DecodeOp_UnknownField, + // For repeated field type. */ + [kRepeatedBase + kUpb_FieldType_Double] = OP_FIXPCK_LG2(3), + [kRepeatedBase + kUpb_FieldType_Float] = OP_FIXPCK_LG2(2), + [kRepeatedBase + kUpb_FieldType_Int64] = OP_VARPCK_LG2(3), + [kRepeatedBase + kUpb_FieldType_UInt64] = OP_VARPCK_LG2(3), + [kRepeatedBase + kUpb_FieldType_Int32] = OP_VARPCK_LG2(2), + [kRepeatedBase + kUpb_FieldType_Fixed64] = OP_FIXPCK_LG2(3), + [kRepeatedBase + kUpb_FieldType_Fixed32] = OP_FIXPCK_LG2(2), + [kRepeatedBase + kUpb_FieldType_Bool] = OP_VARPCK_LG2(0), + [kRepeatedBase + kUpb_FieldType_String] = kUpb_DecodeOp_String, + [kRepeatedBase + kUpb_FieldType_Group] = kUpb_DecodeOp_SubMessage, + [kRepeatedBase + kUpb_FieldType_Message] = kUpb_DecodeOp_SubMessage, + [kRepeatedBase + kUpb_FieldType_Bytes] = kUpb_DecodeOp_Bytes, + [kRepeatedBase + kUpb_FieldType_UInt32] = OP_VARPCK_LG2(2), + [kRepeatedBase + kUpb_FieldType_Enum] = kUpb_DecodeOp_PackedEnum, + [kRepeatedBase + kUpb_FieldType_SFixed32] = OP_FIXPCK_LG2(2), + [kRepeatedBase + kUpb_FieldType_SFixed64] = OP_FIXPCK_LG2(3), + [kRepeatedBase + kUpb_FieldType_SInt32] = OP_VARPCK_LG2(2), + [kRepeatedBase + kUpb_FieldType_SInt64] = OP_VARPCK_LG2(3), + // Omitting kUpb_FakeFieldType_MessageSetItem, because we never emit a + // repeated msgset type + }; + + int ndx = field->UPB_PRIVATE(descriptortype); + if (upb_MiniTableField_IsArray(field)) ndx += kRepeatedBase; + int op = kDelimitedOps[ndx]; + + if (op == kUpb_DecodeOp_SubMessage) { + _upb_Decoder_CheckUnlinked(d, mt, field, &op); + } + + return op; +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_DecodeWireValue(upb_Decoder* d, const char* ptr, + const upb_MiniTable* mt, + const upb_MiniTableField* field, + int wire_type, wireval* val, + int* op) { + static const unsigned kFixed32OkMask = (1 << kUpb_FieldType_Float) | + (1 << kUpb_FieldType_Fixed32) | + (1 << kUpb_FieldType_SFixed32); + + static const unsigned kFixed64OkMask = (1 << kUpb_FieldType_Double) | + (1 << kUpb_FieldType_Fixed64) | + (1 << kUpb_FieldType_SFixed64); + + switch (wire_type) { + case kUpb_WireType_Varint: + ptr = _upb_Decoder_DecodeVarint(d, ptr, &val->uint64_val); + *op = _upb_Decoder_GetVarintOp(field); + _upb_Decoder_Munge(field->UPB_PRIVATE(descriptortype), val); + return ptr; + case kUpb_WireType_32Bit: + *op = kUpb_DecodeOp_Scalar4Byte; + if (((1 << field->UPB_PRIVATE(descriptortype)) & kFixed32OkMask) == 0) { + *op = kUpb_DecodeOp_UnknownField; + } + return upb_WireReader_ReadFixed32(ptr, &val->uint32_val); + case kUpb_WireType_64Bit: + *op = kUpb_DecodeOp_Scalar8Byte; + if (((1 << field->UPB_PRIVATE(descriptortype)) & kFixed64OkMask) == 0) { + *op = kUpb_DecodeOp_UnknownField; + } + return upb_WireReader_ReadFixed64(ptr, &val->uint64_val); + case kUpb_WireType_Delimited: + ptr = upb_Decoder_DecodeSize(d, ptr, &val->size); + *op = _upb_Decoder_GetDelimitedOp(d, mt, field); + return ptr; + case kUpb_WireType_StartGroup: + val->uint32_val = field->UPB_PRIVATE(number); + if (field->UPB_PRIVATE(descriptortype) == kUpb_FieldType_Group) { + *op = kUpb_DecodeOp_SubMessage; + _upb_Decoder_CheckUnlinked(d, mt, field, op); + } else if (field->UPB_PRIVATE(descriptortype) == + kUpb_FakeFieldType_MessageSetItem) { + *op = kUpb_DecodeOp_MessageSetItem; + } else { + *op = kUpb_DecodeOp_UnknownField; + } + return ptr; + default: + break; + } + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); +} + +UPB_FORCEINLINE +static const char* _upb_Decoder_DecodeKnownField( + upb_Decoder* d, const char* ptr, upb_Message* msg, + const upb_MiniTable* layout, const upb_MiniTableField* field, int op, + wireval* val) { + const upb_MiniTableSub* subs = layout->UPB_PRIVATE(subs); + uint8_t mode = field->UPB_PRIVATE(mode); + + if (UPB_UNLIKELY(mode & kUpb_LabelFlags_IsExtension)) { + const upb_MiniTableExtension* ext_layout = + (const upb_MiniTableExtension*)field; + upb_Extension* ext = + _upb_Message_GetOrCreateExtension(msg, ext_layout, &d->arena); + if (UPB_UNLIKELY(!ext)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + } + d->unknown_msg = msg; + msg = (upb_Message*)&ext->data; + subs = &ext->ext->UPB_PRIVATE(sub); + } + + switch (mode & kUpb_FieldMode_Mask) { + case kUpb_FieldMode_Array: + return _upb_Decoder_DecodeToArray(d, ptr, msg, subs, field, val, op); + case kUpb_FieldMode_Map: + return _upb_Decoder_DecodeToMap(d, ptr, msg, subs, field, val); + case kUpb_FieldMode_Scalar: + return _upb_Decoder_DecodeToSubMessage(d, ptr, msg, subs, field, val, op); + default: + UPB_UNREACHABLE(); + } +} + +static const char* _upb_Decoder_ReverseSkipVarint(const char* ptr, + uint32_t val) { + uint32_t seen = 0; + do { + ptr--; + seen <<= 7; + seen |= *ptr & 0x7f; + } while (seen != val); + return ptr; +} + +static const char* _upb_Decoder_DecodeUnknownField(upb_Decoder* d, + const char* ptr, + upb_Message* msg, + int field_number, + int wire_type, wireval val) { + if (field_number == 0) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + + // Since unknown fields are the uncommon case, we do a little extra work here + // to walk backwards through the buffer to find the field start. This frees + // up a register in the fast paths (when the field is known), which leads to + // significant speedups in benchmarks. + const char* start = ptr; + + if (wire_type == kUpb_WireType_Delimited) ptr += val.size; + if (msg) { + switch (wire_type) { + case kUpb_WireType_Varint: + case kUpb_WireType_Delimited: + start--; + while (start[-1] & 0x80) start--; + break; + case kUpb_WireType_32Bit: + start -= 4; + break; + case kUpb_WireType_64Bit: + start -= 8; + break; + default: + break; + } + + assert(start == d->debug_valstart); + uint32_t tag = ((uint32_t)field_number << 3) | wire_type; + start = _upb_Decoder_ReverseSkipVarint(start, tag); + assert(start == d->debug_tagstart); + + if (wire_type == kUpb_WireType_StartGroup) { + d->unknown = start; + d->unknown_msg = msg; + ptr = _upb_Decoder_DecodeUnknownGroup(d, ptr, field_number); + start = d->unknown; + d->unknown = NULL; + } + if (!UPB_PRIVATE(_upb_Message_AddUnknown)(msg, start, ptr - start, + &d->arena)) { + _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + } + } else if (wire_type == kUpb_WireType_StartGroup) { + ptr = _upb_Decoder_DecodeUnknownGroup(d, ptr, field_number); + } + return ptr; +} + +UPB_NOINLINE +static const char* _upb_Decoder_DecodeMessage(upb_Decoder* d, const char* ptr, + upb_Message* msg, + const upb_MiniTable* layout) { + int last_field_index = 0; + +#if UPB_FASTTABLE + // The first time we want to skip fast dispatch, because we may have just been + // invoked by the fast parser to handle a case that it bailed on. + if (!_upb_Decoder_IsDone(d, &ptr)) goto nofast; +#endif + + while (!_upb_Decoder_IsDone(d, &ptr)) { + uint32_t tag; + const upb_MiniTableField* field; + int field_number; + int wire_type; + wireval val; + int op; + + if (_upb_Decoder_TryFastDispatch(d, &ptr, msg, layout)) break; + +#if UPB_FASTTABLE + nofast: +#endif + +#ifndef NDEBUG + d->debug_tagstart = ptr; +#endif + + UPB_ASSERT(ptr < d->input.limit_ptr); + ptr = _upb_Decoder_DecodeTag(d, ptr, &tag); + field_number = tag >> 3; + wire_type = tag & 7; + +#ifndef NDEBUG + d->debug_valstart = ptr; +#endif + + if (wire_type == kUpb_WireType_EndGroup) { + d->end_group = field_number; + return ptr; + } + + field = _upb_Decoder_FindField(d, layout, field_number, &last_field_index); + ptr = _upb_Decoder_DecodeWireValue(d, ptr, layout, field, wire_type, &val, + &op); + + if (op >= 0) { + ptr = _upb_Decoder_DecodeKnownField(d, ptr, msg, layout, field, op, &val); + } else { + switch (op) { + case kUpb_DecodeOp_UnknownField: + ptr = _upb_Decoder_DecodeUnknownField(d, ptr, msg, field_number, + wire_type, val); + break; + case kUpb_DecodeOp_MessageSetItem: + ptr = upb_Decoder_DecodeMessageSetItem(d, ptr, msg, layout); + break; + } + } + } + + return UPB_UNLIKELY(layout && layout->UPB_PRIVATE(required_count)) + ? _upb_Decoder_CheckRequired(d, ptr, msg, layout) + : ptr; +} + +const char* _upb_FastDecoder_DecodeGeneric(struct upb_Decoder* d, + const char* ptr, upb_Message* msg, + intptr_t table, uint64_t hasbits, + uint64_t data) { + (void)data; + *(uint32_t*)msg |= hasbits; + return _upb_Decoder_DecodeMessage(d, ptr, msg, decode_totablep(table)); +} + +static upb_DecodeStatus _upb_Decoder_DecodeTop(struct upb_Decoder* d, + const char* buf, void* msg, + const upb_MiniTable* l) { + if (!_upb_Decoder_TryFastDispatch(d, &buf, msg, l)) { + _upb_Decoder_DecodeMessage(d, buf, msg, l); + } + if (d->end_group != DECODE_NOGROUP) return kUpb_DecodeStatus_Malformed; + if (d->missing_required) return kUpb_DecodeStatus_MissingRequired; + return kUpb_DecodeStatus_Ok; +} + +UPB_NOINLINE +const char* _upb_Decoder_IsDoneFallback(upb_EpsCopyInputStream* e, + const char* ptr, int overrun) { + return _upb_EpsCopyInputStream_IsDoneFallbackInline( + e, ptr, overrun, _upb_Decoder_BufferFlipCallback); +} + +static upb_DecodeStatus upb_Decoder_Decode(upb_Decoder* const decoder, + const char* const buf, + void* const msg, + const upb_MiniTable* const l, + upb_Arena* const arena) { + if (UPB_SETJMP(decoder->err) == 0) { + decoder->status = _upb_Decoder_DecodeTop(decoder, buf, msg, l); + } else { + UPB_ASSERT(decoder->status != kUpb_DecodeStatus_Ok); + } + + UPB_PRIVATE(_upb_Arena_SwapOut)(arena, &decoder->arena); + + return decoder->status; +} + +upb_DecodeStatus upb_Decode(const char* buf, size_t size, upb_Message* msg, + const upb_MiniTable* l, + const upb_ExtensionRegistry* extreg, int options, + upb_Arena* arena) { + upb_Decoder decoder; + unsigned depth = (unsigned)options >> 16; + + upb_EpsCopyInputStream_Init(&decoder.input, &buf, size, + options & kUpb_DecodeOption_AliasString); + + decoder.extreg = extreg; + decoder.unknown = NULL; + decoder.depth = depth ? depth : kUpb_WireFormat_DefaultDepthLimit; + decoder.end_group = DECODE_NOGROUP; + decoder.options = (uint16_t)options; + decoder.missing_required = false; + decoder.status = kUpb_DecodeStatus_Ok; + + // Violating the encapsulation of the arena for performance reasons. + // This is a temporary arena that we swap into and swap out of when we are + // done. The temporary arena only needs to be able to handle allocation, + // not fuse or free, so it does not need many of the members to be initialized + // (particularly parent_or_count). + UPB_PRIVATE(_upb_Arena_SwapIn)(&decoder.arena, arena); + + return upb_Decoder_Decode(&decoder, buf, msg, l, arena); +} + +#undef OP_FIXPCK_LG2 +#undef OP_VARPCK_LG2 + +// We encode backwards, to avoid pre-computing lengths (one-pass encode). + + +#include <setjmp.h> +#include <stdbool.h> +#include <stdint.h> +#include <string.h> + + +// Must be last. + +#define UPB_PB_VARINT_MAX_LEN 10 + +UPB_NOINLINE +static size_t encode_varint64(uint64_t val, char* buf) { + size_t i = 0; + do { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + buf[i++] = byte; + } while (val); + return i; +} + +static uint32_t encode_zz32(int32_t n) { + return ((uint32_t)n << 1) ^ (n >> 31); +} +static uint64_t encode_zz64(int64_t n) { + return ((uint64_t)n << 1) ^ (n >> 63); +} + +typedef struct { + upb_EncodeStatus status; + jmp_buf err; + upb_Arena* arena; + char *buf, *ptr, *limit; + int options; + int depth; + _upb_mapsorter sorter; +} upb_encstate; + +static size_t upb_roundup_pow2(size_t bytes) { + size_t ret = 128; + while (ret < bytes) { + ret *= 2; + } + return ret; +} + +UPB_NORETURN static void encode_err(upb_encstate* e, upb_EncodeStatus s) { + UPB_ASSERT(s != kUpb_EncodeStatus_Ok); + e->status = s; + UPB_LONGJMP(e->err, 1); +} + +UPB_NOINLINE +static void encode_growbuffer(upb_encstate* e, size_t bytes) { + size_t old_size = e->limit - e->buf; + size_t new_size = upb_roundup_pow2(bytes + (e->limit - e->ptr)); + char* new_buf = upb_Arena_Realloc(e->arena, e->buf, old_size, new_size); + + if (!new_buf) encode_err(e, kUpb_EncodeStatus_OutOfMemory); + + // We want previous data at the end, realloc() put it at the beginning. + // TODO: This is somewhat inefficient since we are copying twice. + // Maybe create a realloc() that copies to the end of the new buffer? + if (old_size > 0) { + memmove(new_buf + new_size - old_size, e->buf, old_size); + } + + e->ptr = new_buf + new_size - (e->limit - e->ptr); + e->limit = new_buf + new_size; + e->buf = new_buf; + + e->ptr -= bytes; +} + +/* Call to ensure that at least "bytes" bytes are available for writing at + * e->ptr. Returns false if the bytes could not be allocated. */ +UPB_FORCEINLINE +static void encode_reserve(upb_encstate* e, size_t bytes) { + if ((size_t)(e->ptr - e->buf) < bytes) { + encode_growbuffer(e, bytes); + return; + } + + e->ptr -= bytes; +} + +/* Writes the given bytes to the buffer, handling reserve/advance. */ +static void encode_bytes(upb_encstate* e, const void* data, size_t len) { + if (len == 0) return; /* memcpy() with zero size is UB */ + encode_reserve(e, len); + memcpy(e->ptr, data, len); +} + +static void encode_fixed64(upb_encstate* e, uint64_t val) { + val = UPB_PRIVATE(_upb_BigEndian64)(val); + encode_bytes(e, &val, sizeof(uint64_t)); +} + +static void encode_fixed32(upb_encstate* e, uint32_t val) { + val = UPB_PRIVATE(_upb_BigEndian32)(val); + encode_bytes(e, &val, sizeof(uint32_t)); +} + +UPB_NOINLINE +static void encode_longvarint(upb_encstate* e, uint64_t val) { + size_t len; + char* start; + + encode_reserve(e, UPB_PB_VARINT_MAX_LEN); + len = encode_varint64(val, e->ptr); + start = e->ptr + UPB_PB_VARINT_MAX_LEN - len; + memmove(start, e->ptr, len); + e->ptr = start; +} + +UPB_FORCEINLINE +static void encode_varint(upb_encstate* e, uint64_t val) { + if (val < 128 && e->ptr != e->buf) { + --e->ptr; + *e->ptr = val; + } else { + encode_longvarint(e, val); + } +} + +static void encode_double(upb_encstate* e, double d) { + uint64_t u64; + UPB_ASSERT(sizeof(double) == sizeof(uint64_t)); + memcpy(&u64, &d, sizeof(uint64_t)); + encode_fixed64(e, u64); +} + +static void encode_float(upb_encstate* e, float d) { + uint32_t u32; + UPB_ASSERT(sizeof(float) == sizeof(uint32_t)); + memcpy(&u32, &d, sizeof(uint32_t)); + encode_fixed32(e, u32); +} + +static void encode_tag(upb_encstate* e, uint32_t field_number, + uint8_t wire_type) { + encode_varint(e, (field_number << 3) | wire_type); +} + +static void encode_fixedarray(upb_encstate* e, const upb_Array* arr, + size_t elem_size, uint32_t tag) { + size_t bytes = arr->UPB_PRIVATE(size) * elem_size; + const char* data = _upb_array_constptr(arr); + const char* ptr = data + bytes - elem_size; + + if (tag || !UPB_PRIVATE(_upb_IsLittleEndian)()) { + while (true) { + if (elem_size == 4) { + uint32_t val; + memcpy(&val, ptr, sizeof(val)); + val = UPB_PRIVATE(_upb_BigEndian32)(val); + encode_bytes(e, &val, elem_size); + } else { + UPB_ASSERT(elem_size == 8); + uint64_t val; + memcpy(&val, ptr, sizeof(val)); + val = UPB_PRIVATE(_upb_BigEndian64)(val); + encode_bytes(e, &val, elem_size); + } + + if (tag) encode_varint(e, tag); + if (ptr == data) break; + ptr -= elem_size; + } + } else { + encode_bytes(e, data, bytes); + } +} + +static void encode_message(upb_encstate* e, const upb_Message* msg, + const upb_MiniTable* m, size_t* size); + +static void encode_TaggedMessagePtr(upb_encstate* e, + upb_TaggedMessagePtr tagged, + const upb_MiniTable* m, size_t* size) { + if (upb_TaggedMessagePtr_IsEmpty(tagged)) { + m = UPB_PRIVATE(_upb_MiniTable_Empty)(); + } + encode_message(e, UPB_PRIVATE(_upb_TaggedMessagePtr_GetMessage)(tagged), m, + size); +} + +static void encode_scalar(upb_encstate* e, const void* _field_mem, + const upb_MiniTableSub* subs, + const upb_MiniTableField* f) { + const char* field_mem = _field_mem; + int wire_type; + +#define CASE(ctype, type, wtype, encodeval) \ + { \ + ctype val = *(ctype*)field_mem; \ + encode_##type(e, encodeval); \ + wire_type = wtype; \ + break; \ + } + + switch (f->UPB_PRIVATE(descriptortype)) { + case kUpb_FieldType_Double: + CASE(double, double, kUpb_WireType_64Bit, val); + case kUpb_FieldType_Float: + CASE(float, float, kUpb_WireType_32Bit, val); + case kUpb_FieldType_Int64: + case kUpb_FieldType_UInt64: + CASE(uint64_t, varint, kUpb_WireType_Varint, val); + case kUpb_FieldType_UInt32: + CASE(uint32_t, varint, kUpb_WireType_Varint, val); + case kUpb_FieldType_Int32: + case kUpb_FieldType_Enum: + CASE(int32_t, varint, kUpb_WireType_Varint, (int64_t)val); + case kUpb_FieldType_SFixed64: + case kUpb_FieldType_Fixed64: + CASE(uint64_t, fixed64, kUpb_WireType_64Bit, val); + case kUpb_FieldType_Fixed32: + case kUpb_FieldType_SFixed32: + CASE(uint32_t, fixed32, kUpb_WireType_32Bit, val); + case kUpb_FieldType_Bool: + CASE(bool, varint, kUpb_WireType_Varint, val); + case kUpb_FieldType_SInt32: + CASE(int32_t, varint, kUpb_WireType_Varint, encode_zz32(val)); + case kUpb_FieldType_SInt64: + CASE(int64_t, varint, kUpb_WireType_Varint, encode_zz64(val)); + case kUpb_FieldType_String: + case kUpb_FieldType_Bytes: { + upb_StringView view = *(upb_StringView*)field_mem; + encode_bytes(e, view.data, view.size); + encode_varint(e, view.size); + wire_type = kUpb_WireType_Delimited; + break; + } + case kUpb_FieldType_Group: { + size_t size; + upb_TaggedMessagePtr submsg = *(upb_TaggedMessagePtr*)field_mem; + const upb_MiniTable* subm = + upb_MiniTableSub_Message(subs[f->UPB_PRIVATE(submsg_index)]); + if (submsg == 0) { + return; + } + if (--e->depth == 0) encode_err(e, kUpb_EncodeStatus_MaxDepthExceeded); + encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_EndGroup); + encode_TaggedMessagePtr(e, submsg, subm, &size); + wire_type = kUpb_WireType_StartGroup; + e->depth++; + break; + } + case kUpb_FieldType_Message: { + size_t size; + upb_TaggedMessagePtr submsg = *(upb_TaggedMessagePtr*)field_mem; + const upb_MiniTable* subm = + upb_MiniTableSub_Message(subs[f->UPB_PRIVATE(submsg_index)]); + if (submsg == 0) { + return; + } + if (--e->depth == 0) encode_err(e, kUpb_EncodeStatus_MaxDepthExceeded); + encode_TaggedMessagePtr(e, submsg, subm, &size); + encode_varint(e, size); + wire_type = kUpb_WireType_Delimited; + e->depth++; + break; + } + default: + UPB_UNREACHABLE(); + } +#undef CASE + + encode_tag(e, f->UPB_PRIVATE(number), wire_type); +} + +static void encode_array(upb_encstate* e, const upb_Message* msg, + const upb_MiniTableSub* subs, + const upb_MiniTableField* f) { + const upb_Array* arr = *UPB_PTR_AT(msg, f->UPB_PRIVATE(offset), upb_Array*); + bool packed = upb_MiniTableField_IsPacked(f); + size_t pre_len = e->limit - e->ptr; + + if (arr == NULL || arr->UPB_PRIVATE(size) == 0) { + return; + } + +#define VARINT_CASE(ctype, encode) \ + { \ + const ctype* start = _upb_array_constptr(arr); \ + const ctype* ptr = start + arr->UPB_PRIVATE(size); \ + uint32_t tag = \ + packed ? 0 : (f->UPB_PRIVATE(number) << 3) | kUpb_WireType_Varint; \ + do { \ + ptr--; \ + encode_varint(e, encode); \ + if (tag) encode_varint(e, tag); \ + } while (ptr != start); \ + } \ + break; + +#define TAG(wire_type) (packed ? 0 : (f->UPB_PRIVATE(number) << 3 | wire_type)) + + switch (f->UPB_PRIVATE(descriptortype)) { + case kUpb_FieldType_Double: + encode_fixedarray(e, arr, sizeof(double), TAG(kUpb_WireType_64Bit)); + break; + case kUpb_FieldType_Float: + encode_fixedarray(e, arr, sizeof(float), TAG(kUpb_WireType_32Bit)); + break; + case kUpb_FieldType_SFixed64: + case kUpb_FieldType_Fixed64: + encode_fixedarray(e, arr, sizeof(uint64_t), TAG(kUpb_WireType_64Bit)); + break; + case kUpb_FieldType_Fixed32: + case kUpb_FieldType_SFixed32: + encode_fixedarray(e, arr, sizeof(uint32_t), TAG(kUpb_WireType_32Bit)); + break; + case kUpb_FieldType_Int64: + case kUpb_FieldType_UInt64: + VARINT_CASE(uint64_t, *ptr); + case kUpb_FieldType_UInt32: + VARINT_CASE(uint32_t, *ptr); + case kUpb_FieldType_Int32: + case kUpb_FieldType_Enum: + VARINT_CASE(int32_t, (int64_t)*ptr); + case kUpb_FieldType_Bool: + VARINT_CASE(bool, *ptr); + case kUpb_FieldType_SInt32: + VARINT_CASE(int32_t, encode_zz32(*ptr)); + case kUpb_FieldType_SInt64: + VARINT_CASE(int64_t, encode_zz64(*ptr)); + case kUpb_FieldType_String: + case kUpb_FieldType_Bytes: { + const upb_StringView* start = _upb_array_constptr(arr); + const upb_StringView* ptr = start + arr->UPB_PRIVATE(size); + do { + ptr--; + encode_bytes(e, ptr->data, ptr->size); + encode_varint(e, ptr->size); + encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_Delimited); + } while (ptr != start); + return; + } + case kUpb_FieldType_Group: { + const upb_TaggedMessagePtr* start = _upb_array_constptr(arr); + const upb_TaggedMessagePtr* ptr = start + arr->UPB_PRIVATE(size); + const upb_MiniTable* subm = + upb_MiniTableSub_Message(subs[f->UPB_PRIVATE(submsg_index)]); + if (--e->depth == 0) encode_err(e, kUpb_EncodeStatus_MaxDepthExceeded); + do { + size_t size; + ptr--; + encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_EndGroup); + encode_TaggedMessagePtr(e, *ptr, subm, &size); + encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_StartGroup); + } while (ptr != start); + e->depth++; + return; + } + case kUpb_FieldType_Message: { + const upb_TaggedMessagePtr* start = _upb_array_constptr(arr); + const upb_TaggedMessagePtr* ptr = start + arr->UPB_PRIVATE(size); + const upb_MiniTable* subm = + upb_MiniTableSub_Message(subs[f->UPB_PRIVATE(submsg_index)]); + if (--e->depth == 0) encode_err(e, kUpb_EncodeStatus_MaxDepthExceeded); + do { + size_t size; + ptr--; + encode_TaggedMessagePtr(e, *ptr, subm, &size); + encode_varint(e, size); + encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_Delimited); + } while (ptr != start); + e->depth++; + return; + } + } +#undef VARINT_CASE + + if (packed) { + encode_varint(e, e->limit - e->ptr - pre_len); + encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_Delimited); + } +} + +static void encode_mapentry(upb_encstate* e, uint32_t number, + const upb_MiniTable* layout, + const upb_MapEntry* ent) { + const upb_MiniTableField* key_field = &layout->UPB_PRIVATE(fields)[0]; + const upb_MiniTableField* val_field = &layout->UPB_PRIVATE(fields)[1]; + size_t pre_len = e->limit - e->ptr; + size_t size; + encode_scalar(e, &ent->data.v, layout->UPB_PRIVATE(subs), val_field); + encode_scalar(e, &ent->data.k, layout->UPB_PRIVATE(subs), key_field); + size = (e->limit - e->ptr) - pre_len; + encode_varint(e, size); + encode_tag(e, number, kUpb_WireType_Delimited); +} + +static void encode_map(upb_encstate* e, const upb_Message* msg, + const upb_MiniTableSub* subs, + const upb_MiniTableField* f) { + const upb_Map* map = *UPB_PTR_AT(msg, f->UPB_PRIVATE(offset), const upb_Map*); + const upb_MiniTable* layout = + upb_MiniTableSub_Message(subs[f->UPB_PRIVATE(submsg_index)]); + UPB_ASSERT(layout->UPB_PRIVATE(field_count) == 2); + + if (map == NULL) return; + + if (e->options & kUpb_EncodeOption_Deterministic) { + _upb_sortedmap sorted; + _upb_mapsorter_pushmap( + &e->sorter, layout->UPB_PRIVATE(fields)[0].UPB_PRIVATE(descriptortype), + map, &sorted); + upb_MapEntry ent; + while (_upb_sortedmap_next(&e->sorter, map, &sorted, &ent)) { + encode_mapentry(e, f->UPB_PRIVATE(number), layout, &ent); + } + _upb_mapsorter_popmap(&e->sorter, &sorted); + } else { + intptr_t iter = UPB_STRTABLE_BEGIN; + upb_StringView key; + upb_value val; + while (upb_strtable_next2(&map->table, &key, &val, &iter)) { + upb_MapEntry ent; + _upb_map_fromkey(key, &ent.data.k, map->key_size); + _upb_map_fromvalue(val, &ent.data.v, map->val_size); + encode_mapentry(e, f->UPB_PRIVATE(number), layout, &ent); + } + } +} + +static bool encode_shouldencode(upb_encstate* e, const upb_Message* msg, + const upb_MiniTableSub* subs, + const upb_MiniTableField* f) { + if (f->presence == 0) { + // Proto3 presence or map/array. + const void* mem = UPB_PTR_AT(msg, f->UPB_PRIVATE(offset), void); + switch (UPB_PRIVATE(_upb_MiniTableField_GetRep)(f)) { + case kUpb_FieldRep_1Byte: { + char ch; + memcpy(&ch, mem, 1); + return ch != 0; + } + case kUpb_FieldRep_4Byte: { + uint32_t u32; + memcpy(&u32, mem, 4); + return u32 != 0; + } + case kUpb_FieldRep_8Byte: { + uint64_t u64; + memcpy(&u64, mem, 8); + return u64 != 0; + } + case kUpb_FieldRep_StringView: { + const upb_StringView* str = (const upb_StringView*)mem; + return str->size != 0; + } + default: + UPB_UNREACHABLE(); + } + } else if (f->presence > 0) { + // Proto2 presence: hasbit. + return UPB_PRIVATE(_upb_Message_GetHasbit)(msg, f); + } else { + // Field is in a oneof. + return UPB_PRIVATE(_upb_Message_GetOneofCase)(msg, f) == + f->UPB_PRIVATE(number); + } +} + +static void encode_field(upb_encstate* e, const upb_Message* msg, + const upb_MiniTableSub* subs, + const upb_MiniTableField* field) { + switch (UPB_PRIVATE(_upb_MiniTableField_Mode)(field)) { + case kUpb_FieldMode_Array: + encode_array(e, msg, subs, field); + break; + case kUpb_FieldMode_Map: + encode_map(e, msg, subs, field); + break; + case kUpb_FieldMode_Scalar: + encode_scalar(e, UPB_PTR_AT(msg, field->UPB_PRIVATE(offset), void), subs, + field); + break; + default: + UPB_UNREACHABLE(); + } +} + +static void encode_msgset_item(upb_encstate* e, const upb_Extension* ext) { + size_t size; + encode_tag(e, kUpb_MsgSet_Item, kUpb_WireType_EndGroup); + encode_message(e, ext->data.ptr, + upb_MiniTableExtension_GetSubMessage(ext->ext), &size); + encode_varint(e, size); + encode_tag(e, kUpb_MsgSet_Message, kUpb_WireType_Delimited); + encode_varint(e, upb_MiniTableExtension_Number(ext->ext)); + encode_tag(e, kUpb_MsgSet_TypeId, kUpb_WireType_Varint); + encode_tag(e, kUpb_MsgSet_Item, kUpb_WireType_StartGroup); +} + +static void encode_ext(upb_encstate* e, const upb_Extension* ext, + bool is_message_set) { + if (UPB_UNLIKELY(is_message_set)) { + encode_msgset_item(e, ext); + } else { + encode_field(e, (upb_Message*)&ext->data, &ext->ext->UPB_PRIVATE(sub), + &ext->ext->UPB_PRIVATE(field)); + } +} + +static void encode_message(upb_encstate* e, const upb_Message* msg, + const upb_MiniTable* m, size_t* size) { + size_t pre_len = e->limit - e->ptr; + + if ((e->options & kUpb_EncodeOption_CheckRequired) && + m->UPB_PRIVATE(required_count)) { + uint64_t msg_head; + memcpy(&msg_head, msg, 8); + msg_head = UPB_PRIVATE(_upb_BigEndian64)(msg_head); + if (UPB_PRIVATE(_upb_MiniTable_RequiredMask)(m) & ~msg_head) { + encode_err(e, kUpb_EncodeStatus_MissingRequired); + } + } + + if ((e->options & kUpb_EncodeOption_SkipUnknown) == 0) { + size_t unknown_size; + const char* unknown = upb_Message_GetUnknown(msg, &unknown_size); + + if (unknown) { + encode_bytes(e, unknown, unknown_size); + } + } + + if (m->UPB_PRIVATE(ext) != kUpb_ExtMode_NonExtendable) { + /* Encode all extensions together. Unlike C++, we do not attempt to keep + * these in field number order relative to normal fields or even to each + * other. */ + size_t ext_count; + const upb_Extension* ext = + UPB_PRIVATE(_upb_Message_Getexts)(msg, &ext_count); + if (ext_count) { + if (e->options & kUpb_EncodeOption_Deterministic) { + _upb_sortedmap sorted; + _upb_mapsorter_pushexts(&e->sorter, ext, ext_count, &sorted); + while (_upb_sortedmap_nextext(&e->sorter, &sorted, &ext)) { + encode_ext(e, ext, m->UPB_PRIVATE(ext) == kUpb_ExtMode_IsMessageSet); + } + _upb_mapsorter_popmap(&e->sorter, &sorted); + } else { + const upb_Extension* end = ext + ext_count; + for (; ext != end; ext++) { + encode_ext(e, ext, m->UPB_PRIVATE(ext) == kUpb_ExtMode_IsMessageSet); + } + } + } + } + + if (m->UPB_PRIVATE(field_count)) { + const upb_MiniTableField* f = + &m->UPB_PRIVATE(fields)[m->UPB_PRIVATE(field_count)]; + const upb_MiniTableField* first = &m->UPB_PRIVATE(fields)[0]; + while (f != first) { + f--; + if (encode_shouldencode(e, msg, m->UPB_PRIVATE(subs), f)) { + encode_field(e, msg, m->UPB_PRIVATE(subs), f); + } + } + } + + *size = (e->limit - e->ptr) - pre_len; +} + +static upb_EncodeStatus upb_Encoder_Encode(upb_encstate* const encoder, + const upb_Message* const msg, + const upb_MiniTable* const l, + char** const buf, + size_t* const size) { + // Unfortunately we must continue to perform hackery here because there are + // code paths which blindly copy the returned pointer without bothering to + // check for errors until much later (b/235839510). So we still set *buf to + // NULL on error and we still set it to non-NULL on a successful empty result. + if (UPB_SETJMP(encoder->err) == 0) { + encode_message(encoder, msg, l, size); + *size = encoder->limit - encoder->ptr; + if (*size == 0) { + static char ch; + *buf = &ch; + } else { + UPB_ASSERT(encoder->ptr); + *buf = encoder->ptr; + } + } else { + UPB_ASSERT(encoder->status != kUpb_EncodeStatus_Ok); + *buf = NULL; + *size = 0; + } + + _upb_mapsorter_destroy(&encoder->sorter); + return encoder->status; +} + +upb_EncodeStatus upb_Encode(const upb_Message* msg, const upb_MiniTable* l, + int options, upb_Arena* arena, char** buf, + size_t* size) { + upb_encstate e; + unsigned depth = (unsigned)options >> 16; + + e.status = kUpb_EncodeStatus_Ok; + e.arena = arena; + e.buf = NULL; + e.limit = NULL; + e.ptr = NULL; + e.depth = depth ? depth : kUpb_WireFormat_DefaultDepthLimit; + e.options = options; + _upb_mapsorter_init(&e.sorter); + + return upb_Encoder_Encode(&e, msg, l, buf, size); +} + +// Fast decoder: ~3x the speed of decode.c, but requires x86-64/ARM64. +// Also the table size grows by 2x. +// +// Could potentially be ported to other 64-bit archs that pass at least six +// arguments in registers and have 8 unused high bits in pointers. +// +// The overall design is to create specialized functions for every possible +// field type (eg. oneof boolean field with a 1 byte tag) and then dispatch +// to the specialized function as quickly as possible. + + + +// Must be last. + +#if UPB_FASTTABLE + +// The standard set of arguments passed to each parsing function. +// Thanks to x86-64 calling conventions, these will stay in registers. +#define UPB_PARSE_PARAMS \ + upb_Decoder *d, const char *ptr, upb_Message *msg, intptr_t table, \ + uint64_t hasbits, uint64_t data + +#define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data + +#define RETURN_GENERIC(m) \ + /* Uncomment either of these for debugging purposes. */ \ + /* fprintf(stderr, m); */ \ + /*__builtin_trap(); */ \ + return _upb_FastDecoder_DecodeGeneric(d, ptr, msg, table, hasbits, 0); + +typedef enum { + CARD_s = 0, /* Singular (optional, non-repeated) */ + CARD_o = 1, /* Oneof */ + CARD_r = 2, /* Repeated */ + CARD_p = 3 /* Packed Repeated */ +} upb_card; + +UPB_NOINLINE +static const char* fastdecode_isdonefallback(UPB_PARSE_PARAMS) { + int overrun = data; + ptr = _upb_EpsCopyInputStream_IsDoneFallbackInline( + &d->input, ptr, overrun, _upb_Decoder_BufferFlipCallback); + data = _upb_FastDecoder_LoadTag(ptr); + UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); +} + +UPB_FORCEINLINE +static const char* fastdecode_dispatch(UPB_PARSE_PARAMS) { + int overrun; + switch (upb_EpsCopyInputStream_IsDoneStatus(&d->input, ptr, &overrun)) { + case kUpb_IsDoneStatus_Done: + *(uint32_t*)msg |= hasbits; // Sync hasbits. + const upb_MiniTable* m = decode_totablep(table); + return UPB_UNLIKELY(m->UPB_PRIVATE(required_count)) + ? _upb_Decoder_CheckRequired(d, ptr, msg, m) + : ptr; + case kUpb_IsDoneStatus_NotDone: + break; + case kUpb_IsDoneStatus_NeedFallback: + data = overrun; + UPB_MUSTTAIL return fastdecode_isdonefallback(UPB_PARSE_ARGS); + } + + // Read two bytes of tag data (for a one-byte tag, the high byte is junk). + data = _upb_FastDecoder_LoadTag(ptr); + UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); +} + +UPB_FORCEINLINE +static bool fastdecode_checktag(uint16_t data, int tagbytes) { + if (tagbytes == 1) { + return (data & 0xff) == 0; + } else { + return data == 0; + } +} + +UPB_FORCEINLINE +static const char* fastdecode_longsize(const char* ptr, int* size) { + int i; + UPB_ASSERT(*size & 0x80); + *size &= 0xff; + for (i = 0; i < 3; i++) { + ptr++; + size_t byte = (uint8_t)ptr[-1]; + *size += (byte - 1) << (7 + 7 * i); + if (UPB_LIKELY((byte & 0x80) == 0)) return ptr; + } + ptr++; + size_t byte = (uint8_t)ptr[-1]; + // len is limited by 2gb not 4gb, hence 8 and not 16 as normally expected + // for a 32 bit varint. + if (UPB_UNLIKELY(byte >= 8)) return NULL; + *size += (byte - 1) << 28; + return ptr; +} + +UPB_FORCEINLINE +static const char* fastdecode_delimited( + upb_Decoder* d, const char* ptr, + upb_EpsCopyInputStream_ParseDelimitedFunc* func, void* ctx) { + ptr++; + + // Sign-extend so varint greater than one byte becomes negative, causing + // fast delimited parse to fail. + int len = (int8_t)ptr[-1]; + + if (!upb_EpsCopyInputStream_TryParseDelimitedFast(&d->input, &ptr, len, func, + ctx)) { + // Slow case: Sub-message is >=128 bytes and/or exceeds the current buffer. + // If it exceeds the buffer limit, limit/limit_ptr will change during + // sub-message parsing, so we need to preserve delta, not limit. + if (UPB_UNLIKELY(len & 0x80)) { + // Size varint >1 byte (length >= 128). + ptr = fastdecode_longsize(ptr, &len); + if (!ptr) { + // Corrupt wire format: size exceeded INT_MAX. + return NULL; + } + } + if (!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, len)) { + // Corrupt wire format: invalid limit. + return NULL; + } + int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, len); + ptr = func(&d->input, ptr, ctx); + upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta); + } + return ptr; +} + +/* singular, oneof, repeated field handling ***********************************/ + +typedef struct { + upb_Array* arr; + void* end; +} fastdecode_arr; + +typedef enum { + FD_NEXT_ATLIMIT, + FD_NEXT_SAMEFIELD, + FD_NEXT_OTHERFIELD +} fastdecode_next; + +typedef struct { + void* dst; + fastdecode_next next; + uint32_t tag; +} fastdecode_nextret; + +UPB_FORCEINLINE +static void* fastdecode_resizearr(upb_Decoder* d, void* dst, + fastdecode_arr* farr, int valbytes) { + if (UPB_UNLIKELY(dst == farr->end)) { + size_t old_capacity = farr->arr->UPB_PRIVATE(capacity); + size_t old_bytes = old_capacity * valbytes; + size_t new_capacity = old_capacity * 2; + size_t new_bytes = new_capacity * valbytes; + char* old_ptr = _upb_array_ptr(farr->arr); + char* new_ptr = upb_Arena_Realloc(&d->arena, old_ptr, old_bytes, new_bytes); + uint8_t elem_size_lg2 = __builtin_ctz(valbytes); + UPB_PRIVATE(_upb_Array_SetTaggedPtr)(farr->arr, new_ptr, elem_size_lg2); + farr->arr->UPB_PRIVATE(capacity) = new_capacity; + dst = (void*)(new_ptr + (old_capacity * valbytes)); + farr->end = (void*)(new_ptr + (new_capacity * valbytes)); + } + return dst; +} + +UPB_FORCEINLINE +static bool fastdecode_tagmatch(uint32_t tag, uint64_t data, int tagbytes) { + if (tagbytes == 1) { + return (uint8_t)tag == (uint8_t)data; + } else { + return (uint16_t)tag == (uint16_t)data; + } +} + +UPB_FORCEINLINE +static void fastdecode_commitarr(void* dst, fastdecode_arr* farr, + int valbytes) { + farr->arr->UPB_PRIVATE(size) = + (size_t)((char*)dst - (char*)_upb_array_ptr(farr->arr)) / valbytes; +} + +UPB_FORCEINLINE +static fastdecode_nextret fastdecode_nextrepeated(upb_Decoder* d, void* dst, + const char** ptr, + fastdecode_arr* farr, + uint64_t data, int tagbytes, + int valbytes) { + fastdecode_nextret ret; + dst = (char*)dst + valbytes; + + if (UPB_LIKELY(!_upb_Decoder_IsDone(d, ptr))) { + ret.tag = _upb_FastDecoder_LoadTag(*ptr); + if (fastdecode_tagmatch(ret.tag, data, tagbytes)) { + ret.next = FD_NEXT_SAMEFIELD; + } else { + fastdecode_commitarr(dst, farr, valbytes); + ret.next = FD_NEXT_OTHERFIELD; + } + } else { + fastdecode_commitarr(dst, farr, valbytes); + ret.next = FD_NEXT_ATLIMIT; + } + + ret.dst = dst; + return ret; +} + +UPB_FORCEINLINE +static void* fastdecode_fieldmem(upb_Message* msg, uint64_t data) { + size_t ofs = data >> 48; + return (char*)msg + ofs; +} + +UPB_FORCEINLINE +static void* fastdecode_getfield(upb_Decoder* d, const char* ptr, + upb_Message* msg, uint64_t* data, + uint64_t* hasbits, fastdecode_arr* farr, + int valbytes, upb_card card) { + switch (card) { + case CARD_s: { + uint8_t hasbit_index = *data >> 24; + // Set hasbit and return pointer to scalar field. + *hasbits |= 1ull << hasbit_index; + return fastdecode_fieldmem(msg, *data); + } + case CARD_o: { + uint16_t case_ofs = *data >> 32; + uint32_t* oneof_case = UPB_PTR_AT(msg, case_ofs, uint32_t); + uint8_t field_number = *data >> 24; + *oneof_case = field_number; + return fastdecode_fieldmem(msg, *data); + } + case CARD_r: { + // Get pointer to upb_Array and allocate/expand if necessary. + uint8_t elem_size_lg2 = __builtin_ctz(valbytes); + upb_Array** arr_p = fastdecode_fieldmem(msg, *data); + char* begin; + *(uint32_t*)msg |= *hasbits; + *hasbits = 0; + if (UPB_LIKELY(!*arr_p)) { + farr->arr = UPB_PRIVATE(_upb_Array_New)(&d->arena, 8, elem_size_lg2); + *arr_p = farr->arr; + } else { + farr->arr = *arr_p; + } + begin = _upb_array_ptr(farr->arr); + farr->end = begin + (farr->arr->UPB_PRIVATE(capacity) * valbytes); + *data = _upb_FastDecoder_LoadTag(ptr); + return begin + (farr->arr->UPB_PRIVATE(size) * valbytes); + } + default: + UPB_UNREACHABLE(); + } +} + +UPB_FORCEINLINE +static bool fastdecode_flippacked(uint64_t* data, int tagbytes) { + *data ^= (0x2 ^ 0x0); // Patch data to match packed wiretype. + return fastdecode_checktag(*data, tagbytes); +} + +#define FASTDECODE_CHECKPACKED(tagbytes, card, func) \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \ + UPB_MUSTTAIL return func(UPB_PARSE_ARGS); \ + } \ + RETURN_GENERIC("packed check tag mismatch\n"); \ + } + +/* varint fields **************************************************************/ + +UPB_FORCEINLINE +static uint64_t fastdecode_munge(uint64_t val, int valbytes, bool zigzag) { + if (valbytes == 1) { + return val != 0; + } else if (zigzag) { + if (valbytes == 4) { + uint32_t n = val; + return (n >> 1) ^ -(int32_t)(n & 1); + } else if (valbytes == 8) { + return (val >> 1) ^ -(int64_t)(val & 1); + } + UPB_UNREACHABLE(); + } + return val; +} + +UPB_FORCEINLINE +static const char* fastdecode_varint64(const char* ptr, uint64_t* val) { + ptr++; + *val = (uint8_t)ptr[-1]; + if (UPB_UNLIKELY(*val & 0x80)) { + int i; + for (i = 0; i < 8; i++) { + ptr++; + uint64_t byte = (uint8_t)ptr[-1]; + *val += (byte - 1) << (7 + 7 * i); + if (UPB_LIKELY((byte & 0x80) == 0)) goto done; + } + ptr++; + uint64_t byte = (uint8_t)ptr[-1]; + if (byte > 1) { + return NULL; + } + *val += (byte - 1) << 63; + } +done: + UPB_ASSUME(ptr != NULL); + return ptr; +} + +#define FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, zigzag, packed) \ + uint64_t val; \ + void* dst; \ + fastdecode_arr farr; \ + \ + FASTDECODE_CHECKPACKED(tagbytes, card, packed); \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ + card); \ + if (card == CARD_r) { \ + if (UPB_UNLIKELY(!dst)) { \ + RETURN_GENERIC("need array resize\n"); \ + } \ + } \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ + } \ + \ + ptr += tagbytes; \ + ptr = fastdecode_varint64(ptr, &val); \ + if (ptr == NULL) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ + val = fastdecode_munge(val, valbytes, zigzag); \ + memcpy(dst, &val, valbytes); \ + \ + if (card == CARD_r) { \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, valbytes); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); + +typedef struct { + uint8_t valbytes; + bool zigzag; + void* dst; + fastdecode_arr farr; +} fastdecode_varintdata; + +UPB_FORCEINLINE +static const char* fastdecode_topackedvarint(upb_EpsCopyInputStream* e, + const char* ptr, void* ctx) { + upb_Decoder* d = (upb_Decoder*)e; + fastdecode_varintdata* data = ctx; + void* dst = data->dst; + uint64_t val; + + while (!_upb_Decoder_IsDone(d, &ptr)) { + dst = fastdecode_resizearr(d, dst, &data->farr, data->valbytes); + ptr = fastdecode_varint64(ptr, &val); + if (ptr == NULL) return NULL; + val = fastdecode_munge(val, data->valbytes, data->zigzag); + memcpy(dst, &val, data->valbytes); + dst = (char*)dst + data->valbytes; + } + + fastdecode_commitarr(dst, &data->farr, data->valbytes); + return ptr; +} + +#define FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, zigzag, unpacked) \ + fastdecode_varintdata ctx = {valbytes, zigzag}; \ + \ + FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked); \ + \ + ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \ + valbytes, CARD_r); \ + if (UPB_UNLIKELY(!ctx.dst)) { \ + RETURN_GENERIC("need array resize\n"); \ + } \ + \ + ptr += tagbytes; \ + ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \ + \ + if (UPB_UNLIKELY(ptr == NULL)) { \ + _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(d, ptr, msg, table, hasbits, 0); + +#define FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, zigzag, unpacked, packed) \ + if (card == CARD_p) { \ + FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, zigzag, unpacked); \ + } else { \ + FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, zigzag, packed); \ + } + +#define z_ZZ true +#define b_ZZ false +#define v_ZZ false + +/* Generate all combinations: + * {s,o,r,p} x {b1,v4,z4,v8,z8} x {1bt,2bt} */ + +#define F(card, type, valbytes, tagbytes) \ + UPB_NOINLINE \ + const char* upb_p##card##type##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ + FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \ + CARD_##card, type##_ZZ, \ + upb_pr##type##valbytes##_##tagbytes##bt, \ + upb_pp##type##valbytes##_##tagbytes##bt); \ + } + +#define TYPES(card, tagbytes) \ + F(card, b, 1, tagbytes) \ + F(card, v, 4, tagbytes) \ + F(card, v, 8, tagbytes) \ + F(card, z, 4, tagbytes) \ + F(card, z, 8, tagbytes) + +#define TAGBYTES(card) \ + TYPES(card, 1) \ + TYPES(card, 2) + +TAGBYTES(s) +TAGBYTES(o) +TAGBYTES(r) +TAGBYTES(p) + +#undef z_ZZ +#undef b_ZZ +#undef v_ZZ +#undef o_ONEOF +#undef s_ONEOF +#undef r_ONEOF +#undef F +#undef TYPES +#undef TAGBYTES +#undef FASTDECODE_UNPACKEDVARINT +#undef FASTDECODE_PACKEDVARINT +#undef FASTDECODE_VARINT + +/* fixed fields ***************************************************************/ + +#define FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, packed) \ + void* dst; \ + fastdecode_arr farr; \ + \ + FASTDECODE_CHECKPACKED(tagbytes, card, packed) \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ + card); \ + if (card == CARD_r) { \ + if (UPB_UNLIKELY(!dst)) { \ + RETURN_GENERIC("couldn't allocate array in arena\n"); \ + } \ + } \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ + } \ + \ + ptr += tagbytes; \ + memcpy(dst, ptr, valbytes); \ + ptr += valbytes; \ + \ + if (card == CARD_r) { \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, valbytes); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); + +#define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, unpacked) \ + FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked) \ + \ + ptr += tagbytes; \ + int size = (uint8_t)ptr[0]; \ + ptr++; \ + if (size & 0x80) { \ + ptr = fastdecode_longsize(ptr, &size); \ + } \ + \ + if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckDataSizeAvailable( \ + &d->input, ptr, size) || \ + (size % valbytes) != 0)) { \ + _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ + } \ + \ + upb_Array** arr_p = fastdecode_fieldmem(msg, data); \ + upb_Array* arr = *arr_p; \ + uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \ + int elems = size / valbytes; \ + \ + if (UPB_LIKELY(!arr)) { \ + *arr_p = arr = \ + UPB_PRIVATE(_upb_Array_New)(&d->arena, elems, elem_size_lg2); \ + if (!arr) { \ + _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ + } \ + } else { \ + _upb_Array_ResizeUninitialized(arr, elems, &d->arena); \ + } \ + \ + char* dst = _upb_array_ptr(arr); \ + memcpy(dst, ptr, size); \ + arr->UPB_PRIVATE(size) = elems; \ + \ + ptr += size; \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); + +#define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, unpacked, packed) \ + if (card == CARD_p) { \ + FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, unpacked); \ + } else { \ + FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, packed); \ + } + +/* Generate all combinations: + * {s,o,r,p} x {f4,f8} x {1bt,2bt} */ + +#define F(card, valbytes, tagbytes) \ + UPB_NOINLINE \ + const char* upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ + FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \ + CARD_##card, upb_ppf##valbytes##_##tagbytes##bt, \ + upb_prf##valbytes##_##tagbytes##bt); \ + } + +#define TYPES(card, tagbytes) \ + F(card, 4, tagbytes) \ + F(card, 8, tagbytes) + +#define TAGBYTES(card) \ + TYPES(card, 1) \ + TYPES(card, 2) + +TAGBYTES(s) +TAGBYTES(o) +TAGBYTES(r) +TAGBYTES(p) + +#undef F +#undef TYPES +#undef TAGBYTES +#undef FASTDECODE_UNPACKEDFIXED +#undef FASTDECODE_PACKEDFIXED + +/* string fields **************************************************************/ + +typedef const char* fastdecode_copystr_func(struct upb_Decoder* d, + const char* ptr, upb_Message* msg, + const upb_MiniTable* table, + uint64_t hasbits, + upb_StringView* dst); + +UPB_NOINLINE +static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr, + upb_Message* msg, intptr_t table, + uint64_t hasbits, uint64_t data) { + upb_StringView* dst = (upb_StringView*)data; + if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { + _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); + } + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); +} + +#define FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, validate_utf8) \ + int size = (uint8_t)ptr[0]; /* Could plumb through hasbits. */ \ + ptr++; \ + if (size & 0x80) { \ + ptr = fastdecode_longsize(ptr, &size); \ + } \ + \ + if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, size))) { \ + dst->size = 0; \ + _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ + } \ + \ + const char* s_ptr = ptr; \ + ptr = upb_EpsCopyInputStream_ReadString(&d->input, &s_ptr, size, &d->arena); \ + if (!ptr) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); \ + dst->data = s_ptr; \ + dst->size = size; \ + \ + if (validate_utf8) { \ + data = (uint64_t)dst; \ + UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ + } else { \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ + } + +UPB_NOINLINE +static const char* fastdecode_longstring_utf8(struct upb_Decoder* d, + const char* ptr, upb_Message* msg, + intptr_t table, uint64_t hasbits, + uint64_t data) { + upb_StringView* dst = (upb_StringView*)data; + FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, true); +} + +UPB_NOINLINE +static const char* fastdecode_longstring_noutf8( + struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, + uint64_t hasbits, uint64_t data) { + upb_StringView* dst = (upb_StringView*)data; + FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false); +} + +UPB_FORCEINLINE +static void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size, + int copy, char* data, size_t data_offset, + upb_StringView* dst) { + d->arena.UPB_PRIVATE(ptr) += copy; + dst->data = data + data_offset; + UPB_UNPOISON_MEMORY_REGION(data, copy); + memcpy(data, ptr, copy); + UPB_POISON_MEMORY_REGION(data + data_offset + size, + copy - data_offset - size); +} + +#define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ + card, validate_utf8) \ + upb_StringView* dst; \ + fastdecode_arr farr; \ + int64_t size; \ + size_t arena_has; \ + size_t common_has; \ + char* buf; \ + \ + UPB_ASSERT(!upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0)); \ + UPB_ASSERT(fastdecode_checktag(data, tagbytes)); \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ + sizeof(upb_StringView), card); \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \ + } \ + \ + size = (uint8_t)ptr[tagbytes]; \ + ptr += tagbytes + 1; \ + dst->size = size; \ + \ + buf = d->arena.UPB_PRIVATE(ptr); \ + arena_has = UPB_PRIVATE(_upb_ArenaHas)(&d->arena); \ + common_has = UPB_MIN(arena_has, \ + upb_EpsCopyInputStream_BytesAvailable(&d->input, ptr)); \ + \ + if (UPB_LIKELY(size <= 15 - tagbytes)) { \ + if (arena_has < 16) goto longstr; \ + fastdecode_docopy(d, ptr - tagbytes - 1, size, 16, buf, tagbytes + 1, \ + dst); \ + } else if (UPB_LIKELY(size <= 32)) { \ + if (UPB_UNLIKELY(common_has < 32)) goto longstr; \ + fastdecode_docopy(d, ptr, size, 32, buf, 0, dst); \ + } else if (UPB_LIKELY(size <= 64)) { \ + if (UPB_UNLIKELY(common_has < 64)) goto longstr; \ + fastdecode_docopy(d, ptr, size, 64, buf, 0, dst); \ + } else if (UPB_LIKELY(size < 128)) { \ + if (UPB_UNLIKELY(common_has < 128)) goto longstr; \ + fastdecode_docopy(d, ptr, size, 128, buf, 0, dst); \ + } else { \ + goto longstr; \ + } \ + \ + ptr += size; \ + \ + if (card == CARD_r) { \ + if (validate_utf8 && \ + !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \ + _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \ + } \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ + if (card != CARD_r && validate_utf8) { \ + data = (uint64_t)dst; \ + UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ + \ + longstr: \ + if (card == CARD_r) { \ + fastdecode_commitarr(dst + 1, &farr, sizeof(upb_StringView)); \ + } \ + ptr--; \ + if (validate_utf8) { \ + UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table, \ + hasbits, (uint64_t)dst); \ + } else { \ + UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table, \ + hasbits, (uint64_t)dst); \ + } + +#define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card, \ + copyfunc, validate_utf8) \ + upb_StringView* dst; \ + fastdecode_arr farr; \ + int64_t size; \ + \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + RETURN_GENERIC("string field tag mismatch\n"); \ + } \ + \ + if (UPB_UNLIKELY( \ + !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0))) { \ + UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS); \ + } \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ + sizeof(upb_StringView), card); \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \ + } \ + \ + size = (int8_t)ptr[tagbytes]; \ + ptr += tagbytes + 1; \ + \ + if (UPB_UNLIKELY( \ + !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, size))) { \ + ptr--; \ + if (validate_utf8) { \ + return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, \ + (uint64_t)dst); \ + } else { \ + return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, \ + (uint64_t)dst); \ + } \ + } \ + \ + dst->data = ptr; \ + dst->size = size; \ + ptr = upb_EpsCopyInputStream_ReadStringAliased(&d->input, &dst->data, \ + dst->size); \ + \ + if (card == CARD_r) { \ + if (validate_utf8 && \ + !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \ + _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \ + } \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ + if (card != CARD_r && validate_utf8) { \ + data = (uint64_t)dst; \ + UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); + +/* Generate all combinations: + * {p,c} x {s,o,r} x {s, b} x {1bt,2bt} */ + +#define s_VALIDATE true +#define b_VALIDATE false + +#define F(card, tagbytes, type) \ + UPB_NOINLINE \ + const char* upb_c##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ + FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ + CARD_##card, type##_VALIDATE); \ + } \ + const char* upb_p##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ + FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, \ + CARD_##card, upb_c##card##type##_##tagbytes##bt, \ + type##_VALIDATE); \ + } + +#define UTF8(card, tagbytes) \ + F(card, tagbytes, s) \ + F(card, tagbytes, b) + +#define TAGBYTES(card) \ + UTF8(card, 1) \ + UTF8(card, 2) + +TAGBYTES(s) +TAGBYTES(o) +TAGBYTES(r) + +#undef s_VALIDATE +#undef b_VALIDATE +#undef F +#undef TAGBYTES +#undef FASTDECODE_LONGSTRING +#undef FASTDECODE_COPYSTRING +#undef FASTDECODE_STRING + +/* message fields *************************************************************/ + +UPB_INLINE +upb_Message* decode_newmsg_ceil(upb_Decoder* d, const upb_MiniTable* m, + int msg_ceil_bytes) { + size_t size = m->UPB_PRIVATE(size) + sizeof(upb_Message_Internal); + char* msg_data; + if (UPB_LIKELY(msg_ceil_bytes > 0 && + UPB_PRIVATE(_upb_ArenaHas)(&d->arena) >= msg_ceil_bytes)) { + UPB_ASSERT(size <= (size_t)msg_ceil_bytes); + msg_data = d->arena.UPB_PRIVATE(ptr); + d->arena.UPB_PRIVATE(ptr) += size; + UPB_UNPOISON_MEMORY_REGION(msg_data, msg_ceil_bytes); + memset(msg_data, 0, msg_ceil_bytes); + UPB_POISON_MEMORY_REGION(msg_data + size, msg_ceil_bytes - size); + } else { + msg_data = (char*)upb_Arena_Malloc(&d->arena, size); + memset(msg_data, 0, size); + } + return msg_data + sizeof(upb_Message_Internal); +} + +typedef struct { + intptr_t table; + upb_Message* msg; +} fastdecode_submsgdata; + +UPB_FORCEINLINE +static const char* fastdecode_tosubmsg(upb_EpsCopyInputStream* e, + const char* ptr, void* ctx) { + upb_Decoder* d = (upb_Decoder*)e; + fastdecode_submsgdata* submsg = ctx; + ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0, 0); + UPB_ASSUME(ptr != NULL); + return ptr; +} + +#define FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, \ + msg_ceil_bytes, card) \ + \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + RETURN_GENERIC("submessage field tag mismatch\n"); \ + } \ + \ + if (--d->depth == 0) { \ + _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_MaxDepthExceeded); \ + } \ + \ + upb_Message** dst; \ + uint32_t submsg_idx = (data >> 16) & 0xff; \ + const upb_MiniTable* tablep = decode_totablep(table); \ + const upb_MiniTable* subtablep = upb_MiniTableSub_Message( \ + *UPB_PRIVATE(_upb_MiniTable_GetSubByIndex)(tablep, submsg_idx)); \ + fastdecode_submsgdata submsg = {decode_totable(subtablep)}; \ + fastdecode_arr farr; \ + \ + if (subtablep->UPB_PRIVATE(table_mask) == (uint8_t)-1) { \ + d->depth++; \ + RETURN_GENERIC("submessage doesn't have fast tables."); \ + } \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ + sizeof(upb_Message*), card); \ + \ + if (card == CARD_s) { \ + *(uint32_t*)msg |= hasbits; \ + hasbits = 0; \ + } \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_Message*)); \ + } \ + \ + submsg.msg = *dst; \ + \ + if (card == CARD_r || UPB_LIKELY(!submsg.msg)) { \ + *dst = submsg.msg = decode_newmsg_ceil(d, subtablep, msg_ceil_bytes); \ + } \ + \ + ptr += tagbytes; \ + ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg); \ + \ + if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) { \ + _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ + } \ + \ + if (card == CARD_r) { \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_Message*)); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + d->depth++; \ + data = ret.tag; \ + UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + d->depth++; \ + return ptr; \ + } \ + } \ + \ + d->depth++; \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); + +#define F(card, tagbytes, size_ceil, ceil_arg) \ + const char* upb_p##card##m_##tagbytes##bt_max##size_ceil##b( \ + UPB_PARSE_PARAMS) { \ + FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, ceil_arg, \ + CARD_##card); \ + } + +#define SIZES(card, tagbytes) \ + F(card, tagbytes, 64, 64) \ + F(card, tagbytes, 128, 128) \ + F(card, tagbytes, 192, 192) \ + F(card, tagbytes, 256, 256) \ + F(card, tagbytes, max, -1) + +#define TAGBYTES(card) \ + SIZES(card, 1) \ + SIZES(card, 2) + +TAGBYTES(s) +TAGBYTES(o) +TAGBYTES(r) + +#undef TAGBYTES +#undef SIZES +#undef F +#undef FASTDECODE_SUBMSG + +#endif /* UPB_FASTTABLE */ + + +#include <stddef.h> +#include <stdint.h> + + +// Must be last. + +UPB_NOINLINE UPB_PRIVATE(_upb_WireReader_LongVarint) + UPB_PRIVATE(_upb_WireReader_ReadLongVarint)(const char* ptr, uint64_t val) { + UPB_PRIVATE(_upb_WireReader_LongVarint) ret = {NULL, 0}; + uint64_t byte; + int i; + for (i = 1; i < 10; i++) { + byte = (uint8_t)ptr[i]; + val += (byte - 1) << (i * 7); + if (!(byte & 0x80)) { + ret.ptr = ptr + i + 1; + ret.val = val; + return ret; + } + } + return ret; +} + +const char* UPB_PRIVATE(_upb_WireReader_SkipGroup)( + const char* ptr, uint32_t tag, int depth_limit, + upb_EpsCopyInputStream* stream) { + if (--depth_limit == 0) return NULL; + uint32_t end_group_tag = (tag & ~7ULL) | kUpb_WireType_EndGroup; + while (!upb_EpsCopyInputStream_IsDone(stream, &ptr)) { + uint32_t tag; + ptr = upb_WireReader_ReadTag(ptr, &tag); + if (!ptr) return NULL; + if (tag == end_group_tag) return ptr; + ptr = _upb_WireReader_SkipValue(ptr, tag, depth_limit, stream); + if (!ptr) return NULL; + } + return ptr; +} + +/* + * upb_table Implementation + * + * Implementation is heavily inspired by Lua's ltable.c. + */ + +#include <string.h> + + +// Must be last. + +#define UPB_MAXARRSIZE 16 // 2**16 = 64k. + +// From Chromium. +#define ARRAY_SIZE(x) \ + ((sizeof(x) / sizeof(0 [x])) / ((size_t)(!(sizeof(x) % sizeof(0 [x]))))) + +static const double MAX_LOAD = 0.85; + +/* The minimum utilization of the array part of a mixed hash/array table. This + * is a speed/memory-usage tradeoff (though it's not straightforward because of + * cache effects). The lower this is, the more memory we'll use. */ +static const double MIN_DENSITY = 0.1; + +static bool is_pow2(uint64_t v) { return v == 0 || (v & (v - 1)) == 0; } + +static upb_value _upb_value_val(uint64_t val) { + upb_value ret; + _upb_value_setval(&ret, val); + return ret; +} + +static int log2ceil(uint64_t v) { + int ret = 0; + bool pow2 = is_pow2(v); + while (v >>= 1) ret++; + ret = pow2 ? ret : ret + 1; // Ceiling. + return UPB_MIN(UPB_MAXARRSIZE, ret); +} + +/* A type to represent the lookup key of either a strtable or an inttable. */ +typedef union { + uintptr_t num; + struct { + const char* str; + size_t len; + } str; +} lookupkey_t; + +static lookupkey_t strkey2(const char* str, size_t len) { + lookupkey_t k; + k.str.str = str; + k.str.len = len; + return k; +} + +static lookupkey_t intkey(uintptr_t key) { + lookupkey_t k; + k.num = key; + return k; +} + +typedef uint32_t hashfunc_t(upb_tabkey key); +typedef bool eqlfunc_t(upb_tabkey k1, lookupkey_t k2); + +/* Base table (shared code) ***************************************************/ + +static uint32_t upb_inthash(uintptr_t key) { return (uint32_t)key; } + +static const upb_tabent* upb_getentry(const upb_table* t, uint32_t hash) { + return t->entries + (hash & t->mask); +} + +static bool upb_arrhas(upb_tabval key) { return key.val != (uint64_t)-1; } + +static bool isfull(upb_table* t) { return t->count == t->max_count; } + +static bool init(upb_table* t, uint8_t size_lg2, upb_Arena* a) { + size_t bytes; + + t->count = 0; + t->size_lg2 = size_lg2; + t->mask = upb_table_size(t) ? upb_table_size(t) - 1 : 0; + t->max_count = upb_table_size(t) * MAX_LOAD; + bytes = upb_table_size(t) * sizeof(upb_tabent); + if (bytes > 0) { + t->entries = upb_Arena_Malloc(a, bytes); + if (!t->entries) return false; + memset(t->entries, 0, bytes); + } else { + t->entries = NULL; + } + return true; +} + +static upb_tabent* emptyent(upb_table* t, upb_tabent* e) { + upb_tabent* begin = t->entries; + upb_tabent* end = begin + upb_table_size(t); + for (e = e + 1; e < end; e++) { + if (upb_tabent_isempty(e)) return e; + } + for (e = begin; e < end; e++) { + if (upb_tabent_isempty(e)) return e; + } + UPB_ASSERT(false); + return NULL; +} + +static upb_tabent* getentry_mutable(upb_table* t, uint32_t hash) { + return (upb_tabent*)upb_getentry(t, hash); +} + +static const upb_tabent* findentry(const upb_table* t, lookupkey_t key, + uint32_t hash, eqlfunc_t* eql) { + const upb_tabent* e; + + if (t->size_lg2 == 0) return NULL; + e = upb_getentry(t, hash); + if (upb_tabent_isempty(e)) return NULL; + while (1) { + if (eql(e->key, key)) return e; + if ((e = e->next) == NULL) return NULL; + } +} + +static upb_tabent* findentry_mutable(upb_table* t, lookupkey_t key, + uint32_t hash, eqlfunc_t* eql) { + return (upb_tabent*)findentry(t, key, hash, eql); +} + +static bool lookup(const upb_table* t, lookupkey_t key, upb_value* v, + uint32_t hash, eqlfunc_t* eql) { + const upb_tabent* e = findentry(t, key, hash, eql); + if (e) { + if (v) { + _upb_value_setval(v, e->val.val); + } + return true; + } else { + return false; + } +} + +/* The given key must not already exist in the table. */ +static void insert(upb_table* t, lookupkey_t key, upb_tabkey tabkey, + upb_value val, uint32_t hash, hashfunc_t* hashfunc, + eqlfunc_t* eql) { + upb_tabent* mainpos_e; + upb_tabent* our_e; + + UPB_ASSERT(findentry(t, key, hash, eql) == NULL); + + t->count++; + mainpos_e = getentry_mutable(t, hash); + our_e = mainpos_e; + + if (upb_tabent_isempty(mainpos_e)) { + /* Our main position is empty; use it. */ + our_e->next = NULL; + } else { + /* Collision. */ + upb_tabent* new_e = emptyent(t, mainpos_e); + /* Head of collider's chain. */ + upb_tabent* chain = getentry_mutable(t, hashfunc(mainpos_e->key)); + if (chain == mainpos_e) { + /* Existing ent is in its main position (it has the same hash as us, and + * is the head of our chain). Insert to new ent and append to this chain. + */ + new_e->next = mainpos_e->next; + mainpos_e->next = new_e; + our_e = new_e; + } else { + /* Existing ent is not in its main position (it is a node in some other + * chain). This implies that no existing ent in the table has our hash. + * Evict it (updating its chain) and use its ent for head of our chain. */ + *new_e = *mainpos_e; /* copies next. */ + while (chain->next != mainpos_e) { + chain = (upb_tabent*)chain->next; + UPB_ASSERT(chain); + } + chain->next = new_e; + our_e = mainpos_e; + our_e->next = NULL; + } + } + our_e->key = tabkey; + our_e->val.val = val.val; + UPB_ASSERT(findentry(t, key, hash, eql) == our_e); +} + +static bool rm(upb_table* t, lookupkey_t key, upb_value* val, + upb_tabkey* removed, uint32_t hash, eqlfunc_t* eql) { + upb_tabent* chain = getentry_mutable(t, hash); + if (upb_tabent_isempty(chain)) return false; + if (eql(chain->key, key)) { + /* Element to remove is at the head of its chain. */ + t->count--; + if (val) _upb_value_setval(val, chain->val.val); + if (removed) *removed = chain->key; + if (chain->next) { + upb_tabent* move = (upb_tabent*)chain->next; + *chain = *move; + move->key = 0; /* Make the slot empty. */ + } else { + chain->key = 0; /* Make the slot empty. */ + } + return true; + } else { + /* Element to remove is either in a non-head position or not in the + * table. */ + while (chain->next && !eql(chain->next->key, key)) { + chain = (upb_tabent*)chain->next; + } + if (chain->next) { + /* Found element to remove. */ + upb_tabent* rm = (upb_tabent*)chain->next; + t->count--; + if (val) _upb_value_setval(val, chain->next->val.val); + if (removed) *removed = rm->key; + rm->key = 0; /* Make the slot empty. */ + chain->next = rm->next; + return true; + } else { + /* Element to remove is not in the table. */ + return false; + } + } +} + +static size_t next(const upb_table* t, size_t i) { + do { + if (++i >= upb_table_size(t)) return SIZE_MAX - 1; /* Distinct from -1. */ + } while (upb_tabent_isempty(&t->entries[i])); + + return i; +} + +static size_t begin(const upb_table* t) { return next(t, -1); } + +/* upb_strtable ***************************************************************/ + +/* A simple "subclass" of upb_table that only adds a hash function for strings. + */ + +static upb_tabkey strcopy(lookupkey_t k2, upb_Arena* a) { + uint32_t len = (uint32_t)k2.str.len; + char* str = upb_Arena_Malloc(a, k2.str.len + sizeof(uint32_t) + 1); + if (str == NULL) return 0; + memcpy(str, &len, sizeof(uint32_t)); + if (k2.str.len) memcpy(str + sizeof(uint32_t), k2.str.str, k2.str.len); + str[sizeof(uint32_t) + k2.str.len] = '\0'; + return (uintptr_t)str; +} + +/* Adapted from ABSL's wyhash. */ + +static uint64_t UnalignedLoad64(const void* p) { + uint64_t val; + memcpy(&val, p, 8); + return val; +} + +static uint32_t UnalignedLoad32(const void* p) { + uint32_t val; + memcpy(&val, p, 4); + return val; +} + +#if defined(_MSC_VER) && defined(_M_X64) +#include <intrin.h> +#endif + +/* Computes a * b, returning the low 64 bits of the result and storing the high + * 64 bits in |*high|. */ +static uint64_t upb_umul128(uint64_t v0, uint64_t v1, uint64_t* out_high) { +#ifdef __SIZEOF_INT128__ + __uint128_t p = v0; + p *= v1; + *out_high = (uint64_t)(p >> 64); + return (uint64_t)p; +#elif defined(_MSC_VER) && defined(_M_X64) + return _umul128(v0, v1, out_high); +#else + uint64_t a32 = v0 >> 32; + uint64_t a00 = v0 & 0xffffffff; + uint64_t b32 = v1 >> 32; + uint64_t b00 = v1 & 0xffffffff; + uint64_t high = a32 * b32; + uint64_t low = a00 * b00; + uint64_t mid1 = a32 * b00; + uint64_t mid2 = a00 * b32; + low += (mid1 << 32) + (mid2 << 32); + // Omit carry bit, for mixing we do not care about exact numerical precision. + high += (mid1 >> 32) + (mid2 >> 32); + *out_high = high; + return low; +#endif +} + +static uint64_t WyhashMix(uint64_t v0, uint64_t v1) { + uint64_t high; + uint64_t low = upb_umul128(v0, v1, &high); + return low ^ high; +} + +static uint64_t Wyhash(const void* data, size_t len, uint64_t seed, + const uint64_t salt[]) { + const uint8_t* ptr = (const uint8_t*)data; + uint64_t starting_length = (uint64_t)len; + uint64_t current_state = seed ^ salt[0]; + + if (len > 64) { + // If we have more than 64 bytes, we're going to handle chunks of 64 + // bytes at a time. We're going to build up two separate hash states + // which we will then hash together. + uint64_t duplicated_state = current_state; + + do { + uint64_t a = UnalignedLoad64(ptr); + uint64_t b = UnalignedLoad64(ptr + 8); + uint64_t c = UnalignedLoad64(ptr + 16); + uint64_t d = UnalignedLoad64(ptr + 24); + uint64_t e = UnalignedLoad64(ptr + 32); + uint64_t f = UnalignedLoad64(ptr + 40); + uint64_t g = UnalignedLoad64(ptr + 48); + uint64_t h = UnalignedLoad64(ptr + 56); + + uint64_t cs0 = WyhashMix(a ^ salt[1], b ^ current_state); + uint64_t cs1 = WyhashMix(c ^ salt[2], d ^ current_state); + current_state = (cs0 ^ cs1); + + uint64_t ds0 = WyhashMix(e ^ salt[3], f ^ duplicated_state); + uint64_t ds1 = WyhashMix(g ^ salt[4], h ^ duplicated_state); + duplicated_state = (ds0 ^ ds1); + + ptr += 64; + len -= 64; + } while (len > 64); + + current_state = current_state ^ duplicated_state; + } + + // We now have a data `ptr` with at most 64 bytes and the current state + // of the hashing state machine stored in current_state. + while (len > 16) { + uint64_t a = UnalignedLoad64(ptr); + uint64_t b = UnalignedLoad64(ptr + 8); + + current_state = WyhashMix(a ^ salt[1], b ^ current_state); + + ptr += 16; + len -= 16; + } + + // We now have a data `ptr` with at most 16 bytes. + uint64_t a = 0; + uint64_t b = 0; + if (len > 8) { + // When we have at least 9 and at most 16 bytes, set A to the first 64 + // bits of the input and B to the last 64 bits of the input. Yes, they will + // overlap in the middle if we are working with less than the full 16 + // bytes. + a = UnalignedLoad64(ptr); + b = UnalignedLoad64(ptr + len - 8); + } else if (len > 3) { + // If we have at least 4 and at most 8 bytes, set A to the first 32 + // bits and B to the last 32 bits. + a = UnalignedLoad32(ptr); + b = UnalignedLoad32(ptr + len - 4); + } else if (len > 0) { + // If we have at least 1 and at most 3 bytes, read all of the provided + // bits into A, with some adjustments. + a = ((ptr[0] << 16) | (ptr[len >> 1] << 8) | ptr[len - 1]); + b = 0; + } else { + a = 0; + b = 0; + } + + uint64_t w = WyhashMix(a ^ salt[1], b ^ current_state); + uint64_t z = salt[1] ^ starting_length; + return WyhashMix(w, z); +} + +const uint64_t kWyhashSalt[5] = { + 0x243F6A8885A308D3ULL, 0x13198A2E03707344ULL, 0xA4093822299F31D0ULL, + 0x082EFA98EC4E6C89ULL, 0x452821E638D01377ULL, +}; + +uint32_t _upb_Hash(const void* p, size_t n, uint64_t seed) { + return Wyhash(p, n, seed, kWyhashSalt); +} + +static uint32_t _upb_Hash_NoSeed(const char* p, size_t n) { + return _upb_Hash(p, n, 0); +} + +static uint32_t strhash(upb_tabkey key) { + uint32_t len; + char* str = upb_tabstr(key, &len); + return _upb_Hash_NoSeed(str, len); +} + +static bool streql(upb_tabkey k1, lookupkey_t k2) { + uint32_t len; + char* str = upb_tabstr(k1, &len); + return len == k2.str.len && (len == 0 || memcmp(str, k2.str.str, len) == 0); +} + +bool upb_strtable_init(upb_strtable* t, size_t expected_size, upb_Arena* a) { + // Multiply by approximate reciprocal of MAX_LOAD (0.85), with pow2 + // denominator. + size_t need_entries = (expected_size + 1) * 1204 / 1024; + UPB_ASSERT(need_entries >= expected_size * 0.85); + int size_lg2 = upb_Log2Ceiling(need_entries); + return init(&t->t, size_lg2, a); +} + +void upb_strtable_clear(upb_strtable* t) { + size_t bytes = upb_table_size(&t->t) * sizeof(upb_tabent); + t->t.count = 0; + memset((char*)t->t.entries, 0, bytes); +} + +bool upb_strtable_resize(upb_strtable* t, size_t size_lg2, upb_Arena* a) { + upb_strtable new_table; + if (!init(&new_table.t, size_lg2, a)) return false; + + intptr_t iter = UPB_STRTABLE_BEGIN; + upb_StringView key; + upb_value val; + while (upb_strtable_next2(t, &key, &val, &iter)) { + upb_strtable_insert(&new_table, key.data, key.size, val, a); + } + *t = new_table; + return true; +} + +bool upb_strtable_insert(upb_strtable* t, const char* k, size_t len, + upb_value v, upb_Arena* a) { + lookupkey_t key; + upb_tabkey tabkey; + uint32_t hash; + + if (isfull(&t->t)) { + /* Need to resize. New table of double the size, add old elements to it. */ + if (!upb_strtable_resize(t, t->t.size_lg2 + 1, a)) { + return false; + } + } + + key = strkey2(k, len); + tabkey = strcopy(key, a); + if (tabkey == 0) return false; + + hash = _upb_Hash_NoSeed(key.str.str, key.str.len); + insert(&t->t, key, tabkey, v, hash, &strhash, &streql); + return true; +} + +bool upb_strtable_lookup2(const upb_strtable* t, const char* key, size_t len, + upb_value* v) { + uint32_t hash = _upb_Hash_NoSeed(key, len); + return lookup(&t->t, strkey2(key, len), v, hash, &streql); +} + +bool upb_strtable_remove2(upb_strtable* t, const char* key, size_t len, + upb_value* val) { + uint32_t hash = _upb_Hash_NoSeed(key, len); + upb_tabkey tabkey; + return rm(&t->t, strkey2(key, len), val, &tabkey, hash, &streql); +} + +/* Iteration */ + +void upb_strtable_begin(upb_strtable_iter* i, const upb_strtable* t) { + i->t = t; + i->index = begin(&t->t); +} + +void upb_strtable_next(upb_strtable_iter* i) { + i->index = next(&i->t->t, i->index); +} + +bool upb_strtable_done(const upb_strtable_iter* i) { + if (!i->t) return true; + return i->index >= upb_table_size(&i->t->t) || + upb_tabent_isempty(str_tabent(i)); +} + +upb_StringView upb_strtable_iter_key(const upb_strtable_iter* i) { + upb_StringView key; + uint32_t len; + UPB_ASSERT(!upb_strtable_done(i)); + key.data = upb_tabstr(str_tabent(i)->key, &len); + key.size = len; + return key; +} + +upb_value upb_strtable_iter_value(const upb_strtable_iter* i) { + UPB_ASSERT(!upb_strtable_done(i)); + return _upb_value_val(str_tabent(i)->val.val); +} + +void upb_strtable_iter_setdone(upb_strtable_iter* i) { + i->t = NULL; + i->index = SIZE_MAX; +} + +bool upb_strtable_iter_isequal(const upb_strtable_iter* i1, + const upb_strtable_iter* i2) { + if (upb_strtable_done(i1) && upb_strtable_done(i2)) return true; + return i1->t == i2->t && i1->index == i2->index; +} + +/* upb_inttable ***************************************************************/ + +/* For inttables we use a hybrid structure where small keys are kept in an + * array and large keys are put in the hash table. */ + +static uint32_t inthash(upb_tabkey key) { return upb_inthash(key); } + +static bool inteql(upb_tabkey k1, lookupkey_t k2) { return k1 == k2.num; } + +static upb_tabval* mutable_array(upb_inttable* t) { + return (upb_tabval*)t->array; +} + +static upb_tabval* inttable_val(upb_inttable* t, uintptr_t key) { + if (key < t->array_size) { + return upb_arrhas(t->array[key]) ? &(mutable_array(t)[key]) : NULL; + } else { + upb_tabent* e = + findentry_mutable(&t->t, intkey(key), upb_inthash(key), &inteql); + return e ? &e->val : NULL; + } +} + +static const upb_tabval* inttable_val_const(const upb_inttable* t, + uintptr_t key) { + return inttable_val((upb_inttable*)t, key); +} + +size_t upb_inttable_count(const upb_inttable* t) { + return t->t.count + t->array_count; +} + +static void check(upb_inttable* t) { + UPB_UNUSED(t); +#if defined(UPB_DEBUG_TABLE) && !defined(NDEBUG) + { + // This check is very expensive (makes inserts/deletes O(N)). + size_t count = 0; + intptr_t iter = UPB_INTTABLE_BEGIN; + uintptr_t key; + upb_value val; + while (upb_inttable_next(t, &key, &val, &iter)) { + UPB_ASSERT(upb_inttable_lookup(t, key, NULL)); + } + UPB_ASSERT(count == upb_inttable_count(t)); + } +#endif +} + +bool upb_inttable_sizedinit(upb_inttable* t, size_t asize, int hsize_lg2, + upb_Arena* a) { + size_t array_bytes; + + if (!init(&t->t, hsize_lg2, a)) return false; + /* Always make the array part at least 1 long, so that we know key 0 + * won't be in the hash part, which simplifies things. */ + t->array_size = UPB_MAX(1, asize); + t->array_count = 0; + array_bytes = t->array_size * sizeof(upb_value); + t->array = upb_Arena_Malloc(a, array_bytes); + if (!t->array) { + return false; + } + memset(mutable_array(t), 0xff, array_bytes); + check(t); + return true; +} + +bool upb_inttable_init(upb_inttable* t, upb_Arena* a) { + return upb_inttable_sizedinit(t, 0, 4, a); +} + +bool upb_inttable_insert(upb_inttable* t, uintptr_t key, upb_value val, + upb_Arena* a) { + upb_tabval tabval; + tabval.val = val.val; + UPB_ASSERT( + upb_arrhas(tabval)); /* This will reject (uint64_t)-1. Fix this. */ + + if (key < t->array_size) { + UPB_ASSERT(!upb_arrhas(t->array[key])); + t->array_count++; + mutable_array(t)[key].val = val.val; + } else { + if (isfull(&t->t)) { + /* Need to resize the hash part, but we re-use the array part. */ + size_t i; + upb_table new_table; + + if (!init(&new_table, t->t.size_lg2 + 1, a)) { + return false; + } + + for (i = begin(&t->t); i < upb_table_size(&t->t); i = next(&t->t, i)) { + const upb_tabent* e = &t->t.entries[i]; + uint32_t hash; + upb_value v; + + _upb_value_setval(&v, e->val.val); + hash = upb_inthash(e->key); + insert(&new_table, intkey(e->key), e->key, v, hash, &inthash, &inteql); + } + + UPB_ASSERT(t->t.count == new_table.count); + + t->t = new_table; + } + insert(&t->t, intkey(key), key, val, upb_inthash(key), &inthash, &inteql); + } + check(t); + return true; +} + +bool upb_inttable_lookup(const upb_inttable* t, uintptr_t key, upb_value* v) { + const upb_tabval* table_v = inttable_val_const(t, key); + if (!table_v) return false; + if (v) _upb_value_setval(v, table_v->val); + return true; +} + +bool upb_inttable_replace(upb_inttable* t, uintptr_t key, upb_value val) { + upb_tabval* table_v = inttable_val(t, key); + if (!table_v) return false; + table_v->val = val.val; + return true; +} + +bool upb_inttable_remove(upb_inttable* t, uintptr_t key, upb_value* val) { + bool success; + if (key < t->array_size) { + if (upb_arrhas(t->array[key])) { + upb_tabval empty = UPB_TABVALUE_EMPTY_INIT; + t->array_count--; + if (val) { + _upb_value_setval(val, t->array[key].val); + } + mutable_array(t)[key] = empty; + success = true; + } else { + success = false; + } + } else { + success = rm(&t->t, intkey(key), val, NULL, upb_inthash(key), &inteql); + } + check(t); + return success; +} + +void upb_inttable_compact(upb_inttable* t, upb_Arena* a) { + /* A power-of-two histogram of the table keys. */ + size_t counts[UPB_MAXARRSIZE + 1] = {0}; + + /* The max key in each bucket. */ + uintptr_t max[UPB_MAXARRSIZE + 1] = {0}; + + { + intptr_t iter = UPB_INTTABLE_BEGIN; + uintptr_t key; + upb_value val; + while (upb_inttable_next(t, &key, &val, &iter)) { + int bucket = log2ceil(key); + max[bucket] = UPB_MAX(max[bucket], key); + counts[bucket]++; + } + } + + /* Find the largest power of two that satisfies the MIN_DENSITY + * definition (while actually having some keys). */ + size_t arr_count = upb_inttable_count(t); + int size_lg2; + upb_inttable new_t; + + for (size_lg2 = ARRAY_SIZE(counts) - 1; size_lg2 > 0; size_lg2--) { + if (counts[size_lg2] == 0) { + /* We can halve again without losing any entries. */ + continue; + } else if (arr_count >= (1 << size_lg2) * MIN_DENSITY) { + break; + } + + arr_count -= counts[size_lg2]; + } + + UPB_ASSERT(arr_count <= upb_inttable_count(t)); + + { + /* Insert all elements into new, perfectly-sized table. */ + size_t arr_size = max[size_lg2] + 1; /* +1 so arr[max] will fit. */ + size_t hash_count = upb_inttable_count(t) - arr_count; + size_t hash_size = hash_count ? (hash_count / MAX_LOAD) + 1 : 0; + int hashsize_lg2 = log2ceil(hash_size); + + upb_inttable_sizedinit(&new_t, arr_size, hashsize_lg2, a); + + { + intptr_t iter = UPB_INTTABLE_BEGIN; + uintptr_t key; + upb_value val; + while (upb_inttable_next(t, &key, &val, &iter)) { + upb_inttable_insert(&new_t, key, val, a); + } + } + + UPB_ASSERT(new_t.array_size == arr_size); + UPB_ASSERT(new_t.t.size_lg2 == hashsize_lg2); + } + *t = new_t; +} + +// Iteration. + +bool upb_inttable_next(const upb_inttable* t, uintptr_t* key, upb_value* val, + intptr_t* iter) { + intptr_t i = *iter; + if ((size_t)(i + 1) <= t->array_size) { + while ((size_t)++i < t->array_size) { + upb_tabval ent = t->array[i]; + if (upb_arrhas(ent)) { + *key = i; + *val = _upb_value_val(ent.val); + *iter = i; + return true; + } + } + i--; // Back up to exactly one position before the start of the table. + } + + size_t tab_idx = next(&t->t, i - t->array_size); + if (tab_idx < upb_table_size(&t->t)) { + upb_tabent* ent = &t->t.entries[tab_idx]; + *key = ent->key; + *val = _upb_value_val(ent->val.val); + *iter = tab_idx + t->array_size; + return true; + } + + return false; +} + +void upb_inttable_removeiter(upb_inttable* t, intptr_t* iter) { + intptr_t i = *iter; + if ((size_t)i < t->array_size) { + t->array_count--; + mutable_array(t)[i].val = -1; + } else { + upb_tabent* ent = &t->t.entries[i - t->array_size]; + upb_tabent* prev = NULL; + + // Linear search, not great. + upb_tabent* end = &t->t.entries[upb_table_size(&t->t)]; + for (upb_tabent* e = t->t.entries; e != end; e++) { + if (e->next == ent) { + prev = e; + break; + } + } + + if (prev) { + prev->next = ent->next; + } + + t->t.count--; + ent->key = 0; + ent->next = NULL; + } +} + +bool upb_strtable_next2(const upb_strtable* t, upb_StringView* key, + upb_value* val, intptr_t* iter) { + size_t tab_idx = next(&t->t, *iter); + if (tab_idx < upb_table_size(&t->t)) { + upb_tabent* ent = &t->t.entries[tab_idx]; + uint32_t len; + key->data = upb_tabstr(ent->key, &len); + key->size = len; + *val = _upb_value_val(ent->val.val); + *iter = tab_idx; + return true; + } + + return false; +} + +void upb_strtable_removeiter(upb_strtable* t, intptr_t* iter) { + intptr_t i = *iter; + upb_tabent* ent = &t->t.entries[i]; + upb_tabent* prev = NULL; + + // Linear search, not great. + upb_tabent* end = &t->t.entries[upb_table_size(&t->t)]; + for (upb_tabent* e = t->t.entries; e != end; e++) { + if (e->next == ent) { + prev = e; + break; + } + } + + if (prev) { + prev->next = ent->next; + } + + t->t.count--; + ent->key = 0; + ent->next = NULL; +} + +void upb_strtable_setentryvalue(upb_strtable* t, intptr_t iter, upb_value v) { + upb_tabent* ent = &t->t.entries[iter]; + ent->val.val = v.val; +} + + +// Must be last. + +const char* upb_BufToUint64(const char* ptr, const char* end, uint64_t* val) { + uint64_t u64 = 0; + while (ptr < end) { + unsigned ch = *ptr - '0'; + if (ch >= 10) break; + if (u64 > UINT64_MAX / 10 || u64 * 10 > UINT64_MAX - ch) { + return NULL; // integer overflow + } + u64 *= 10; + u64 += ch; + ptr++; + } + + *val = u64; + return ptr; +} + +const char* upb_BufToInt64(const char* ptr, const char* end, int64_t* val, + bool* is_neg) { + bool neg = false; + uint64_t u64; + + if (ptr != end && *ptr == '-') { + ptr++; + neg = true; + } + + ptr = upb_BufToUint64(ptr, end, &u64); + if (!ptr || u64 > (uint64_t)INT64_MAX + neg) { + return NULL; // integer overflow + } + + *val = neg ? -u64 : u64; + if (is_neg) *is_neg = neg; + return ptr; +} + + +#include <float.h> +#include <stdlib.h> + +// Must be last. + +/* Miscellaneous utilities ****************************************************/ + +static void upb_FixLocale(char* p) { + /* printf() is dependent on locales; sadly there is no easy and portable way + * to avoid this. This little post-processing step will translate 1,2 -> 1.2 + * since JSON needs the latter. Arguably a hack, but it is simple and the + * alternatives are far more complicated, platform-dependent, and/or larger + * in code size. */ + for (; *p; p++) { + if (*p == ',') *p = '.'; + } +} + +void _upb_EncodeRoundTripDouble(double val, char* buf, size_t size) { + assert(size >= kUpb_RoundTripBufferSize); + snprintf(buf, size, "%.*g", DBL_DIG, val); + if (strtod(buf, NULL) != val) { + snprintf(buf, size, "%.*g", DBL_DIG + 2, val); + assert(strtod(buf, NULL) == val); + } + upb_FixLocale(buf); +} + +void _upb_EncodeRoundTripFloat(float val, char* buf, size_t size) { + assert(size >= kUpb_RoundTripBufferSize); + snprintf(buf, size, "%.*g", FLT_DIG, val); + if (strtof(buf, NULL) != val) { + snprintf(buf, size, "%.*g", FLT_DIG + 3, val); + assert(strtof(buf, NULL) == val); + } + upb_FixLocale(buf); +} + + +#include <stdlib.h> +#include <string.h> + +// Must be last. + +// Determine the locale-specific radix character by calling sprintf() to print +// the number 1.5, then stripping off the digits. As far as I can tell, this +// is the only portable, thread-safe way to get the C library to divulge the +// locale's radix character. No, localeconv() is NOT thread-safe. + +static int GetLocaleRadix(char *data, size_t capacity) { + char temp[16]; + const int size = snprintf(temp, sizeof(temp), "%.1f", 1.5); + UPB_ASSERT(temp[0] == '1'); + UPB_ASSERT(temp[size - 1] == '5'); + UPB_ASSERT(size < capacity); + temp[size - 1] = '\0'; + strcpy(data, temp + 1); + return size - 2; +} + +// Populates a string identical to *input except that the character pointed to +// by pos (which should be '.') is replaced with the locale-specific radix. + +static void LocalizeRadix(const char *input, const char *pos, char *output) { + const int len1 = pos - input; + + char radix[8]; + const int len2 = GetLocaleRadix(radix, sizeof(radix)); + + memcpy(output, input, len1); + memcpy(output + len1, radix, len2); + strcpy(output + len1 + len2, input + len1 + 1); +} + +double _upb_NoLocaleStrtod(const char *str, char **endptr) { + // We cannot simply set the locale to "C" temporarily with setlocale() + // as this is not thread-safe. Instead, we try to parse in the current + // locale first. If parsing stops at a '.' character, then this is a + // pretty good hint that we're actually in some other locale in which + // '.' is not the radix character. + + char *temp_endptr; + double result = strtod(str, &temp_endptr); + if (endptr != NULL) *endptr = temp_endptr; + if (*temp_endptr != '.') return result; + + // Parsing halted on a '.'. Perhaps we're in a different locale? Let's + // try to replace the '.' with a locale-specific radix character and + // try again. + + char localized[80]; + LocalizeRadix(str, temp_endptr, localized); + char *localized_endptr; + result = strtod(localized, &localized_endptr); + if ((localized_endptr - &localized[0]) > (temp_endptr - str)) { + // This attempt got further, so replacing the decimal must have helped. + // Update endptr to point at the right location. + if (endptr != NULL) { + // size_diff is non-zero if the localized radix has multiple bytes. + int size_diff = strlen(localized) - strlen(str); + *endptr = (char *)str + (localized_endptr - &localized[0] - size_diff); + } + } + + return result; +} + + +// Must be last. + +int upb_Unicode_ToUTF8(uint32_t cp, char* out) { + if (cp <= 0x7f) { + out[0] = cp; + return 1; + } + if (cp <= 0x07ff) { + out[0] = (cp >> 6) | 0xc0; + out[1] = (cp & 0x3f) | 0x80; + return 2; + } + if (cp <= 0xffff) { + out[0] = (cp >> 12) | 0xe0; + out[1] = ((cp >> 6) & 0x3f) | 0x80; + out[2] = (cp & 0x3f) | 0x80; + return 3; + } + if (cp <= 0x10ffff) { + out[0] = (cp >> 18) | 0xf0; + out[1] = ((cp >> 12) & 0x3f) | 0x80; + out[2] = ((cp >> 6) & 0x3f) | 0x80; + out[3] = (cp & 0x3f) | 0x80; + return 4; + } + return 0; +} + + +#include <string.h> + + +// Must be last. + +const struct upb_Extension* _upb_Message_Getext( + const struct upb_Message* msg, const upb_MiniTableExtension* e) { + size_t n; + const struct upb_Extension* ext = UPB_PRIVATE(_upb_Message_Getexts)(msg, &n); + + // For now we use linear search exclusively to find extensions. + // If this becomes an issue due to messages with lots of extensions, + // we can introduce a table of some sort. + for (size_t i = 0; i < n; i++) { + if (ext[i].ext == e) { + return &ext[i]; + } + } + + return NULL; +} + +const struct upb_Extension* UPB_PRIVATE(_upb_Message_Getexts)( + const struct upb_Message* msg, size_t* count) { + upb_Message_InternalData* in = upb_Message_GetInternalData(msg); + if (in) { + *count = (in->size - in->ext_begin) / sizeof(struct upb_Extension); + return UPB_PTR_AT(in, in->ext_begin, void); + } else { + *count = 0; + return NULL; + } +} + +struct upb_Extension* _upb_Message_GetOrCreateExtension( + struct upb_Message* msg, const upb_MiniTableExtension* e, upb_Arena* a) { + struct upb_Extension* ext = + (struct upb_Extension*)_upb_Message_Getext(msg, e); + if (ext) return ext; + if (!UPB_PRIVATE(_upb_Message_Realloc)(msg, sizeof(struct upb_Extension), a)) + return NULL; + upb_Message_InternalData* in = upb_Message_GetInternalData(msg); + in->ext_begin -= sizeof(struct upb_Extension); + ext = UPB_PTR_AT(in, in->ext_begin, void); + memset(ext, 0, sizeof(struct upb_Extension)); + ext->ext = e; + return ext; +} + + +#include <math.h> +#include <string.h> + + +// Must be last. + +const float kUpb_FltInfinity = INFINITY; +const double kUpb_Infinity = INFINITY; +const double kUpb_NaN = NAN; + +bool UPB_PRIVATE(_upb_Message_Realloc)(struct upb_Message* msg, size_t need, + upb_Arena* a) { + const size_t overhead = sizeof(upb_Message_InternalData); + + upb_Message_Internal* owner = upb_Message_Getinternal(msg); + upb_Message_InternalData* in = owner->internal; + if (!in) { + // No internal data, allocate from scratch. + size_t size = UPB_MAX(128, upb_Log2CeilingSize(need + overhead)); + in = upb_Arena_Malloc(a, size); + if (!in) return false; + + in->size = size; + in->unknown_end = overhead; + in->ext_begin = size; + owner->internal = in; + } else if (in->ext_begin - in->unknown_end < need) { + // Internal data is too small, reallocate. + size_t new_size = upb_Log2CeilingSize(in->size + need); + size_t ext_bytes = in->size - in->ext_begin; + size_t new_ext_begin = new_size - ext_bytes; + in = upb_Arena_Realloc(a, in, in->size, new_size); + if (!in) return false; + + if (ext_bytes) { + // Need to move extension data to the end. + char* ptr = (char*)in; + memmove(ptr + new_ext_begin, ptr + in->ext_begin, ext_bytes); + } + in->ext_begin = new_ext_begin; + in->size = new_size; + owner->internal = in; + } + + UPB_ASSERT(in->ext_begin - in->unknown_end >= need); + return true; +} + + +const char _kUpb_ToBase92[] = { + ' ', '!', '#', '$', '%', '&', '(', ')', '*', '+', ',', '-', '.', '/', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', + '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', + 'Z', '[', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '{', '|', '}', '~', +}; + +const int8_t _kUpb_FromBase92[] = { + 0, 1, -1, 2, 3, 4, 5, -1, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, -1, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, + 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, +}; + + +#include <assert.h> +#include <stddef.h> +#include <stdint.h> + + +// Must be last. + +typedef struct { + uint64_t present_values_mask; + uint32_t last_written_value; +} upb_MtDataEncoderInternal_EnumState; + +typedef struct { + uint64_t msg_modifiers; + uint32_t last_field_num; + enum { + kUpb_OneofState_NotStarted, + kUpb_OneofState_StartedOneof, + kUpb_OneofState_EmittedOneofField, + } oneof_state; +} upb_MtDataEncoderInternal_MsgState; + +typedef struct { + char* buf_start; // Only for checking kUpb_MtDataEncoder_MinSize. + union { + upb_MtDataEncoderInternal_EnumState enum_state; + upb_MtDataEncoderInternal_MsgState msg_state; + } state; +} upb_MtDataEncoderInternal; + +static upb_MtDataEncoderInternal* upb_MtDataEncoder_GetInternal( + upb_MtDataEncoder* e, char* buf_start) { + UPB_ASSERT(sizeof(upb_MtDataEncoderInternal) <= sizeof(e->internal)); + upb_MtDataEncoderInternal* ret = (upb_MtDataEncoderInternal*)e->internal; + ret->buf_start = buf_start; + return ret; +} + +static char* upb_MtDataEncoder_PutRaw(upb_MtDataEncoder* e, char* ptr, + char ch) { + upb_MtDataEncoderInternal* in = (upb_MtDataEncoderInternal*)e->internal; + UPB_ASSERT(ptr - in->buf_start < kUpb_MtDataEncoder_MinSize); + if (ptr == e->end) return NULL; + *ptr++ = ch; + return ptr; +} + +static char* upb_MtDataEncoder_Put(upb_MtDataEncoder* e, char* ptr, char ch) { + return upb_MtDataEncoder_PutRaw(e, ptr, _upb_ToBase92(ch)); +} + +static char* upb_MtDataEncoder_PutBase92Varint(upb_MtDataEncoder* e, char* ptr, + uint32_t val, int min, int max) { + int shift = upb_Log2Ceiling(_upb_FromBase92(max) - _upb_FromBase92(min) + 1); + UPB_ASSERT(shift <= 6); + uint32_t mask = (1 << shift) - 1; + do { + uint32_t bits = val & mask; + ptr = upb_MtDataEncoder_Put(e, ptr, bits + _upb_FromBase92(min)); + if (!ptr) return NULL; + val >>= shift; + } while (val); + return ptr; +} + +char* upb_MtDataEncoder_PutModifier(upb_MtDataEncoder* e, char* ptr, + uint64_t mod) { + if (mod) { + ptr = upb_MtDataEncoder_PutBase92Varint(e, ptr, mod, + kUpb_EncodedValue_MinModifier, + kUpb_EncodedValue_MaxModifier); + } + return ptr; +} + +char* upb_MtDataEncoder_EncodeExtension(upb_MtDataEncoder* e, char* ptr, + upb_FieldType type, uint32_t field_num, + uint64_t field_mod) { + upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); + in->state.msg_state.msg_modifiers = 0; + in->state.msg_state.last_field_num = 0; + in->state.msg_state.oneof_state = kUpb_OneofState_NotStarted; + + ptr = upb_MtDataEncoder_PutRaw(e, ptr, kUpb_EncodedVersion_ExtensionV1); + if (!ptr) return NULL; + + return upb_MtDataEncoder_PutField(e, ptr, type, field_num, field_mod); +} + +char* upb_MtDataEncoder_EncodeMap(upb_MtDataEncoder* e, char* ptr, + upb_FieldType key_type, + upb_FieldType value_type, uint64_t key_mod, + uint64_t value_mod) { + upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); + in->state.msg_state.msg_modifiers = 0; + in->state.msg_state.last_field_num = 0; + in->state.msg_state.oneof_state = kUpb_OneofState_NotStarted; + + ptr = upb_MtDataEncoder_PutRaw(e, ptr, kUpb_EncodedVersion_MapV1); + if (!ptr) return NULL; + + ptr = upb_MtDataEncoder_PutField(e, ptr, key_type, 1, key_mod); + if (!ptr) return NULL; + + return upb_MtDataEncoder_PutField(e, ptr, value_type, 2, value_mod); +} + +char* upb_MtDataEncoder_EncodeMessageSet(upb_MtDataEncoder* e, char* ptr) { + (void)upb_MtDataEncoder_GetInternal(e, ptr); + return upb_MtDataEncoder_PutRaw(e, ptr, kUpb_EncodedVersion_MessageSetV1); +} + +char* upb_MtDataEncoder_StartMessage(upb_MtDataEncoder* e, char* ptr, + uint64_t msg_mod) { + upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); + in->state.msg_state.msg_modifiers = msg_mod; + in->state.msg_state.last_field_num = 0; + in->state.msg_state.oneof_state = kUpb_OneofState_NotStarted; + + ptr = upb_MtDataEncoder_PutRaw(e, ptr, kUpb_EncodedVersion_MessageV1); + if (!ptr) return NULL; + + return upb_MtDataEncoder_PutModifier(e, ptr, msg_mod); +} + +static char* _upb_MtDataEncoder_MaybePutFieldSkip(upb_MtDataEncoder* e, + char* ptr, + uint32_t field_num) { + upb_MtDataEncoderInternal* in = (upb_MtDataEncoderInternal*)e->internal; + if (field_num <= in->state.msg_state.last_field_num) return NULL; + if (in->state.msg_state.last_field_num + 1 != field_num) { + // Put skip. + UPB_ASSERT(field_num > in->state.msg_state.last_field_num); + uint32_t skip = field_num - in->state.msg_state.last_field_num; + ptr = upb_MtDataEncoder_PutBase92Varint( + e, ptr, skip, kUpb_EncodedValue_MinSkip, kUpb_EncodedValue_MaxSkip); + if (!ptr) return NULL; + } + in->state.msg_state.last_field_num = field_num; + return ptr; +} + +static char* _upb_MtDataEncoder_PutFieldType(upb_MtDataEncoder* e, char* ptr, + upb_FieldType type, + uint64_t field_mod) { + static const char kUpb_TypeToEncoded[] = { + [kUpb_FieldType_Double] = kUpb_EncodedType_Double, + [kUpb_FieldType_Float] = kUpb_EncodedType_Float, + [kUpb_FieldType_Int64] = kUpb_EncodedType_Int64, + [kUpb_FieldType_UInt64] = kUpb_EncodedType_UInt64, + [kUpb_FieldType_Int32] = kUpb_EncodedType_Int32, + [kUpb_FieldType_Fixed64] = kUpb_EncodedType_Fixed64, + [kUpb_FieldType_Fixed32] = kUpb_EncodedType_Fixed32, + [kUpb_FieldType_Bool] = kUpb_EncodedType_Bool, + [kUpb_FieldType_String] = kUpb_EncodedType_String, + [kUpb_FieldType_Group] = kUpb_EncodedType_Group, + [kUpb_FieldType_Message] = kUpb_EncodedType_Message, + [kUpb_FieldType_Bytes] = kUpb_EncodedType_Bytes, + [kUpb_FieldType_UInt32] = kUpb_EncodedType_UInt32, + [kUpb_FieldType_Enum] = kUpb_EncodedType_OpenEnum, + [kUpb_FieldType_SFixed32] = kUpb_EncodedType_SFixed32, + [kUpb_FieldType_SFixed64] = kUpb_EncodedType_SFixed64, + [kUpb_FieldType_SInt32] = kUpb_EncodedType_SInt32, + [kUpb_FieldType_SInt64] = kUpb_EncodedType_SInt64, + }; + + int encoded_type = kUpb_TypeToEncoded[type]; + + if (field_mod & kUpb_FieldModifier_IsClosedEnum) { + UPB_ASSERT(type == kUpb_FieldType_Enum); + encoded_type = kUpb_EncodedType_ClosedEnum; + } + + if (field_mod & kUpb_FieldModifier_IsRepeated) { + // Repeated fields shift the type number up (unlike other modifiers which + // are bit flags). + encoded_type += kUpb_EncodedType_RepeatedBase; + } + + return upb_MtDataEncoder_Put(e, ptr, encoded_type); +} + +static char* _upb_MtDataEncoder_MaybePutModifiers(upb_MtDataEncoder* e, + char* ptr, upb_FieldType type, + uint64_t field_mod) { + upb_MtDataEncoderInternal* in = (upb_MtDataEncoderInternal*)e->internal; + uint32_t encoded_modifiers = 0; + if ((field_mod & kUpb_FieldModifier_IsRepeated) && + upb_FieldType_IsPackable(type)) { + bool field_is_packed = field_mod & kUpb_FieldModifier_IsPacked; + bool default_is_packed = in->state.msg_state.msg_modifiers & + kUpb_MessageModifier_DefaultIsPacked; + if (field_is_packed != default_is_packed) { + encoded_modifiers |= kUpb_EncodedFieldModifier_FlipPacked; + } + } + + if (type == kUpb_FieldType_String) { + bool field_validates_utf8 = field_mod & kUpb_FieldModifier_ValidateUtf8; + bool message_validates_utf8 = + in->state.msg_state.msg_modifiers & kUpb_MessageModifier_ValidateUtf8; + if (field_validates_utf8 != message_validates_utf8) { + // Old binaries do not recognize the field modifier. We need the failure + // mode to be too lax rather than too strict. Our caller should have + // handled this (see _upb_MessageDef_ValidateUtf8()). + assert(!message_validates_utf8); + encoded_modifiers |= kUpb_EncodedFieldModifier_FlipValidateUtf8; + } + } + + if (field_mod & kUpb_FieldModifier_IsProto3Singular) { + encoded_modifiers |= kUpb_EncodedFieldModifier_IsProto3Singular; + } + + if (field_mod & kUpb_FieldModifier_IsRequired) { + encoded_modifiers |= kUpb_EncodedFieldModifier_IsRequired; + } + + return upb_MtDataEncoder_PutModifier(e, ptr, encoded_modifiers); +} + +char* upb_MtDataEncoder_PutField(upb_MtDataEncoder* e, char* ptr, + upb_FieldType type, uint32_t field_num, + uint64_t field_mod) { + upb_MtDataEncoder_GetInternal(e, ptr); + + ptr = _upb_MtDataEncoder_MaybePutFieldSkip(e, ptr, field_num); + if (!ptr) return NULL; + + ptr = _upb_MtDataEncoder_PutFieldType(e, ptr, type, field_mod); + if (!ptr) return NULL; + + return _upb_MtDataEncoder_MaybePutModifiers(e, ptr, type, field_mod); +} + +char* upb_MtDataEncoder_StartOneof(upb_MtDataEncoder* e, char* ptr) { + upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); + if (in->state.msg_state.oneof_state == kUpb_OneofState_NotStarted) { + ptr = upb_MtDataEncoder_Put(e, ptr, _upb_FromBase92(kUpb_EncodedValue_End)); + } else { + ptr = upb_MtDataEncoder_Put( + e, ptr, _upb_FromBase92(kUpb_EncodedValue_OneofSeparator)); + } + in->state.msg_state.oneof_state = kUpb_OneofState_StartedOneof; + return ptr; +} + +char* upb_MtDataEncoder_PutOneofField(upb_MtDataEncoder* e, char* ptr, + uint32_t field_num) { + upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); + if (in->state.msg_state.oneof_state == kUpb_OneofState_EmittedOneofField) { + ptr = upb_MtDataEncoder_Put( + e, ptr, _upb_FromBase92(kUpb_EncodedValue_FieldSeparator)); + if (!ptr) return NULL; + } + ptr = upb_MtDataEncoder_PutBase92Varint(e, ptr, field_num, _upb_ToBase92(0), + _upb_ToBase92(63)); + in->state.msg_state.oneof_state = kUpb_OneofState_EmittedOneofField; + return ptr; +} + +char* upb_MtDataEncoder_StartEnum(upb_MtDataEncoder* e, char* ptr) { + upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); + in->state.enum_state.present_values_mask = 0; + in->state.enum_state.last_written_value = 0; + + return upb_MtDataEncoder_PutRaw(e, ptr, kUpb_EncodedVersion_EnumV1); +} + +static char* upb_MtDataEncoder_FlushDenseEnumMask(upb_MtDataEncoder* e, + char* ptr) { + upb_MtDataEncoderInternal* in = (upb_MtDataEncoderInternal*)e->internal; + ptr = upb_MtDataEncoder_Put(e, ptr, in->state.enum_state.present_values_mask); + in->state.enum_state.present_values_mask = 0; + in->state.enum_state.last_written_value += 5; + return ptr; +} + +char* upb_MtDataEncoder_PutEnumValue(upb_MtDataEncoder* e, char* ptr, + uint32_t val) { + // TODO: optimize this encoding. + upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); + UPB_ASSERT(val >= in->state.enum_state.last_written_value); + uint32_t delta = val - in->state.enum_state.last_written_value; + if (delta >= 5 && in->state.enum_state.present_values_mask) { + ptr = upb_MtDataEncoder_FlushDenseEnumMask(e, ptr); + if (!ptr) { + return NULL; + } + delta -= 5; + } + + if (delta >= 5) { + ptr = upb_MtDataEncoder_PutBase92Varint( + e, ptr, delta, kUpb_EncodedValue_MinSkip, kUpb_EncodedValue_MaxSkip); + in->state.enum_state.last_written_value += delta; + delta = 0; + } + + UPB_ASSERT((in->state.enum_state.present_values_mask >> delta) == 0); + in->state.enum_state.present_values_mask |= 1ULL << delta; + return ptr; +} + +char* upb_MtDataEncoder_EndEnum(upb_MtDataEncoder* e, char* ptr) { + upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); + if (!in->state.enum_state.present_values_mask) return ptr; + return upb_MtDataEncoder_FlushDenseEnumMask(e, ptr); +} + + +#include <stddef.h> + +// Must be last. + +// A MiniTable for an empty message, used for unlinked sub-messages. +const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty) = { + .UPB_PRIVATE(subs) = NULL, + .UPB_PRIVATE(fields) = NULL, + .UPB_PRIVATE(size) = 0, + .UPB_PRIVATE(field_count) = 0, + .UPB_PRIVATE(ext) = kUpb_ExtMode_NonExtendable, + .UPB_PRIVATE(dense_below) = 0, + .UPB_PRIVATE(table_mask) = -1, + .UPB_PRIVATE(required_count) = 0, +}; + + // Must be last. @@ -12003,3450 +15447,6 @@ return s; } - -#include <assert.h> -#include <stdbool.h> -#include <stddef.h> -#include <stdint.h> -#include <string.h> - - -// Must be last. - -// A few fake field types for our tables. -enum { - kUpb_FakeFieldType_FieldNotFound = 0, - kUpb_FakeFieldType_MessageSetItem = 19, -}; - -// DecodeOp: an action to be performed for a wire-type/field-type combination. -enum { - // Special ops: we don't write data to regular fields for these. - kUpb_DecodeOp_UnknownField = -1, - kUpb_DecodeOp_MessageSetItem = -2, - - // Scalar-only ops. - kUpb_DecodeOp_Scalar1Byte = 0, - kUpb_DecodeOp_Scalar4Byte = 2, - kUpb_DecodeOp_Scalar8Byte = 3, - kUpb_DecodeOp_Enum = 1, - - // Scalar/repeated ops. - kUpb_DecodeOp_String = 4, - kUpb_DecodeOp_Bytes = 5, - kUpb_DecodeOp_SubMessage = 6, - - // Repeated-only ops (also see macros below). - kUpb_DecodeOp_PackedEnum = 13, -}; - -// For packed fields it is helpful to be able to recover the lg2 of the data -// size from the op. -#define OP_FIXPCK_LG2(n) (n + 5) /* n in [2, 3] => op in [7, 8] */ -#define OP_VARPCK_LG2(n) (n + 9) /* n in [0, 2, 3] => op in [9, 11, 12] */ - -typedef union { - bool bool_val; - uint32_t uint32_val; - uint64_t uint64_val; - uint32_t size; -} wireval; - -// Ideally these two functions should take the owning MiniTable pointer as a -// first argument, then we could just put them in mini_table/message.h as nice -// clean getters. But we don't have that so instead we gotta write these -// Frankenfunctions that take an array of subtables. -// TODO: Move these to mini_table/ anyway since there are other places -// that could use them. - -// Returns the MiniTable corresponding to a given MiniTableField -// from an array of MiniTableSubs. -static const upb_MiniTable* _upb_MiniTableSubs_MessageByField( - const upb_MiniTableSub* subs, const upb_MiniTableField* field) { - return upb_MiniTableSub_Message(subs[field->UPB_PRIVATE(submsg_index)]); -} - -// Returns the MiniTableEnum corresponding to a given MiniTableField -// from an array of MiniTableSub. -static const upb_MiniTableEnum* _upb_MiniTableSubs_EnumByField( - const upb_MiniTableSub* subs, const upb_MiniTableField* field) { - return upb_MiniTableSub_Enum(subs[field->UPB_PRIVATE(submsg_index)]); -} - -static const char* _upb_Decoder_DecodeMessage(upb_Decoder* d, const char* ptr, - upb_Message* msg, - const upb_MiniTable* layout); - -UPB_NORETURN static void* _upb_Decoder_ErrorJmp(upb_Decoder* d, - upb_DecodeStatus status) { - UPB_ASSERT(status != kUpb_DecodeStatus_Ok); - d->status = status; - UPB_LONGJMP(d->err, 1); -} - -const char* _upb_FastDecoder_ErrorJmp(upb_Decoder* d, int status) { - UPB_ASSERT(status != kUpb_DecodeStatus_Ok); - d->status = status; - UPB_LONGJMP(d->err, 1); - return NULL; -} - -static void _upb_Decoder_VerifyUtf8(upb_Decoder* d, const char* buf, int len) { - if (!_upb_Decoder_VerifyUtf8Inline(buf, len)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); - } -} - -static bool _upb_Decoder_Reserve(upb_Decoder* d, upb_Array* arr, size_t elem) { - bool need_realloc = - arr->UPB_PRIVATE(capacity) - arr->UPB_PRIVATE(size) < elem; - if (need_realloc && !UPB_PRIVATE(_upb_Array_Realloc)( - arr, arr->UPB_PRIVATE(size) + elem, &d->arena)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - } - return need_realloc; -} - -typedef struct { - const char* ptr; - uint64_t val; -} _upb_DecodeLongVarintReturn; - -UPB_NOINLINE -static _upb_DecodeLongVarintReturn _upb_Decoder_DecodeLongVarint( - const char* ptr, uint64_t val) { - _upb_DecodeLongVarintReturn ret = {NULL, 0}; - uint64_t byte; - int i; - for (i = 1; i < 10; i++) { - byte = (uint8_t)ptr[i]; - val += (byte - 1) << (i * 7); - if (!(byte & 0x80)) { - ret.ptr = ptr + i + 1; - ret.val = val; - return ret; - } - } - return ret; -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_DecodeVarint(upb_Decoder* d, const char* ptr, - uint64_t* val) { - uint64_t byte = (uint8_t)*ptr; - if (UPB_LIKELY((byte & 0x80) == 0)) { - *val = byte; - return ptr + 1; - } else { - _upb_DecodeLongVarintReturn res = _upb_Decoder_DecodeLongVarint(ptr, byte); - if (!res.ptr) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); - *val = res.val; - return res.ptr; - } -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_DecodeTag(upb_Decoder* d, const char* ptr, - uint32_t* val) { - uint64_t byte = (uint8_t)*ptr; - if (UPB_LIKELY((byte & 0x80) == 0)) { - *val = byte; - return ptr + 1; - } else { - const char* start = ptr; - _upb_DecodeLongVarintReturn res = _upb_Decoder_DecodeLongVarint(ptr, byte); - if (!res.ptr || res.ptr - start > 5 || res.val > UINT32_MAX) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); - } - *val = res.val; - return res.ptr; - } -} - -UPB_FORCEINLINE -static const char* upb_Decoder_DecodeSize(upb_Decoder* d, const char* ptr, - uint32_t* size) { - uint64_t size64; - ptr = _upb_Decoder_DecodeVarint(d, ptr, &size64); - if (size64 >= INT32_MAX || - !upb_EpsCopyInputStream_CheckSize(&d->input, ptr, (int)size64)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); - } - *size = size64; - return ptr; -} - -static void _upb_Decoder_MungeInt32(wireval* val) { - if (!UPB_PRIVATE(_upb_IsLittleEndian)()) { - /* The next stage will memcpy(dst, &val, 4) */ - val->uint32_val = val->uint64_val; - } -} - -static void _upb_Decoder_Munge(int type, wireval* val) { - switch (type) { - case kUpb_FieldType_Bool: - val->bool_val = val->uint64_val != 0; - break; - case kUpb_FieldType_SInt32: { - uint32_t n = val->uint64_val; - val->uint32_val = (n >> 1) ^ -(int32_t)(n & 1); - break; - } - case kUpb_FieldType_SInt64: { - uint64_t n = val->uint64_val; - val->uint64_val = (n >> 1) ^ -(int64_t)(n & 1); - break; - } - case kUpb_FieldType_Int32: - case kUpb_FieldType_UInt32: - case kUpb_FieldType_Enum: - _upb_Decoder_MungeInt32(val); - break; - } -} - -static upb_Message* _upb_Decoder_NewSubMessage(upb_Decoder* d, - const upb_MiniTableSub* subs, - const upb_MiniTableField* field, - upb_TaggedMessagePtr* target) { - const upb_MiniTable* subl = _upb_MiniTableSubs_MessageByField(subs, field); - UPB_ASSERT(subl); - upb_Message* msg = _upb_Message_New(subl, &d->arena); - if (!msg) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - - // Extensions should not be unlinked. A message extension should not be - // registered until its sub-message type is available to be linked. - bool is_empty = UPB_PRIVATE(_upb_MiniTable_IsEmpty)(subl); - bool is_extension = field->UPB_PRIVATE(mode) & kUpb_LabelFlags_IsExtension; - UPB_ASSERT(!(is_empty && is_extension)); - - if (is_empty && !(d->options & kUpb_DecodeOption_ExperimentalAllowUnlinked)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_UnlinkedSubMessage); - } - - upb_TaggedMessagePtr tagged = - UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(msg, is_empty); - memcpy(target, &tagged, sizeof(tagged)); - return msg; -} - -static upb_Message* _upb_Decoder_ReuseSubMessage( - upb_Decoder* d, const upb_MiniTableSub* subs, - const upb_MiniTableField* field, upb_TaggedMessagePtr* target) { - upb_TaggedMessagePtr tagged = *target; - const upb_MiniTable* subl = _upb_MiniTableSubs_MessageByField(subs, field); - UPB_ASSERT(subl); - if (!upb_TaggedMessagePtr_IsEmpty(tagged) || - UPB_PRIVATE(_upb_MiniTable_IsEmpty)(subl)) { - return UPB_PRIVATE(_upb_TaggedMessagePtr_GetMessage)(tagged); - } - - // We found an empty message from a previous parse that was performed before - // this field was linked. But it is linked now, so we want to allocate a new - // message of the correct type and promote data into it before continuing. - upb_Message* existing = - UPB_PRIVATE(_upb_TaggedMessagePtr_GetEmptyMessage)(tagged); - upb_Message* promoted = _upb_Decoder_NewSubMessage(d, subs, field, target); - size_t size; - const char* unknown = upb_Message_GetUnknown(existing, &size); - upb_DecodeStatus status = upb_Decode(unknown, size, promoted, subl, d->extreg, - d->options, &d->arena); - if (status != kUpb_DecodeStatus_Ok) _upb_Decoder_ErrorJmp(d, status); - return promoted; -} - -static const char* _upb_Decoder_ReadString(upb_Decoder* d, const char* ptr, - int size, upb_StringView* str) { - const char* str_ptr = ptr; - ptr = upb_EpsCopyInputStream_ReadString(&d->input, &str_ptr, size, &d->arena); - if (!ptr) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - str->data = str_ptr; - str->size = size; - return ptr; -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_RecurseSubMessage(upb_Decoder* d, - const char* ptr, - upb_Message* submsg, - const upb_MiniTable* subl, - uint32_t expected_end_group) { - if (--d->depth < 0) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_MaxDepthExceeded); - } - ptr = _upb_Decoder_DecodeMessage(d, ptr, submsg, subl); - d->depth++; - if (d->end_group != expected_end_group) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); - } - return ptr; -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_DecodeSubMessage( - upb_Decoder* d, const char* ptr, upb_Message* submsg, - const upb_MiniTableSub* subs, const upb_MiniTableField* field, int size) { - int saved_delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, size); - const upb_MiniTable* subl = _upb_MiniTableSubs_MessageByField(subs, field); - UPB_ASSERT(subl); - ptr = _upb_Decoder_RecurseSubMessage(d, ptr, submsg, subl, DECODE_NOGROUP); - upb_EpsCopyInputStream_PopLimit(&d->input, ptr, saved_delta); - return ptr; -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_DecodeGroup(upb_Decoder* d, const char* ptr, - upb_Message* submsg, - const upb_MiniTable* subl, - uint32_t number) { - if (_upb_Decoder_IsDone(d, &ptr)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); - } - ptr = _upb_Decoder_RecurseSubMessage(d, ptr, submsg, subl, number); - d->end_group = DECODE_NOGROUP; - return ptr; -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_DecodeUnknownGroup(upb_Decoder* d, - const char* ptr, - uint32_t number) { - return _upb_Decoder_DecodeGroup(d, ptr, NULL, NULL, number); -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_DecodeKnownGroup( - upb_Decoder* d, const char* ptr, upb_Message* submsg, - const upb_MiniTableSub* subs, const upb_MiniTableField* field) { - const upb_MiniTable* subl = _upb_MiniTableSubs_MessageByField(subs, field); - UPB_ASSERT(subl); - return _upb_Decoder_DecodeGroup(d, ptr, submsg, subl, - field->UPB_PRIVATE(number)); -} - -static char* upb_Decoder_EncodeVarint32(uint32_t val, char* ptr) { - do { - uint8_t byte = val & 0x7fU; - val >>= 7; - if (val) byte |= 0x80U; - *(ptr++) = byte; - } while (val); - return ptr; -} - -static void _upb_Decoder_AddUnknownVarints(upb_Decoder* d, upb_Message* msg, - uint32_t val1, uint32_t val2) { - char buf[20]; - char* end = buf; - end = upb_Decoder_EncodeVarint32(val1, end); - end = upb_Decoder_EncodeVarint32(val2, end); - - if (!UPB_PRIVATE(_upb_Message_AddUnknown)(msg, buf, end - buf, &d->arena)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - } -} - -UPB_FORCEINLINE -static bool _upb_Decoder_CheckEnum(upb_Decoder* d, const char* ptr, - upb_Message* msg, const upb_MiniTableEnum* e, - const upb_MiniTableField* field, - wireval* val) { - const uint32_t v = val->uint32_val; - - if (UPB_LIKELY(upb_MiniTableEnum_CheckValue(e, v))) return true; - - // Unrecognized enum goes into unknown fields. - // For packed fields the tag could be arbitrarily far in the past, - // so we just re-encode the tag and value here. - const uint32_t tag = - ((uint32_t)field->UPB_PRIVATE(number) << 3) | kUpb_WireType_Varint; - upb_Message* unknown_msg = - field->UPB_PRIVATE(mode) & kUpb_LabelFlags_IsExtension ? d->unknown_msg - : msg; - _upb_Decoder_AddUnknownVarints(d, unknown_msg, tag, v); - return false; -} - -UPB_NOINLINE -static const char* _upb_Decoder_DecodeEnumArray(upb_Decoder* d, const char* ptr, - upb_Message* msg, - upb_Array* arr, - const upb_MiniTableSub* subs, - const upb_MiniTableField* field, - wireval* val) { - const upb_MiniTableEnum* e = _upb_MiniTableSubs_EnumByField(subs, field); - if (!_upb_Decoder_CheckEnum(d, ptr, msg, e, field, val)) return ptr; - void* mem = UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) * 4, void); - arr->UPB_PRIVATE(size)++; - memcpy(mem, val, 4); - return ptr; -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_DecodeFixedPacked( - upb_Decoder* d, const char* ptr, upb_Array* arr, wireval* val, - const upb_MiniTableField* field, int lg2) { - int mask = (1 << lg2) - 1; - size_t count = val->size >> lg2; - if ((val->size & mask) != 0) { - // Length isn't a round multiple of elem size. - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); - } - _upb_Decoder_Reserve(d, arr, count); - void* mem = - UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) << lg2, void); - arr->UPB_PRIVATE(size) += count; - // Note: if/when the decoder supports multi-buffer input, we will need to - // handle buffer seams here. - if (UPB_PRIVATE(_upb_IsLittleEndian)()) { - ptr = upb_EpsCopyInputStream_Copy(&d->input, ptr, mem, val->size); - } else { - int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, val->size); - char* dst = mem; - while (!_upb_Decoder_IsDone(d, &ptr)) { - if (lg2 == 2) { - ptr = upb_WireReader_ReadFixed32(ptr, dst); - dst += 4; - } else { - UPB_ASSERT(lg2 == 3); - ptr = upb_WireReader_ReadFixed64(ptr, dst); - dst += 8; - } - } - upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta); - } - - return ptr; -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_DecodeVarintPacked( - upb_Decoder* d, const char* ptr, upb_Array* arr, wireval* val, - const upb_MiniTableField* field, int lg2) { - int scale = 1 << lg2; - int saved_limit = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, val->size); - char* out = - UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) << lg2, void); - while (!_upb_Decoder_IsDone(d, &ptr)) { - wireval elem; - ptr = _upb_Decoder_DecodeVarint(d, ptr, &elem.uint64_val); - _upb_Decoder_Munge(field->UPB_PRIVATE(descriptortype), &elem); - if (_upb_Decoder_Reserve(d, arr, 1)) { - out = - UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) << lg2, void); - } - arr->UPB_PRIVATE(size)++; - memcpy(out, &elem, scale); - out += scale; - } - upb_EpsCopyInputStream_PopLimit(&d->input, ptr, saved_limit); - return ptr; -} - -UPB_NOINLINE -static const char* _upb_Decoder_DecodeEnumPacked( - upb_Decoder* d, const char* ptr, upb_Message* msg, upb_Array* arr, - const upb_MiniTableSub* subs, const upb_MiniTableField* field, - wireval* val) { - const upb_MiniTableEnum* e = _upb_MiniTableSubs_EnumByField(subs, field); - int saved_limit = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, val->size); - char* out = UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) * 4, void); - while (!_upb_Decoder_IsDone(d, &ptr)) { - wireval elem; - ptr = _upb_Decoder_DecodeVarint(d, ptr, &elem.uint64_val); - _upb_Decoder_MungeInt32(&elem); - if (!_upb_Decoder_CheckEnum(d, ptr, msg, e, field, &elem)) { - continue; - } - if (_upb_Decoder_Reserve(d, arr, 1)) { - out = UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) * 4, void); - } - arr->UPB_PRIVATE(size)++; - memcpy(out, &elem, 4); - out += 4; - } - upb_EpsCopyInputStream_PopLimit(&d->input, ptr, saved_limit); - return ptr; -} - -upb_Array* _upb_Decoder_CreateArray(upb_Decoder* d, - const upb_MiniTableField* field) { - const upb_FieldType field_type = field->UPB_PRIVATE(descriptortype); - const size_t lg2 = UPB_PRIVATE(_upb_FieldType_SizeLg2)(field_type); - upb_Array* ret = UPB_PRIVATE(_upb_Array_New)(&d->arena, 4, lg2); - if (!ret) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - return ret; -} - -static const char* _upb_Decoder_DecodeToArray(upb_Decoder* d, const char* ptr, - upb_Message* msg, - const upb_MiniTableSub* subs, - const upb_MiniTableField* field, - wireval* val, int op) { - upb_Array** arrp = UPB_PTR_AT(msg, field->UPB_PRIVATE(offset), void); - upb_Array* arr = *arrp; - void* mem; - - if (arr) { - _upb_Decoder_Reserve(d, arr, 1); - } else { - arr = _upb_Decoder_CreateArray(d, field); - *arrp = arr; - } - - switch (op) { - case kUpb_DecodeOp_Scalar1Byte: - case kUpb_DecodeOp_Scalar4Byte: - case kUpb_DecodeOp_Scalar8Byte: - /* Append scalar value. */ - mem = UPB_PTR_AT(_upb_array_ptr(arr), arr->UPB_PRIVATE(size) << op, void); - arr->UPB_PRIVATE(size)++; - memcpy(mem, val, 1 << op); - return ptr; - case kUpb_DecodeOp_String: - _upb_Decoder_VerifyUtf8(d, ptr, val->size); - /* Fallthrough. */ - case kUpb_DecodeOp_Bytes: { - /* Append bytes. */ - upb_StringView* str = - (upb_StringView*)_upb_array_ptr(arr) + arr->UPB_PRIVATE(size); - arr->UPB_PRIVATE(size)++; - return _upb_Decoder_ReadString(d, ptr, val->size, str); - } - case kUpb_DecodeOp_SubMessage: { - /* Append submessage / group. */ - upb_TaggedMessagePtr* target = UPB_PTR_AT( - _upb_array_ptr(arr), arr->UPB_PRIVATE(size) * sizeof(void*), - upb_TaggedMessagePtr); - upb_Message* submsg = _upb_Decoder_NewSubMessage(d, subs, field, target); - arr->UPB_PRIVATE(size)++; - if (UPB_UNLIKELY(field->UPB_PRIVATE(descriptortype) == - kUpb_FieldType_Group)) { - return _upb_Decoder_DecodeKnownGroup(d, ptr, submsg, subs, field); - } else { - return _upb_Decoder_DecodeSubMessage(d, ptr, submsg, subs, field, - val->size); - } - } - case OP_FIXPCK_LG2(2): - case OP_FIXPCK_LG2(3): - return _upb_Decoder_DecodeFixedPacked(d, ptr, arr, val, field, - op - OP_FIXPCK_LG2(0)); - case OP_VARPCK_LG2(0): - case OP_VARPCK_LG2(2): - case OP_VARPCK_LG2(3): - return _upb_Decoder_DecodeVarintPacked(d, ptr, arr, val, field, - op - OP_VARPCK_LG2(0)); - case kUpb_DecodeOp_Enum: - return _upb_Decoder_DecodeEnumArray(d, ptr, msg, arr, subs, field, val); - case kUpb_DecodeOp_PackedEnum: - return _upb_Decoder_DecodeEnumPacked(d, ptr, msg, arr, subs, field, val); - default: - UPB_UNREACHABLE(); - } -} - -upb_Map* _upb_Decoder_CreateMap(upb_Decoder* d, const upb_MiniTable* entry) { - /* Maps descriptor type -> upb map size. */ - static const uint8_t kSizeInMap[] = { - [0] = -1, // invalid descriptor type */ - [kUpb_FieldType_Double] = 8, - [kUpb_FieldType_Float] = 4, - [kUpb_FieldType_Int64] = 8, - [kUpb_FieldType_UInt64] = 8, - [kUpb_FieldType_Int32] = 4, - [kUpb_FieldType_Fixed64] = 8, - [kUpb_FieldType_Fixed32] = 4, - [kUpb_FieldType_Bool] = 1, - [kUpb_FieldType_String] = UPB_MAPTYPE_STRING, - [kUpb_FieldType_Group] = sizeof(void*), - [kUpb_FieldType_Message] = sizeof(void*), - [kUpb_FieldType_Bytes] = UPB_MAPTYPE_STRING, - [kUpb_FieldType_UInt32] = 4, - [kUpb_FieldType_Enum] = 4, - [kUpb_FieldType_SFixed32] = 4, - [kUpb_FieldType_SFixed64] = 8, - [kUpb_FieldType_SInt32] = 4, - [kUpb_FieldType_SInt64] = 8, - }; - - const upb_MiniTableField* key_field = &entry->UPB_PRIVATE(fields)[0]; - const upb_MiniTableField* val_field = &entry->UPB_PRIVATE(fields)[1]; - char key_size = kSizeInMap[key_field->UPB_PRIVATE(descriptortype)]; - char val_size = kSizeInMap[val_field->UPB_PRIVATE(descriptortype)]; - UPB_ASSERT(key_field->UPB_PRIVATE(offset) == offsetof(upb_MapEntryData, k)); - UPB_ASSERT(val_field->UPB_PRIVATE(offset) == offsetof(upb_MapEntryData, v)); - upb_Map* ret = _upb_Map_New(&d->arena, key_size, val_size); - if (!ret) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - return ret; -} - -static const char* _upb_Decoder_DecodeToMap(upb_Decoder* d, const char* ptr, - upb_Message* msg, - const upb_MiniTableSub* subs, - const upb_MiniTableField* field, - wireval* val) { - upb_Map** map_p = UPB_PTR_AT(msg, field->UPB_PRIVATE(offset), upb_Map*); - upb_Map* map = *map_p; - upb_MapEntry ent; - UPB_ASSERT(upb_MiniTableField_Type(field) == kUpb_FieldType_Message); - const upb_MiniTable* entry = _upb_MiniTableSubs_MessageByField(subs, field); - - UPB_ASSERT(entry); - UPB_ASSERT(entry->UPB_PRIVATE(field_count) == 2); - UPB_ASSERT(upb_MiniTableField_IsScalar(&entry->UPB_PRIVATE(fields)[0])); - UPB_ASSERT(upb_MiniTableField_IsScalar(&entry->UPB_PRIVATE(fields)[1])); - - if (!map) { - map = _upb_Decoder_CreateMap(d, entry); - *map_p = map; - } - - // Parse map entry. - memset(&ent, 0, sizeof(ent)); - - if (entry->UPB_PRIVATE(fields)[1].UPB_PRIVATE(descriptortype) == - kUpb_FieldType_Message || - entry->UPB_PRIVATE(fields)[1].UPB_PRIVATE(descriptortype) == - kUpb_FieldType_Group) { - // Create proactively to handle the case where it doesn't appear. - upb_TaggedMessagePtr msg; - _upb_Decoder_NewSubMessage(d, entry->UPB_PRIVATE(subs), - &entry->UPB_PRIVATE(fields)[1], &msg); - ent.data.v.val = upb_value_uintptr(msg); - } - - ptr = _upb_Decoder_DecodeSubMessage(d, ptr, (upb_Message*)&ent.data, subs, - field, val->size); - // check if ent had any unknown fields - size_t size; - upb_Message_GetUnknown((upb_Message*)&ent.data, &size); - if (size != 0) { - char* buf; - size_t size; - uint32_t tag = - ((uint32_t)field->UPB_PRIVATE(number) << 3) | kUpb_WireType_Delimited; - upb_EncodeStatus status = - upb_Encode((upb_Message*)&ent.data, entry, 0, &d->arena, &buf, &size); - if (status != kUpb_EncodeStatus_Ok) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - } - _upb_Decoder_AddUnknownVarints(d, msg, tag, size); - if (!UPB_PRIVATE(_upb_Message_AddUnknown)(msg, buf, size, &d->arena)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - } - } else { - if (_upb_Map_Insert(map, &ent.data.k, map->key_size, &ent.data.v, - map->val_size, - &d->arena) == kUpb_MapInsertStatus_OutOfMemory) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - } - } - return ptr; -} - -static const char* _upb_Decoder_DecodeToSubMessage( - upb_Decoder* d, const char* ptr, upb_Message* msg, - const upb_MiniTableSub* subs, const upb_MiniTableField* field, wireval* val, - int op) { - void* mem = UPB_PTR_AT(msg, field->UPB_PRIVATE(offset), void); - int type = field->UPB_PRIVATE(descriptortype); - - if (UPB_UNLIKELY(op == kUpb_DecodeOp_Enum) && - !_upb_Decoder_CheckEnum(d, ptr, msg, - _upb_MiniTableSubs_EnumByField(subs, field), - field, val)) { - return ptr; - } - - /* Set presence if necessary. */ - if (field->presence > 0) { - UPB_PRIVATE(_upb_Message_SetHasbit)(msg, field); - } else if (field->presence < 0) { - /* Oneof case */ - uint32_t* oneof_case = UPB_PRIVATE(_upb_Message_OneofCasePtr)(msg, field); - if (op == kUpb_DecodeOp_SubMessage && - *oneof_case != field->UPB_PRIVATE(number)) { - memset(mem, 0, sizeof(void*)); - } - *oneof_case = field->UPB_PRIVATE(number); - } - - /* Store into message. */ - switch (op) { - case kUpb_DecodeOp_SubMessage: { - upb_TaggedMessagePtr* submsgp = mem; - upb_Message* submsg; - if (*submsgp) { - submsg = _upb_Decoder_ReuseSubMessage(d, subs, field, submsgp); - } else { - submsg = _upb_Decoder_NewSubMessage(d, subs, field, submsgp); - } - if (UPB_UNLIKELY(type == kUpb_FieldType_Group)) { - ptr = _upb_Decoder_DecodeKnownGroup(d, ptr, submsg, subs, field); - } else { - ptr = _upb_Decoder_DecodeSubMessage(d, ptr, submsg, subs, field, - val->size); - } - break; - } - case kUpb_DecodeOp_String: - _upb_Decoder_VerifyUtf8(d, ptr, val->size); - /* Fallthrough. */ - case kUpb_DecodeOp_Bytes: - return _upb_Decoder_ReadString(d, ptr, val->size, mem); - case kUpb_DecodeOp_Scalar8Byte: - memcpy(mem, val, 8); - break; - case kUpb_DecodeOp_Enum: - case kUpb_DecodeOp_Scalar4Byte: - memcpy(mem, val, 4); - break; - case kUpb_DecodeOp_Scalar1Byte: - memcpy(mem, val, 1); - break; - default: - UPB_UNREACHABLE(); - } - - return ptr; -} - -UPB_NOINLINE -const char* _upb_Decoder_CheckRequired(upb_Decoder* d, const char* ptr, - const upb_Message* msg, - const upb_MiniTable* m) { - UPB_ASSERT(m->UPB_PRIVATE(required_count)); - if (UPB_LIKELY((d->options & kUpb_DecodeOption_CheckRequired) == 0)) { - return ptr; - } - uint64_t msg_head; - memcpy(&msg_head, msg, 8); - msg_head = UPB_PRIVATE(_upb_BigEndian64)(msg_head); - if (UPB_PRIVATE(_upb_MiniTable_RequiredMask)(m) & ~msg_head) { - d->missing_required = true; - } - return ptr; -} - -UPB_FORCEINLINE -static bool _upb_Decoder_TryFastDispatch(upb_Decoder* d, const char** ptr, - upb_Message* msg, - const upb_MiniTable* m) { -#if UPB_FASTTABLE - if (m && m->UPB_PRIVATE(table_mask) != (unsigned char)-1) { - uint16_t tag = _upb_FastDecoder_LoadTag(*ptr); - intptr_t table = decode_totable(m); - *ptr = _upb_FastDecoder_TagDispatch(d, *ptr, msg, table, 0, tag); - return true; - } -#endif - return false; -} - -static const char* upb_Decoder_SkipField(upb_Decoder* d, const char* ptr, - uint32_t tag) { - int field_number = tag >> 3; - int wire_type = tag & 7; - switch (wire_type) { - case kUpb_WireType_Varint: { - uint64_t val; - return _upb_Decoder_DecodeVarint(d, ptr, &val); - } - case kUpb_WireType_64Bit: - return ptr + 8; - case kUpb_WireType_32Bit: - return ptr + 4; - case kUpb_WireType_Delimited: { - uint32_t size; - ptr = upb_Decoder_DecodeSize(d, ptr, &size); - return ptr + size; - } - case kUpb_WireType_StartGroup: - return _upb_Decoder_DecodeUnknownGroup(d, ptr, field_number); - default: - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); - } -} - -enum { - kStartItemTag = ((kUpb_MsgSet_Item << 3) | kUpb_WireType_StartGroup), - kEndItemTag = ((kUpb_MsgSet_Item << 3) | kUpb_WireType_EndGroup), - kTypeIdTag = ((kUpb_MsgSet_TypeId << 3) | kUpb_WireType_Varint), - kMessageTag = ((kUpb_MsgSet_Message << 3) | kUpb_WireType_Delimited), -}; - -static void upb_Decoder_AddKnownMessageSetItem( - upb_Decoder* d, upb_Message* msg, const upb_MiniTableExtension* item_mt, - const char* data, uint32_t size) { - upb_Extension* ext = - _upb_Message_GetOrCreateExtension(msg, item_mt, &d->arena); - if (UPB_UNLIKELY(!ext)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - } - upb_Message* submsg = _upb_Decoder_NewSubMessage( - d, &ext->ext->UPB_PRIVATE(sub), upb_MiniTableExtension_AsField(ext->ext), - (upb_TaggedMessagePtr*)&ext->data); - upb_DecodeStatus status = upb_Decode( - data, size, submsg, upb_MiniTableExtension_GetSubMessage(item_mt), - d->extreg, d->options, &d->arena); - if (status != kUpb_DecodeStatus_Ok) _upb_Decoder_ErrorJmp(d, status); -} - -static void upb_Decoder_AddUnknownMessageSetItem(upb_Decoder* d, - upb_Message* msg, - uint32_t type_id, - const char* message_data, - uint32_t message_size) { - char buf[60]; - char* ptr = buf; - ptr = upb_Decoder_EncodeVarint32(kStartItemTag, ptr); - ptr = upb_Decoder_EncodeVarint32(kTypeIdTag, ptr); - ptr = upb_Decoder_EncodeVarint32(type_id, ptr); - ptr = upb_Decoder_EncodeVarint32(kMessageTag, ptr); - ptr = upb_Decoder_EncodeVarint32(message_size, ptr); - char* split = ptr; - - ptr = upb_Decoder_EncodeVarint32(kEndItemTag, ptr); - char* end = ptr; - - if (!UPB_PRIVATE(_upb_Message_AddUnknown)(msg, buf, split - buf, &d->arena) || - !UPB_PRIVATE(_upb_Message_AddUnknown)(msg, message_data, message_size, - &d->arena) || - !UPB_PRIVATE(_upb_Message_AddUnknown)(msg, split, end - split, - &d->arena)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - } -} - -static void upb_Decoder_AddMessageSetItem(upb_Decoder* d, upb_Message* msg, - const upb_MiniTable* t, - uint32_t type_id, const char* data, - uint32_t size) { - const upb_MiniTableExtension* item_mt = - upb_ExtensionRegistry_Lookup(d->extreg, t, type_id); - if (item_mt) { - upb_Decoder_AddKnownMessageSetItem(d, msg, item_mt, data, size); - } else { - upb_Decoder_AddUnknownMessageSetItem(d, msg, type_id, data, size); - } -} - -static const char* upb_Decoder_DecodeMessageSetItem( - upb_Decoder* d, const char* ptr, upb_Message* msg, - const upb_MiniTable* layout) { - uint32_t type_id = 0; - upb_StringView preserved = {NULL, 0}; - typedef enum { - kUpb_HaveId = 1 << 0, - kUpb_HavePayload = 1 << 1, - } StateMask; - StateMask state_mask = 0; - while (!_upb_Decoder_IsDone(d, &ptr)) { - uint32_t tag; - ptr = _upb_Decoder_DecodeTag(d, ptr, &tag); - switch (tag) { - case kEndItemTag: - return ptr; - case kTypeIdTag: { - uint64_t tmp; - ptr = _upb_Decoder_DecodeVarint(d, ptr, &tmp); - if (state_mask & kUpb_HaveId) break; // Ignore dup. - state_mask |= kUpb_HaveId; - type_id = tmp; - if (state_mask & kUpb_HavePayload) { - upb_Decoder_AddMessageSetItem(d, msg, layout, type_id, preserved.data, - preserved.size); - } - break; - } - case kMessageTag: { - uint32_t size; - ptr = upb_Decoder_DecodeSize(d, ptr, &size); - const char* data = ptr; - ptr += size; - if (state_mask & kUpb_HavePayload) break; // Ignore dup. - state_mask |= kUpb_HavePayload; - if (state_mask & kUpb_HaveId) { - upb_Decoder_AddMessageSetItem(d, msg, layout, type_id, data, size); - } else { - // Out of order, we must preserve the payload. - preserved.data = data; - preserved.size = size; - } - break; - } - default: - // We do not preserve unexpected fields inside a message set item. - ptr = upb_Decoder_SkipField(d, ptr, tag); - break; - } - } - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); -} - -static const upb_MiniTableField* _upb_Decoder_FindField(upb_Decoder* d, - const upb_MiniTable* t, - uint32_t field_number, - int* last_field_index) { - static upb_MiniTableField none = { - 0, 0, 0, 0, kUpb_FakeFieldType_FieldNotFound, 0}; - if (t == NULL) return &none; - - size_t idx = ((size_t)field_number) - 1; // 0 wraps to SIZE_MAX - if (idx < t->UPB_PRIVATE(dense_below)) { - /* Fastest case: index into dense fields. */ - goto found; - } - - if (t->UPB_PRIVATE(dense_below) < t->UPB_PRIVATE(field_count)) { - /* Linear search non-dense fields. Resume scanning from last_field_index - * since fields are usually in order. */ - size_t last = *last_field_index; - for (idx = last; idx < t->UPB_PRIVATE(field_count); idx++) { - if (t->UPB_PRIVATE(fields)[idx].UPB_PRIVATE(number) == field_number) { - goto found; - } - } - - for (idx = t->UPB_PRIVATE(dense_below); idx < last; idx++) { - if (t->UPB_PRIVATE(fields)[idx].UPB_PRIVATE(number) == field_number) { - goto found; - } - } - } - - if (d->extreg) { - switch (t->UPB_PRIVATE(ext)) { - case kUpb_ExtMode_Extendable: { - const upb_MiniTableExtension* ext = - upb_ExtensionRegistry_Lookup(d->extreg, t, field_number); - if (ext) return upb_MiniTableExtension_AsField(ext); - break; - } - case kUpb_ExtMode_IsMessageSet: - if (field_number == kUpb_MsgSet_Item) { - static upb_MiniTableField item = { - 0, 0, 0, 0, kUpb_FakeFieldType_MessageSetItem, 0}; - return &item; - } - break; - } - } - - return &none; /* Unknown field. */ - -found: - UPB_ASSERT(t->UPB_PRIVATE(fields)[idx].UPB_PRIVATE(number) == field_number); - *last_field_index = idx; - return &t->UPB_PRIVATE(fields)[idx]; -} - -int _upb_Decoder_GetVarintOp(const upb_MiniTableField* field) { - static const int8_t kVarintOps[] = { - [kUpb_FakeFieldType_FieldNotFound] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Double] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Float] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Int64] = kUpb_DecodeOp_Scalar8Byte, - [kUpb_FieldType_UInt64] = kUpb_DecodeOp_Scalar8Byte, - [kUpb_FieldType_Int32] = kUpb_DecodeOp_Scalar4Byte, - [kUpb_FieldType_Fixed64] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Fixed32] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Bool] = kUpb_DecodeOp_Scalar1Byte, - [kUpb_FieldType_String] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Group] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Message] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Bytes] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_UInt32] = kUpb_DecodeOp_Scalar4Byte, - [kUpb_FieldType_Enum] = kUpb_DecodeOp_Enum, - [kUpb_FieldType_SFixed32] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_SFixed64] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_SInt32] = kUpb_DecodeOp_Scalar4Byte, - [kUpb_FieldType_SInt64] = kUpb_DecodeOp_Scalar8Byte, - [kUpb_FakeFieldType_MessageSetItem] = kUpb_DecodeOp_UnknownField, - }; - - return kVarintOps[field->UPB_PRIVATE(descriptortype)]; -} - -UPB_FORCEINLINE -static void _upb_Decoder_CheckUnlinked(upb_Decoder* d, const upb_MiniTable* mt, - const upb_MiniTableField* field, - int* op) { - // If sub-message is not linked, treat as unknown. - if (field->UPB_PRIVATE(mode) & kUpb_LabelFlags_IsExtension) return; - const upb_MiniTable* mt_sub = - _upb_MiniTableSubs_MessageByField(mt->UPB_PRIVATE(subs), field); - if ((d->options & kUpb_DecodeOption_ExperimentalAllowUnlinked) || - !UPB_PRIVATE(_upb_MiniTable_IsEmpty)(mt_sub)) { - return; - } -#ifndef NDEBUG - const upb_MiniTableField* oneof = upb_MiniTable_GetOneof(mt, field); - if (oneof) { - // All other members of the oneof must be message fields that are also - // unlinked. - do { - UPB_ASSERT(upb_MiniTableField_CType(oneof) == kUpb_CType_Message); - const upb_MiniTableSub* oneof_sub = - &mt->UPB_PRIVATE(subs)[oneof->UPB_PRIVATE(submsg_index)]; - UPB_ASSERT(!oneof_sub); - } while (upb_MiniTable_NextOneofField(mt, &oneof)); - } -#endif // NDEBUG - *op = kUpb_DecodeOp_UnknownField; -} - -int _upb_Decoder_GetDelimitedOp(upb_Decoder* d, const upb_MiniTable* mt, - const upb_MiniTableField* field) { - enum { kRepeatedBase = 19 }; - - static const int8_t kDelimitedOps[] = { - /* For non-repeated field type. */ - [kUpb_FakeFieldType_FieldNotFound] = - kUpb_DecodeOp_UnknownField, // Field not found. - [kUpb_FieldType_Double] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Float] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Int64] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_UInt64] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Int32] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Fixed64] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Fixed32] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Bool] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_String] = kUpb_DecodeOp_String, - [kUpb_FieldType_Group] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Message] = kUpb_DecodeOp_SubMessage, - [kUpb_FieldType_Bytes] = kUpb_DecodeOp_Bytes, - [kUpb_FieldType_UInt32] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_Enum] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_SFixed32] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_SFixed64] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_SInt32] = kUpb_DecodeOp_UnknownField, - [kUpb_FieldType_SInt64] = kUpb_DecodeOp_UnknownField, - [kUpb_FakeFieldType_MessageSetItem] = kUpb_DecodeOp_UnknownField, - // For repeated field type. */ - [kRepeatedBase + kUpb_FieldType_Double] = OP_FIXPCK_LG2(3), - [kRepeatedBase + kUpb_FieldType_Float] = OP_FIXPCK_LG2(2), - [kRepeatedBase + kUpb_FieldType_Int64] = OP_VARPCK_LG2(3), - [kRepeatedBase + kUpb_FieldType_UInt64] = OP_VARPCK_LG2(3), - [kRepeatedBase + kUpb_FieldType_Int32] = OP_VARPCK_LG2(2), - [kRepeatedBase + kUpb_FieldType_Fixed64] = OP_FIXPCK_LG2(3), - [kRepeatedBase + kUpb_FieldType_Fixed32] = OP_FIXPCK_LG2(2), - [kRepeatedBase + kUpb_FieldType_Bool] = OP_VARPCK_LG2(0), - [kRepeatedBase + kUpb_FieldType_String] = kUpb_DecodeOp_String, - [kRepeatedBase + kUpb_FieldType_Group] = kUpb_DecodeOp_SubMessage, - [kRepeatedBase + kUpb_FieldType_Message] = kUpb_DecodeOp_SubMessage, - [kRepeatedBase + kUpb_FieldType_Bytes] = kUpb_DecodeOp_Bytes, - [kRepeatedBase + kUpb_FieldType_UInt32] = OP_VARPCK_LG2(2), - [kRepeatedBase + kUpb_FieldType_Enum] = kUpb_DecodeOp_PackedEnum, - [kRepeatedBase + kUpb_FieldType_SFixed32] = OP_FIXPCK_LG2(2), - [kRepeatedBase + kUpb_FieldType_SFixed64] = OP_FIXPCK_LG2(3), - [kRepeatedBase + kUpb_FieldType_SInt32] = OP_VARPCK_LG2(2), - [kRepeatedBase + kUpb_FieldType_SInt64] = OP_VARPCK_LG2(3), - // Omitting kUpb_FakeFieldType_MessageSetItem, because we never emit a - // repeated msgset type - }; - - int ndx = field->UPB_PRIVATE(descriptortype); - if (upb_MiniTableField_IsArray(field)) ndx += kRepeatedBase; - int op = kDelimitedOps[ndx]; - - if (op == kUpb_DecodeOp_SubMessage) { - _upb_Decoder_CheckUnlinked(d, mt, field, &op); - } - - return op; -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_DecodeWireValue(upb_Decoder* d, const char* ptr, - const upb_MiniTable* mt, - const upb_MiniTableField* field, - int wire_type, wireval* val, - int* op) { - static const unsigned kFixed32OkMask = (1 << kUpb_FieldType_Float) | - (1 << kUpb_FieldType_Fixed32) | - (1 << kUpb_FieldType_SFixed32); - - static const unsigned kFixed64OkMask = (1 << kUpb_FieldType_Double) | - (1 << kUpb_FieldType_Fixed64) | - (1 << kUpb_FieldType_SFixed64); - - switch (wire_type) { - case kUpb_WireType_Varint: - ptr = _upb_Decoder_DecodeVarint(d, ptr, &val->uint64_val); - *op = _upb_Decoder_GetVarintOp(field); - _upb_Decoder_Munge(field->UPB_PRIVATE(descriptortype), val); - return ptr; - case kUpb_WireType_32Bit: - *op = kUpb_DecodeOp_Scalar4Byte; - if (((1 << field->UPB_PRIVATE(descriptortype)) & kFixed32OkMask) == 0) { - *op = kUpb_DecodeOp_UnknownField; - } - return upb_WireReader_ReadFixed32(ptr, &val->uint32_val); - case kUpb_WireType_64Bit: - *op = kUpb_DecodeOp_Scalar8Byte; - if (((1 << field->UPB_PRIVATE(descriptortype)) & kFixed64OkMask) == 0) { - *op = kUpb_DecodeOp_UnknownField; - } - return upb_WireReader_ReadFixed64(ptr, &val->uint64_val); - case kUpb_WireType_Delimited: - ptr = upb_Decoder_DecodeSize(d, ptr, &val->size); - *op = _upb_Decoder_GetDelimitedOp(d, mt, field); - return ptr; - case kUpb_WireType_StartGroup: - val->uint32_val = field->UPB_PRIVATE(number); - if (field->UPB_PRIVATE(descriptortype) == kUpb_FieldType_Group) { - *op = kUpb_DecodeOp_SubMessage; - _upb_Decoder_CheckUnlinked(d, mt, field, op); - } else if (field->UPB_PRIVATE(descriptortype) == - kUpb_FakeFieldType_MessageSetItem) { - *op = kUpb_DecodeOp_MessageSetItem; - } else { - *op = kUpb_DecodeOp_UnknownField; - } - return ptr; - default: - break; - } - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); -} - -UPB_FORCEINLINE -static const char* _upb_Decoder_DecodeKnownField( - upb_Decoder* d, const char* ptr, upb_Message* msg, - const upb_MiniTable* layout, const upb_MiniTableField* field, int op, - wireval* val) { - const upb_MiniTableSub* subs = layout->UPB_PRIVATE(subs); - uint8_t mode = field->UPB_PRIVATE(mode); - - if (UPB_UNLIKELY(mode & kUpb_LabelFlags_IsExtension)) { - const upb_MiniTableExtension* ext_layout = - (const upb_MiniTableExtension*)field; - upb_Extension* ext = - _upb_Message_GetOrCreateExtension(msg, ext_layout, &d->arena); - if (UPB_UNLIKELY(!ext)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - } - d->unknown_msg = msg; - msg = (upb_Message*)&ext->data; - subs = &ext->ext->UPB_PRIVATE(sub); - } - - switch (mode & kUpb_FieldMode_Mask) { - case kUpb_FieldMode_Array: - return _upb_Decoder_DecodeToArray(d, ptr, msg, subs, field, val, op); - case kUpb_FieldMode_Map: - return _upb_Decoder_DecodeToMap(d, ptr, msg, subs, field, val); - case kUpb_FieldMode_Scalar: - return _upb_Decoder_DecodeToSubMessage(d, ptr, msg, subs, field, val, op); - default: - UPB_UNREACHABLE(); - } -} - -static const char* _upb_Decoder_ReverseSkipVarint(const char* ptr, - uint32_t val) { - uint32_t seen = 0; - do { - ptr--; - seen <<= 7; - seen |= *ptr & 0x7f; - } while (seen != val); - return ptr; -} - -static const char* _upb_Decoder_DecodeUnknownField(upb_Decoder* d, - const char* ptr, - upb_Message* msg, - int field_number, - int wire_type, wireval val) { - if (field_number == 0) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); - - // Since unknown fields are the uncommon case, we do a little extra work here - // to walk backwards through the buffer to find the field start. This frees - // up a register in the fast paths (when the field is known), which leads to - // significant speedups in benchmarks. - const char* start = ptr; - - if (wire_type == kUpb_WireType_Delimited) ptr += val.size; - if (msg) { - switch (wire_type) { - case kUpb_WireType_Varint: - case kUpb_WireType_Delimited: - start--; - while (start[-1] & 0x80) start--; - break; - case kUpb_WireType_32Bit: - start -= 4; - break; - case kUpb_WireType_64Bit: - start -= 8; - break; - default: - break; - } - - assert(start == d->debug_valstart); - uint32_t tag = ((uint32_t)field_number << 3) | wire_type; - start = _upb_Decoder_ReverseSkipVarint(start, tag); - assert(start == d->debug_tagstart); - - if (wire_type == kUpb_WireType_StartGroup) { - d->unknown = start; - d->unknown_msg = msg; - ptr = _upb_Decoder_DecodeUnknownGroup(d, ptr, field_number); - start = d->unknown; - d->unknown = NULL; - } - if (!UPB_PRIVATE(_upb_Message_AddUnknown)(msg, start, ptr - start, - &d->arena)) { - _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - } - } else if (wire_type == kUpb_WireType_StartGroup) { - ptr = _upb_Decoder_DecodeUnknownGroup(d, ptr, field_number); - } - return ptr; -} - -UPB_NOINLINE -static const char* _upb_Decoder_DecodeMessage(upb_Decoder* d, const char* ptr, - upb_Message* msg, - const upb_MiniTable* layout) { - int last_field_index = 0; - -#if UPB_FASTTABLE - // The first time we want to skip fast dispatch, because we may have just been - // invoked by the fast parser to handle a case that it bailed on. - if (!_upb_Decoder_IsDone(d, &ptr)) goto nofast; -#endif - - while (!_upb_Decoder_IsDone(d, &ptr)) { - uint32_t tag; - const upb_MiniTableField* field; - int field_number; - int wire_type; - wireval val; - int op; - - if (_upb_Decoder_TryFastDispatch(d, &ptr, msg, layout)) break; - -#if UPB_FASTTABLE - nofast: -#endif - -#ifndef NDEBUG - d->debug_tagstart = ptr; -#endif - - UPB_ASSERT(ptr < d->input.limit_ptr); - ptr = _upb_Decoder_DecodeTag(d, ptr, &tag); - field_number = tag >> 3; - wire_type = tag & 7; - -#ifndef NDEBUG - d->debug_valstart = ptr; -#endif - - if (wire_type == kUpb_WireType_EndGroup) { - d->end_group = field_number; - return ptr; - } - - field = _upb_Decoder_FindField(d, layout, field_number, &last_field_index); - ptr = _upb_Decoder_DecodeWireValue(d, ptr, layout, field, wire_type, &val, - &op); - - if (op >= 0) { - ptr = _upb_Decoder_DecodeKnownField(d, ptr, msg, layout, field, op, &val); - } else { - switch (op) { - case kUpb_DecodeOp_UnknownField: - ptr = _upb_Decoder_DecodeUnknownField(d, ptr, msg, field_number, - wire_type, val); - break; - case kUpb_DecodeOp_MessageSetItem: - ptr = upb_Decoder_DecodeMessageSetItem(d, ptr, msg, layout); - break; - } - } - } - - return UPB_UNLIKELY(layout && layout->UPB_PRIVATE(required_count)) - ? _upb_Decoder_CheckRequired(d, ptr, msg, layout) - : ptr; -} - -const char* _upb_FastDecoder_DecodeGeneric(struct upb_Decoder* d, - const char* ptr, upb_Message* msg, - intptr_t table, uint64_t hasbits, - uint64_t data) { - (void)data; - *(uint32_t*)msg |= hasbits; - return _upb_Decoder_DecodeMessage(d, ptr, msg, decode_totablep(table)); -} - -static upb_DecodeStatus _upb_Decoder_DecodeTop(struct upb_Decoder* d, - const char* buf, void* msg, - const upb_MiniTable* l) { - if (!_upb_Decoder_TryFastDispatch(d, &buf, msg, l)) { - _upb_Decoder_DecodeMessage(d, buf, msg, l); - } - if (d->end_group != DECODE_NOGROUP) return kUpb_DecodeStatus_Malformed; - if (d->missing_required) return kUpb_DecodeStatus_MissingRequired; - return kUpb_DecodeStatus_Ok; -} - -UPB_NOINLINE -const char* _upb_Decoder_IsDoneFallback(upb_EpsCopyInputStream* e, - const char* ptr, int overrun) { - return _upb_EpsCopyInputStream_IsDoneFallbackInline( - e, ptr, overrun, _upb_Decoder_BufferFlipCallback); -} - -static upb_DecodeStatus upb_Decoder_Decode(upb_Decoder* const decoder, - const char* const buf, - void* const msg, - const upb_MiniTable* const l, - upb_Arena* const arena) { - if (UPB_SETJMP(decoder->err) == 0) { - decoder->status = _upb_Decoder_DecodeTop(decoder, buf, msg, l); - } else { - UPB_ASSERT(decoder->status != kUpb_DecodeStatus_Ok); - } - - UPB_PRIVATE(_upb_Arena_SwapOut)(arena, &decoder->arena); - - return decoder->status; -} - -upb_DecodeStatus upb_Decode(const char* buf, size_t size, upb_Message* msg, - const upb_MiniTable* l, - const upb_ExtensionRegistry* extreg, int options, - upb_Arena* arena) { - upb_Decoder decoder; - unsigned depth = (unsigned)options >> 16; - - upb_EpsCopyInputStream_Init(&decoder.input, &buf, size, - options & kUpb_DecodeOption_AliasString); - - decoder.extreg = extreg; - decoder.unknown = NULL; - decoder.depth = depth ? depth : kUpb_WireFormat_DefaultDepthLimit; - decoder.end_group = DECODE_NOGROUP; - decoder.options = (uint16_t)options; - decoder.missing_required = false; - decoder.status = kUpb_DecodeStatus_Ok; - - // Violating the encapsulation of the arena for performance reasons. - // This is a temporary arena that we swap into and swap out of when we are - // done. The temporary arena only needs to be able to handle allocation, - // not fuse or free, so it does not need many of the members to be initialized - // (particularly parent_or_count). - UPB_PRIVATE(_upb_Arena_SwapIn)(&decoder.arena, arena); - - return upb_Decoder_Decode(&decoder, buf, msg, l, arena); -} - -#undef OP_FIXPCK_LG2 -#undef OP_VARPCK_LG2 - -// We encode backwards, to avoid pre-computing lengths (one-pass encode). - - -#include <setjmp.h> -#include <stdbool.h> -#include <stdint.h> -#include <string.h> - - -// Must be last. - -#define UPB_PB_VARINT_MAX_LEN 10 - -UPB_NOINLINE -static size_t encode_varint64(uint64_t val, char* buf) { - size_t i = 0; - do { - uint8_t byte = val & 0x7fU; - val >>= 7; - if (val) byte |= 0x80U; - buf[i++] = byte; - } while (val); - return i; -} - -static uint32_t encode_zz32(int32_t n) { - return ((uint32_t)n << 1) ^ (n >> 31); -} -static uint64_t encode_zz64(int64_t n) { - return ((uint64_t)n << 1) ^ (n >> 63); -} - -typedef struct { - upb_EncodeStatus status; - jmp_buf err; - upb_Arena* arena; - char *buf, *ptr, *limit; - int options; - int depth; - _upb_mapsorter sorter; -} upb_encstate; - -static size_t upb_roundup_pow2(size_t bytes) { - size_t ret = 128; - while (ret < bytes) { - ret *= 2; - } - return ret; -} - -UPB_NORETURN static void encode_err(upb_encstate* e, upb_EncodeStatus s) { - UPB_ASSERT(s != kUpb_EncodeStatus_Ok); - e->status = s; - UPB_LONGJMP(e->err, 1); -} - -UPB_NOINLINE -static void encode_growbuffer(upb_encstate* e, size_t bytes) { - size_t old_size = e->limit - e->buf; - size_t new_size = upb_roundup_pow2(bytes + (e->limit - e->ptr)); - char* new_buf = upb_Arena_Realloc(e->arena, e->buf, old_size, new_size); - - if (!new_buf) encode_err(e, kUpb_EncodeStatus_OutOfMemory); - - // We want previous data at the end, realloc() put it at the beginning. - // TODO: This is somewhat inefficient since we are copying twice. - // Maybe create a realloc() that copies to the end of the new buffer? - if (old_size > 0) { - memmove(new_buf + new_size - old_size, e->buf, old_size); - } - - e->ptr = new_buf + new_size - (e->limit - e->ptr); - e->limit = new_buf + new_size; - e->buf = new_buf; - - e->ptr -= bytes; -} - -/* Call to ensure that at least "bytes" bytes are available for writing at - * e->ptr. Returns false if the bytes could not be allocated. */ -UPB_FORCEINLINE -static void encode_reserve(upb_encstate* e, size_t bytes) { - if ((size_t)(e->ptr - e->buf) < bytes) { - encode_growbuffer(e, bytes); - return; - } - - e->ptr -= bytes; -} - -/* Writes the given bytes to the buffer, handling reserve/advance. */ -static void encode_bytes(upb_encstate* e, const void* data, size_t len) { - if (len == 0) return; /* memcpy() with zero size is UB */ - encode_reserve(e, len); - memcpy(e->ptr, data, len); -} - -static void encode_fixed64(upb_encstate* e, uint64_t val) { - val = UPB_PRIVATE(_upb_BigEndian64)(val); - encode_bytes(e, &val, sizeof(uint64_t)); -} - -static void encode_fixed32(upb_encstate* e, uint32_t val) { - val = UPB_PRIVATE(_upb_BigEndian32)(val); - encode_bytes(e, &val, sizeof(uint32_t)); -} - -UPB_NOINLINE -static void encode_longvarint(upb_encstate* e, uint64_t val) { - size_t len; - char* start; - - encode_reserve(e, UPB_PB_VARINT_MAX_LEN); - len = encode_varint64(val, e->ptr); - start = e->ptr + UPB_PB_VARINT_MAX_LEN - len; - memmove(start, e->ptr, len); - e->ptr = start; -} - -UPB_FORCEINLINE -static void encode_varint(upb_encstate* e, uint64_t val) { - if (val < 128 && e->ptr != e->buf) { - --e->ptr; - *e->ptr = val; - } else { - encode_longvarint(e, val); - } -} - -static void encode_double(upb_encstate* e, double d) { - uint64_t u64; - UPB_ASSERT(sizeof(double) == sizeof(uint64_t)); - memcpy(&u64, &d, sizeof(uint64_t)); - encode_fixed64(e, u64); -} - -static void encode_float(upb_encstate* e, float d) { - uint32_t u32; - UPB_ASSERT(sizeof(float) == sizeof(uint32_t)); - memcpy(&u32, &d, sizeof(uint32_t)); - encode_fixed32(e, u32); -} - -static void encode_tag(upb_encstate* e, uint32_t field_number, - uint8_t wire_type) { - encode_varint(e, (field_number << 3) | wire_type); -} - -static void encode_fixedarray(upb_encstate* e, const upb_Array* arr, - size_t elem_size, uint32_t tag) { - size_t bytes = arr->UPB_PRIVATE(size) * elem_size; - const char* data = _upb_array_constptr(arr); - const char* ptr = data + bytes - elem_size; - - if (tag || !UPB_PRIVATE(_upb_IsLittleEndian)()) { - while (true) { - if (elem_size == 4) { - uint32_t val; - memcpy(&val, ptr, sizeof(val)); - val = UPB_PRIVATE(_upb_BigEndian32)(val); - encode_bytes(e, &val, elem_size); - } else { - UPB_ASSERT(elem_size == 8); - uint64_t val; - memcpy(&val, ptr, sizeof(val)); - val = UPB_PRIVATE(_upb_BigEndian64)(val); - encode_bytes(e, &val, elem_size); - } - - if (tag) encode_varint(e, tag); - if (ptr == data) break; - ptr -= elem_size; - } - } else { - encode_bytes(e, data, bytes); - } -} - -static void encode_message(upb_encstate* e, const upb_Message* msg, - const upb_MiniTable* m, size_t* size); - -static void encode_TaggedMessagePtr(upb_encstate* e, - upb_TaggedMessagePtr tagged, - const upb_MiniTable* m, size_t* size) { - if (upb_TaggedMessagePtr_IsEmpty(tagged)) { - m = UPB_PRIVATE(_upb_MiniTable_Empty)(); - } - encode_message(e, UPB_PRIVATE(_upb_TaggedMessagePtr_GetMessage)(tagged), m, - size); -} - -static void encode_scalar(upb_encstate* e, const void* _field_mem, - const upb_MiniTableSub* subs, - const upb_MiniTableField* f) { - const char* field_mem = _field_mem; - int wire_type; - -#define CASE(ctype, type, wtype, encodeval) \ - { \ - ctype val = *(ctype*)field_mem; \ - encode_##type(e, encodeval); \ - wire_type = wtype; \ - break; \ - } - - switch (f->UPB_PRIVATE(descriptortype)) { - case kUpb_FieldType_Double: - CASE(double, double, kUpb_WireType_64Bit, val); - case kUpb_FieldType_Float: - CASE(float, float, kUpb_WireType_32Bit, val); - case kUpb_FieldType_Int64: - case kUpb_FieldType_UInt64: - CASE(uint64_t, varint, kUpb_WireType_Varint, val); - case kUpb_FieldType_UInt32: - CASE(uint32_t, varint, kUpb_WireType_Varint, val); - case kUpb_FieldType_Int32: - case kUpb_FieldType_Enum: - CASE(int32_t, varint, kUpb_WireType_Varint, (int64_t)val); - case kUpb_FieldType_SFixed64: - case kUpb_FieldType_Fixed64: - CASE(uint64_t, fixed64, kUpb_WireType_64Bit, val); - case kUpb_FieldType_Fixed32: - case kUpb_FieldType_SFixed32: - CASE(uint32_t, fixed32, kUpb_WireType_32Bit, val); - case kUpb_FieldType_Bool: - CASE(bool, varint, kUpb_WireType_Varint, val); - case kUpb_FieldType_SInt32: - CASE(int32_t, varint, kUpb_WireType_Varint, encode_zz32(val)); - case kUpb_FieldType_SInt64: - CASE(int64_t, varint, kUpb_WireType_Varint, encode_zz64(val)); - case kUpb_FieldType_String: - case kUpb_FieldType_Bytes: { - upb_StringView view = *(upb_StringView*)field_mem; - encode_bytes(e, view.data, view.size); - encode_varint(e, view.size); - wire_type = kUpb_WireType_Delimited; - break; - } - case kUpb_FieldType_Group: { - size_t size; - upb_TaggedMessagePtr submsg = *(upb_TaggedMessagePtr*)field_mem; - const upb_MiniTable* subm = - upb_MiniTableSub_Message(subs[f->UPB_PRIVATE(submsg_index)]); - if (submsg == 0) { - return; - } - if (--e->depth == 0) encode_err(e, kUpb_EncodeStatus_MaxDepthExceeded); - encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_EndGroup); - encode_TaggedMessagePtr(e, submsg, subm, &size); - wire_type = kUpb_WireType_StartGroup; - e->depth++; - break; - } - case kUpb_FieldType_Message: { - size_t size; - upb_TaggedMessagePtr submsg = *(upb_TaggedMessagePtr*)field_mem; - const upb_MiniTable* subm = - upb_MiniTableSub_Message(subs[f->UPB_PRIVATE(submsg_index)]); - if (submsg == 0) { - return; - } - if (--e->depth == 0) encode_err(e, kUpb_EncodeStatus_MaxDepthExceeded); - encode_TaggedMessagePtr(e, submsg, subm, &size); - encode_varint(e, size); - wire_type = kUpb_WireType_Delimited; - e->depth++; - break; - } - default: - UPB_UNREACHABLE(); - } -#undef CASE - - encode_tag(e, f->UPB_PRIVATE(number), wire_type); -} - -static void encode_array(upb_encstate* e, const upb_Message* msg, - const upb_MiniTableSub* subs, - const upb_MiniTableField* f) { - const upb_Array* arr = *UPB_PTR_AT(msg, f->UPB_PRIVATE(offset), upb_Array*); - bool packed = upb_MiniTableField_IsPacked(f); - size_t pre_len = e->limit - e->ptr; - - if (arr == NULL || arr->UPB_PRIVATE(size) == 0) { - return; - } - -#define VARINT_CASE(ctype, encode) \ - { \ - const ctype* start = _upb_array_constptr(arr); \ - const ctype* ptr = start + arr->UPB_PRIVATE(size); \ - uint32_t tag = \ - packed ? 0 : (f->UPB_PRIVATE(number) << 3) | kUpb_WireType_Varint; \ - do { \ - ptr--; \ - encode_varint(e, encode); \ - if (tag) encode_varint(e, tag); \ - } while (ptr != start); \ - } \ - break; - -#define TAG(wire_type) (packed ? 0 : (f->UPB_PRIVATE(number) << 3 | wire_type)) - - switch (f->UPB_PRIVATE(descriptortype)) { - case kUpb_FieldType_Double: - encode_fixedarray(e, arr, sizeof(double), TAG(kUpb_WireType_64Bit)); - break; - case kUpb_FieldType_Float: - encode_fixedarray(e, arr, sizeof(float), TAG(kUpb_WireType_32Bit)); - break; - case kUpb_FieldType_SFixed64: - case kUpb_FieldType_Fixed64: - encode_fixedarray(e, arr, sizeof(uint64_t), TAG(kUpb_WireType_64Bit)); - break; - case kUpb_FieldType_Fixed32: - case kUpb_FieldType_SFixed32: - encode_fixedarray(e, arr, sizeof(uint32_t), TAG(kUpb_WireType_32Bit)); - break; - case kUpb_FieldType_Int64: - case kUpb_FieldType_UInt64: - VARINT_CASE(uint64_t, *ptr); - case kUpb_FieldType_UInt32: - VARINT_CASE(uint32_t, *ptr); - case kUpb_FieldType_Int32: - case kUpb_FieldType_Enum: - VARINT_CASE(int32_t, (int64_t)*ptr); - case kUpb_FieldType_Bool: - VARINT_CASE(bool, *ptr); - case kUpb_FieldType_SInt32: - VARINT_CASE(int32_t, encode_zz32(*ptr)); - case kUpb_FieldType_SInt64: - VARINT_CASE(int64_t, encode_zz64(*ptr)); - case kUpb_FieldType_String: - case kUpb_FieldType_Bytes: { - const upb_StringView* start = _upb_array_constptr(arr); - const upb_StringView* ptr = start + arr->UPB_PRIVATE(size); - do { - ptr--; - encode_bytes(e, ptr->data, ptr->size); - encode_varint(e, ptr->size); - encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_Delimited); - } while (ptr != start); - return; - } - case kUpb_FieldType_Group: { - const upb_TaggedMessagePtr* start = _upb_array_constptr(arr); - const upb_TaggedMessagePtr* ptr = start + arr->UPB_PRIVATE(size); - const upb_MiniTable* subm = - upb_MiniTableSub_Message(subs[f->UPB_PRIVATE(submsg_index)]); - if (--e->depth == 0) encode_err(e, kUpb_EncodeStatus_MaxDepthExceeded); - do { - size_t size; - ptr--; - encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_EndGroup); - encode_TaggedMessagePtr(e, *ptr, subm, &size); - encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_StartGroup); - } while (ptr != start); - e->depth++; - return; - } - case kUpb_FieldType_Message: { - const upb_TaggedMessagePtr* start = _upb_array_constptr(arr); - const upb_TaggedMessagePtr* ptr = start + arr->UPB_PRIVATE(size); - const upb_MiniTable* subm = - upb_MiniTableSub_Message(subs[f->UPB_PRIVATE(submsg_index)]); - if (--e->depth == 0) encode_err(e, kUpb_EncodeStatus_MaxDepthExceeded); - do { - size_t size; - ptr--; - encode_TaggedMessagePtr(e, *ptr, subm, &size); - encode_varint(e, size); - encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_Delimited); - } while (ptr != start); - e->depth++; - return; - } - } -#undef VARINT_CASE - - if (packed) { - encode_varint(e, e->limit - e->ptr - pre_len); - encode_tag(e, f->UPB_PRIVATE(number), kUpb_WireType_Delimited); - } -} - -static void encode_mapentry(upb_encstate* e, uint32_t number, - const upb_MiniTable* layout, - const upb_MapEntry* ent) { - const upb_MiniTableField* key_field = &layout->UPB_PRIVATE(fields)[0]; - const upb_MiniTableField* val_field = &layout->UPB_PRIVATE(fields)[1]; - size_t pre_len = e->limit - e->ptr; - size_t size; - encode_scalar(e, &ent->data.v, layout->UPB_PRIVATE(subs), val_field); - encode_scalar(e, &ent->data.k, layout->UPB_PRIVATE(subs), key_field); - size = (e->limit - e->ptr) - pre_len; - encode_varint(e, size); - encode_tag(e, number, kUpb_WireType_Delimited); -} - -static void encode_map(upb_encstate* e, const upb_Message* msg, - const upb_MiniTableSub* subs, - const upb_MiniTableField* f) { - const upb_Map* map = *UPB_PTR_AT(msg, f->UPB_PRIVATE(offset), const upb_Map*); - const upb_MiniTable* layout = - upb_MiniTableSub_Message(subs[f->UPB_PRIVATE(submsg_index)]); - UPB_ASSERT(layout->UPB_PRIVATE(field_count) == 2); - - if (map == NULL) return; - - if (e->options & kUpb_EncodeOption_Deterministic) { - _upb_sortedmap sorted; - _upb_mapsorter_pushmap( - &e->sorter, layout->UPB_PRIVATE(fields)[0].UPB_PRIVATE(descriptortype), - map, &sorted); - upb_MapEntry ent; - while (_upb_sortedmap_next(&e->sorter, map, &sorted, &ent)) { - encode_mapentry(e, f->UPB_PRIVATE(number), layout, &ent); - } - _upb_mapsorter_popmap(&e->sorter, &sorted); - } else { - intptr_t iter = UPB_STRTABLE_BEGIN; - upb_StringView key; - upb_value val; - while (upb_strtable_next2(&map->table, &key, &val, &iter)) { - upb_MapEntry ent; - _upb_map_fromkey(key, &ent.data.k, map->key_size); - _upb_map_fromvalue(val, &ent.data.v, map->val_size); - encode_mapentry(e, f->UPB_PRIVATE(number), layout, &ent); - } - } -} - -static bool encode_shouldencode(upb_encstate* e, const upb_Message* msg, - const upb_MiniTableSub* subs, - const upb_MiniTableField* f) { - if (f->presence == 0) { - // Proto3 presence or map/array. - const void* mem = UPB_PTR_AT(msg, f->UPB_PRIVATE(offset), void); - switch (UPB_PRIVATE(_upb_MiniTableField_GetRep)(f)) { - case kUpb_FieldRep_1Byte: { - char ch; - memcpy(&ch, mem, 1); - return ch != 0; - } - case kUpb_FieldRep_4Byte: { - uint32_t u32; - memcpy(&u32, mem, 4); - return u32 != 0; - } - case kUpb_FieldRep_8Byte: { - uint64_t u64; - memcpy(&u64, mem, 8); - return u64 != 0; - } - case kUpb_FieldRep_StringView: { - const upb_StringView* str = (const upb_StringView*)mem; - return str->size != 0; - } - default: - UPB_UNREACHABLE(); - } - } else if (f->presence > 0) { - // Proto2 presence: hasbit. - return UPB_PRIVATE(_upb_Message_GetHasbit)(msg, f); - } else { - // Field is in a oneof. - return UPB_PRIVATE(_upb_Message_GetOneofCase)(msg, f) == - f->UPB_PRIVATE(number); - } -} - -static void encode_field(upb_encstate* e, const upb_Message* msg, - const upb_MiniTableSub* subs, - const upb_MiniTableField* field) { - switch (UPB_PRIVATE(_upb_MiniTableField_Mode)(field)) { - case kUpb_FieldMode_Array: - encode_array(e, msg, subs, field); - break; - case kUpb_FieldMode_Map: - encode_map(e, msg, subs, field); - break; - case kUpb_FieldMode_Scalar: - encode_scalar(e, UPB_PTR_AT(msg, field->UPB_PRIVATE(offset), void), subs, - field); - break; - default: - UPB_UNREACHABLE(); - } -} - -static void encode_msgset_item(upb_encstate* e, const upb_Extension* ext) { - size_t size; - encode_tag(e, kUpb_MsgSet_Item, kUpb_WireType_EndGroup); - encode_message(e, ext->data.ptr, - upb_MiniTableExtension_GetSubMessage(ext->ext), &size); - encode_varint(e, size); - encode_tag(e, kUpb_MsgSet_Message, kUpb_WireType_Delimited); - encode_varint(e, upb_MiniTableExtension_Number(ext->ext)); - encode_tag(e, kUpb_MsgSet_TypeId, kUpb_WireType_Varint); - encode_tag(e, kUpb_MsgSet_Item, kUpb_WireType_StartGroup); -} - -static void encode_ext(upb_encstate* e, const upb_Extension* ext, - bool is_message_set) { - if (UPB_UNLIKELY(is_message_set)) { - encode_msgset_item(e, ext); - } else { - encode_field(e, (upb_Message*)&ext->data, &ext->ext->UPB_PRIVATE(sub), - &ext->ext->UPB_PRIVATE(field)); - } -} - -static void encode_message(upb_encstate* e, const upb_Message* msg, - const upb_MiniTable* m, size_t* size) { - size_t pre_len = e->limit - e->ptr; - - if ((e->options & kUpb_EncodeOption_CheckRequired) && - m->UPB_PRIVATE(required_count)) { - uint64_t msg_head; - memcpy(&msg_head, msg, 8); - msg_head = UPB_PRIVATE(_upb_BigEndian64)(msg_head); - if (UPB_PRIVATE(_upb_MiniTable_RequiredMask)(m) & ~msg_head) { - encode_err(e, kUpb_EncodeStatus_MissingRequired); - } - } - - if ((e->options & kUpb_EncodeOption_SkipUnknown) == 0) { - size_t unknown_size; - const char* unknown = upb_Message_GetUnknown(msg, &unknown_size); - - if (unknown) { - encode_bytes(e, unknown, unknown_size); - } - } - - if (m->UPB_PRIVATE(ext) != kUpb_ExtMode_NonExtendable) { - /* Encode all extensions together. Unlike C++, we do not attempt to keep - * these in field number order relative to normal fields or even to each - * other. */ - size_t ext_count; - const upb_Extension* ext = - UPB_PRIVATE(_upb_Message_Getexts)(msg, &ext_count); - if (ext_count) { - if (e->options & kUpb_EncodeOption_Deterministic) { - _upb_sortedmap sorted; - _upb_mapsorter_pushexts(&e->sorter, ext, ext_count, &sorted); - while (_upb_sortedmap_nextext(&e->sorter, &sorted, &ext)) { - encode_ext(e, ext, m->UPB_PRIVATE(ext) == kUpb_ExtMode_IsMessageSet); - } - _upb_mapsorter_popmap(&e->sorter, &sorted); - } else { - const upb_Extension* end = ext + ext_count; - for (; ext != end; ext++) { - encode_ext(e, ext, m->UPB_PRIVATE(ext) == kUpb_ExtMode_IsMessageSet); - } - } - } - } - - if (m->UPB_PRIVATE(field_count)) { - const upb_MiniTableField* f = - &m->UPB_PRIVATE(fields)[m->UPB_PRIVATE(field_count)]; - const upb_MiniTableField* first = &m->UPB_PRIVATE(fields)[0]; - while (f != first) { - f--; - if (encode_shouldencode(e, msg, m->UPB_PRIVATE(subs), f)) { - encode_field(e, msg, m->UPB_PRIVATE(subs), f); - } - } - } - - *size = (e->limit - e->ptr) - pre_len; -} - -static upb_EncodeStatus upb_Encoder_Encode(upb_encstate* const encoder, - const upb_Message* const msg, - const upb_MiniTable* const l, - char** const buf, - size_t* const size) { - // Unfortunately we must continue to perform hackery here because there are - // code paths which blindly copy the returned pointer without bothering to - // check for errors until much later (b/235839510). So we still set *buf to - // NULL on error and we still set it to non-NULL on a successful empty result. - if (UPB_SETJMP(encoder->err) == 0) { - encode_message(encoder, msg, l, size); - *size = encoder->limit - encoder->ptr; - if (*size == 0) { - static char ch; - *buf = &ch; - } else { - UPB_ASSERT(encoder->ptr); - *buf = encoder->ptr; - } - } else { - UPB_ASSERT(encoder->status != kUpb_EncodeStatus_Ok); - *buf = NULL; - *size = 0; - } - - _upb_mapsorter_destroy(&encoder->sorter); - return encoder->status; -} - -upb_EncodeStatus upb_Encode(const upb_Message* msg, const upb_MiniTable* l, - int options, upb_Arena* arena, char** buf, - size_t* size) { - upb_encstate e; - unsigned depth = (unsigned)options >> 16; - - e.status = kUpb_EncodeStatus_Ok; - e.arena = arena; - e.buf = NULL; - e.limit = NULL; - e.ptr = NULL; - e.depth = depth ? depth : kUpb_WireFormat_DefaultDepthLimit; - e.options = options; - _upb_mapsorter_init(&e.sorter); - - return upb_Encoder_Encode(&e, msg, l, buf, size); -} - -// Fast decoder: ~3x the speed of decode.c, but requires x86-64/ARM64. -// Also the table size grows by 2x. -// -// Could potentially be ported to other 64-bit archs that pass at least six -// arguments in registers and have 8 unused high bits in pointers. -// -// The overall design is to create specialized functions for every possible -// field type (eg. oneof boolean field with a 1 byte tag) and then dispatch -// to the specialized function as quickly as possible. - - - -// Must be last. - -#if UPB_FASTTABLE - -// The standard set of arguments passed to each parsing function. -// Thanks to x86-64 calling conventions, these will stay in registers. -#define UPB_PARSE_PARAMS \ - upb_Decoder *d, const char *ptr, upb_Message *msg, intptr_t table, \ - uint64_t hasbits, uint64_t data - -#define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data - -#define RETURN_GENERIC(m) \ - /* Uncomment either of these for debugging purposes. */ \ - /* fprintf(stderr, m); */ \ - /*__builtin_trap(); */ \ - return _upb_FastDecoder_DecodeGeneric(d, ptr, msg, table, hasbits, 0); - -typedef enum { - CARD_s = 0, /* Singular (optional, non-repeated) */ - CARD_o = 1, /* Oneof */ - CARD_r = 2, /* Repeated */ - CARD_p = 3 /* Packed Repeated */ -} upb_card; - -UPB_NOINLINE -static const char* fastdecode_isdonefallback(UPB_PARSE_PARAMS) { - int overrun = data; - ptr = _upb_EpsCopyInputStream_IsDoneFallbackInline( - &d->input, ptr, overrun, _upb_Decoder_BufferFlipCallback); - data = _upb_FastDecoder_LoadTag(ptr); - UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); -} - -UPB_FORCEINLINE -static const char* fastdecode_dispatch(UPB_PARSE_PARAMS) { - int overrun; - switch (upb_EpsCopyInputStream_IsDoneStatus(&d->input, ptr, &overrun)) { - case kUpb_IsDoneStatus_Done: - *(uint32_t*)msg |= hasbits; // Sync hasbits. - const upb_MiniTable* m = decode_totablep(table); - return UPB_UNLIKELY(m->UPB_PRIVATE(required_count)) - ? _upb_Decoder_CheckRequired(d, ptr, msg, m) - : ptr; - case kUpb_IsDoneStatus_NotDone: - break; - case kUpb_IsDoneStatus_NeedFallback: - data = overrun; - UPB_MUSTTAIL return fastdecode_isdonefallback(UPB_PARSE_ARGS); - } - - // Read two bytes of tag data (for a one-byte tag, the high byte is junk). - data = _upb_FastDecoder_LoadTag(ptr); - UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); -} - -UPB_FORCEINLINE -static bool fastdecode_checktag(uint16_t data, int tagbytes) { - if (tagbytes == 1) { - return (data & 0xff) == 0; - } else { - return data == 0; - } -} - -UPB_FORCEINLINE -static const char* fastdecode_longsize(const char* ptr, int* size) { - int i; - UPB_ASSERT(*size & 0x80); - *size &= 0xff; - for (i = 0; i < 3; i++) { - ptr++; - size_t byte = (uint8_t)ptr[-1]; - *size += (byte - 1) << (7 + 7 * i); - if (UPB_LIKELY((byte & 0x80) == 0)) return ptr; - } - ptr++; - size_t byte = (uint8_t)ptr[-1]; - // len is limited by 2gb not 4gb, hence 8 and not 16 as normally expected - // for a 32 bit varint. - if (UPB_UNLIKELY(byte >= 8)) return NULL; - *size += (byte - 1) << 28; - return ptr; -} - -UPB_FORCEINLINE -static const char* fastdecode_delimited( - upb_Decoder* d, const char* ptr, - upb_EpsCopyInputStream_ParseDelimitedFunc* func, void* ctx) { - ptr++; - - // Sign-extend so varint greater than one byte becomes negative, causing - // fast delimited parse to fail. - int len = (int8_t)ptr[-1]; - - if (!upb_EpsCopyInputStream_TryParseDelimitedFast(&d->input, &ptr, len, func, - ctx)) { - // Slow case: Sub-message is >=128 bytes and/or exceeds the current buffer. - // If it exceeds the buffer limit, limit/limit_ptr will change during - // sub-message parsing, so we need to preserve delta, not limit. - if (UPB_UNLIKELY(len & 0x80)) { - // Size varint >1 byte (length >= 128). - ptr = fastdecode_longsize(ptr, &len); - if (!ptr) { - // Corrupt wire format: size exceeded INT_MAX. - return NULL; - } - } - if (!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, len)) { - // Corrupt wire format: invalid limit. - return NULL; - } - int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, len); - ptr = func(&d->input, ptr, ctx); - upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta); - } - return ptr; -} - -/* singular, oneof, repeated field handling ***********************************/ - -typedef struct { - upb_Array* arr; - void* end; -} fastdecode_arr; - -typedef enum { - FD_NEXT_ATLIMIT, - FD_NEXT_SAMEFIELD, - FD_NEXT_OTHERFIELD -} fastdecode_next; - -typedef struct { - void* dst; - fastdecode_next next; - uint32_t tag; -} fastdecode_nextret; - -UPB_FORCEINLINE -static void* fastdecode_resizearr(upb_Decoder* d, void* dst, - fastdecode_arr* farr, int valbytes) { - if (UPB_UNLIKELY(dst == farr->end)) { - size_t old_capacity = farr->arr->UPB_PRIVATE(capacity); - size_t old_bytes = old_capacity * valbytes; - size_t new_capacity = old_capacity * 2; - size_t new_bytes = new_capacity * valbytes; - char* old_ptr = _upb_array_ptr(farr->arr); - char* new_ptr = upb_Arena_Realloc(&d->arena, old_ptr, old_bytes, new_bytes); - uint8_t elem_size_lg2 = __builtin_ctz(valbytes); - UPB_PRIVATE(_upb_Array_SetTaggedPtr)(farr->arr, new_ptr, elem_size_lg2); - farr->arr->UPB_PRIVATE(capacity) = new_capacity; - dst = (void*)(new_ptr + (old_capacity * valbytes)); - farr->end = (void*)(new_ptr + (new_capacity * valbytes)); - } - return dst; -} - -UPB_FORCEINLINE -static bool fastdecode_tagmatch(uint32_t tag, uint64_t data, int tagbytes) { - if (tagbytes == 1) { - return (uint8_t)tag == (uint8_t)data; - } else { - return (uint16_t)tag == (uint16_t)data; - } -} - -UPB_FORCEINLINE -static void fastdecode_commitarr(void* dst, fastdecode_arr* farr, - int valbytes) { - farr->arr->UPB_PRIVATE(size) = - (size_t)((char*)dst - (char*)_upb_array_ptr(farr->arr)) / valbytes; -} - -UPB_FORCEINLINE -static fastdecode_nextret fastdecode_nextrepeated(upb_Decoder* d, void* dst, - const char** ptr, - fastdecode_arr* farr, - uint64_t data, int tagbytes, - int valbytes) { - fastdecode_nextret ret; - dst = (char*)dst + valbytes; - - if (UPB_LIKELY(!_upb_Decoder_IsDone(d, ptr))) { - ret.tag = _upb_FastDecoder_LoadTag(*ptr); - if (fastdecode_tagmatch(ret.tag, data, tagbytes)) { - ret.next = FD_NEXT_SAMEFIELD; - } else { - fastdecode_commitarr(dst, farr, valbytes); - ret.next = FD_NEXT_OTHERFIELD; - } - } else { - fastdecode_commitarr(dst, farr, valbytes); - ret.next = FD_NEXT_ATLIMIT; - } - - ret.dst = dst; - return ret; -} - -UPB_FORCEINLINE -static void* fastdecode_fieldmem(upb_Message* msg, uint64_t data) { - size_t ofs = data >> 48; - return (char*)msg + ofs; -} - -UPB_FORCEINLINE -static void* fastdecode_getfield(upb_Decoder* d, const char* ptr, - upb_Message* msg, uint64_t* data, - uint64_t* hasbits, fastdecode_arr* farr, - int valbytes, upb_card card) { - switch (card) { - case CARD_s: { - uint8_t hasbit_index = *data >> 24; - // Set hasbit and return pointer to scalar field. - *hasbits |= 1ull << hasbit_index; - return fastdecode_fieldmem(msg, *data); - } - case CARD_o: { - uint16_t case_ofs = *data >> 32; - uint32_t* oneof_case = UPB_PTR_AT(msg, case_ofs, uint32_t); - uint8_t field_number = *data >> 24; - *oneof_case = field_number; - return fastdecode_fieldmem(msg, *data); - } - case CARD_r: { - // Get pointer to upb_Array and allocate/expand if necessary. - uint8_t elem_size_lg2 = __builtin_ctz(valbytes); - upb_Array** arr_p = fastdecode_fieldmem(msg, *data); - char* begin; - *(uint32_t*)msg |= *hasbits; - *hasbits = 0; - if (UPB_LIKELY(!*arr_p)) { - farr->arr = UPB_PRIVATE(_upb_Array_New)(&d->arena, 8, elem_size_lg2); - *arr_p = farr->arr; - } else { - farr->arr = *arr_p; - } - begin = _upb_array_ptr(farr->arr); - farr->end = begin + (farr->arr->UPB_PRIVATE(capacity) * valbytes); - *data = _upb_FastDecoder_LoadTag(ptr); - return begin + (farr->arr->UPB_PRIVATE(size) * valbytes); - } - default: - UPB_UNREACHABLE(); - } -} - -UPB_FORCEINLINE -static bool fastdecode_flippacked(uint64_t* data, int tagbytes) { - *data ^= (0x2 ^ 0x0); // Patch data to match packed wiretype. - return fastdecode_checktag(*data, tagbytes); -} - -#define FASTDECODE_CHECKPACKED(tagbytes, card, func) \ - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ - if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \ - UPB_MUSTTAIL return func(UPB_PARSE_ARGS); \ - } \ - RETURN_GENERIC("packed check tag mismatch\n"); \ - } - -/* varint fields **************************************************************/ - -UPB_FORCEINLINE -static uint64_t fastdecode_munge(uint64_t val, int valbytes, bool zigzag) { - if (valbytes == 1) { - return val != 0; - } else if (zigzag) { - if (valbytes == 4) { - uint32_t n = val; - return (n >> 1) ^ -(int32_t)(n & 1); - } else if (valbytes == 8) { - return (val >> 1) ^ -(int64_t)(val & 1); - } - UPB_UNREACHABLE(); - } - return val; -} - -UPB_FORCEINLINE -static const char* fastdecode_varint64(const char* ptr, uint64_t* val) { - ptr++; - *val = (uint8_t)ptr[-1]; - if (UPB_UNLIKELY(*val & 0x80)) { - int i; - for (i = 0; i < 8; i++) { - ptr++; - uint64_t byte = (uint8_t)ptr[-1]; - *val += (byte - 1) << (7 + 7 * i); - if (UPB_LIKELY((byte & 0x80) == 0)) goto done; - } - ptr++; - uint64_t byte = (uint8_t)ptr[-1]; - if (byte > 1) { - return NULL; - } - *val += (byte - 1) << 63; - } -done: - UPB_ASSUME(ptr != NULL); - return ptr; -} - -#define FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, card, zigzag, packed) \ - uint64_t val; \ - void* dst; \ - fastdecode_arr farr; \ - \ - FASTDECODE_CHECKPACKED(tagbytes, card, packed); \ - \ - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ - card); \ - if (card == CARD_r) { \ - if (UPB_UNLIKELY(!dst)) { \ - RETURN_GENERIC("need array resize\n"); \ - } \ - } \ - \ - again: \ - if (card == CARD_r) { \ - dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ - } \ - \ - ptr += tagbytes; \ - ptr = fastdecode_varint64(ptr, &val); \ - if (ptr == NULL) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ - val = fastdecode_munge(val, valbytes, zigzag); \ - memcpy(dst, &val, valbytes); \ - \ - if (card == CARD_r) { \ - fastdecode_nextret ret = fastdecode_nextrepeated( \ - d, dst, &ptr, &farr, data, tagbytes, valbytes); \ - switch (ret.next) { \ - case FD_NEXT_SAMEFIELD: \ - dst = ret.dst; \ - goto again; \ - case FD_NEXT_OTHERFIELD: \ - data = ret.tag; \ - UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ - case FD_NEXT_ATLIMIT: \ - return ptr; \ - } \ - } \ - \ - UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); - -typedef struct { - uint8_t valbytes; - bool zigzag; - void* dst; - fastdecode_arr farr; -} fastdecode_varintdata; - -UPB_FORCEINLINE -static const char* fastdecode_topackedvarint(upb_EpsCopyInputStream* e, - const char* ptr, void* ctx) { - upb_Decoder* d = (upb_Decoder*)e; - fastdecode_varintdata* data = ctx; - void* dst = data->dst; - uint64_t val; - - while (!_upb_Decoder_IsDone(d, &ptr)) { - dst = fastdecode_resizearr(d, dst, &data->farr, data->valbytes); - ptr = fastdecode_varint64(ptr, &val); - if (ptr == NULL) return NULL; - val = fastdecode_munge(val, data->valbytes, data->zigzag); - memcpy(dst, &val, data->valbytes); - dst = (char*)dst + data->valbytes; - } - - fastdecode_commitarr(dst, &data->farr, data->valbytes); - return ptr; -} - -#define FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, zigzag, unpacked) \ - fastdecode_varintdata ctx = {valbytes, zigzag}; \ - \ - FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked); \ - \ - ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \ - valbytes, CARD_r); \ - if (UPB_UNLIKELY(!ctx.dst)) { \ - RETURN_GENERIC("need array resize\n"); \ - } \ - \ - ptr += tagbytes; \ - ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \ - \ - if (UPB_UNLIKELY(ptr == NULL)) { \ - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ - } \ - \ - UPB_MUSTTAIL return fastdecode_dispatch(d, ptr, msg, table, hasbits, 0); - -#define FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, card, zigzag, unpacked, packed) \ - if (card == CARD_p) { \ - FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, zigzag, unpacked); \ - } else { \ - FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, card, zigzag, packed); \ - } - -#define z_ZZ true -#define b_ZZ false -#define v_ZZ false - -/* Generate all combinations: - * {s,o,r,p} x {b1,v4,z4,v8,z8} x {1bt,2bt} */ - -#define F(card, type, valbytes, tagbytes) \ - UPB_NOINLINE \ - const char* upb_p##card##type##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ - FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \ - CARD_##card, type##_ZZ, \ - upb_pr##type##valbytes##_##tagbytes##bt, \ - upb_pp##type##valbytes##_##tagbytes##bt); \ - } - -#define TYPES(card, tagbytes) \ - F(card, b, 1, tagbytes) \ - F(card, v, 4, tagbytes) \ - F(card, v, 8, tagbytes) \ - F(card, z, 4, tagbytes) \ - F(card, z, 8, tagbytes) - -#define TAGBYTES(card) \ - TYPES(card, 1) \ - TYPES(card, 2) - -TAGBYTES(s) -TAGBYTES(o) -TAGBYTES(r) -TAGBYTES(p) - -#undef z_ZZ -#undef b_ZZ -#undef v_ZZ -#undef o_ONEOF -#undef s_ONEOF -#undef r_ONEOF -#undef F -#undef TYPES -#undef TAGBYTES -#undef FASTDECODE_UNPACKEDVARINT -#undef FASTDECODE_PACKEDVARINT -#undef FASTDECODE_VARINT - -/* fixed fields ***************************************************************/ - -#define FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, card, packed) \ - void* dst; \ - fastdecode_arr farr; \ - \ - FASTDECODE_CHECKPACKED(tagbytes, card, packed) \ - \ - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ - card); \ - if (card == CARD_r) { \ - if (UPB_UNLIKELY(!dst)) { \ - RETURN_GENERIC("couldn't allocate array in arena\n"); \ - } \ - } \ - \ - again: \ - if (card == CARD_r) { \ - dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ - } \ - \ - ptr += tagbytes; \ - memcpy(dst, ptr, valbytes); \ - ptr += valbytes; \ - \ - if (card == CARD_r) { \ - fastdecode_nextret ret = fastdecode_nextrepeated( \ - d, dst, &ptr, &farr, data, tagbytes, valbytes); \ - switch (ret.next) { \ - case FD_NEXT_SAMEFIELD: \ - dst = ret.dst; \ - goto again; \ - case FD_NEXT_OTHERFIELD: \ - data = ret.tag; \ - UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ - case FD_NEXT_ATLIMIT: \ - return ptr; \ - } \ - } \ - \ - UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); - -#define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, unpacked) \ - FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked) \ - \ - ptr += tagbytes; \ - int size = (uint8_t)ptr[0]; \ - ptr++; \ - if (size & 0x80) { \ - ptr = fastdecode_longsize(ptr, &size); \ - } \ - \ - if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckDataSizeAvailable( \ - &d->input, ptr, size) || \ - (size % valbytes) != 0)) { \ - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ - } \ - \ - upb_Array** arr_p = fastdecode_fieldmem(msg, data); \ - upb_Array* arr = *arr_p; \ - uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \ - int elems = size / valbytes; \ - \ - if (UPB_LIKELY(!arr)) { \ - *arr_p = arr = \ - UPB_PRIVATE(_upb_Array_New)(&d->arena, elems, elem_size_lg2); \ - if (!arr) { \ - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ - } \ - } else { \ - _upb_Array_ResizeUninitialized(arr, elems, &d->arena); \ - } \ - \ - char* dst = _upb_array_ptr(arr); \ - memcpy(dst, ptr, size); \ - arr->UPB_PRIVATE(size) = elems; \ - \ - ptr += size; \ - UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); - -#define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, card, unpacked, packed) \ - if (card == CARD_p) { \ - FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, unpacked); \ - } else { \ - FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, card, packed); \ - } - -/* Generate all combinations: - * {s,o,r,p} x {f4,f8} x {1bt,2bt} */ - -#define F(card, valbytes, tagbytes) \ - UPB_NOINLINE \ - const char* upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ - FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \ - CARD_##card, upb_ppf##valbytes##_##tagbytes##bt, \ - upb_prf##valbytes##_##tagbytes##bt); \ - } - -#define TYPES(card, tagbytes) \ - F(card, 4, tagbytes) \ - F(card, 8, tagbytes) - -#define TAGBYTES(card) \ - TYPES(card, 1) \ - TYPES(card, 2) - -TAGBYTES(s) -TAGBYTES(o) -TAGBYTES(r) -TAGBYTES(p) - -#undef F -#undef TYPES -#undef TAGBYTES -#undef FASTDECODE_UNPACKEDFIXED -#undef FASTDECODE_PACKEDFIXED - -/* string fields **************************************************************/ - -typedef const char* fastdecode_copystr_func(struct upb_Decoder* d, - const char* ptr, upb_Message* msg, - const upb_MiniTable* table, - uint64_t hasbits, - upb_StringView* dst); - -UPB_NOINLINE -static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr, - upb_Message* msg, intptr_t table, - uint64_t hasbits, uint64_t data) { - upb_StringView* dst = (upb_StringView*)data; - if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); - } - UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); -} - -#define FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, validate_utf8) \ - int size = (uint8_t)ptr[0]; /* Could plumb through hasbits. */ \ - ptr++; \ - if (size & 0x80) { \ - ptr = fastdecode_longsize(ptr, &size); \ - } \ - \ - if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, size))) { \ - dst->size = 0; \ - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ - } \ - \ - const char* s_ptr = ptr; \ - ptr = upb_EpsCopyInputStream_ReadString(&d->input, &s_ptr, size, &d->arena); \ - if (!ptr) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); \ - dst->data = s_ptr; \ - dst->size = size; \ - \ - if (validate_utf8) { \ - data = (uint64_t)dst; \ - UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ - } else { \ - UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ - } - -UPB_NOINLINE -static const char* fastdecode_longstring_utf8(struct upb_Decoder* d, - const char* ptr, upb_Message* msg, - intptr_t table, uint64_t hasbits, - uint64_t data) { - upb_StringView* dst = (upb_StringView*)data; - FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, true); -} - -UPB_NOINLINE -static const char* fastdecode_longstring_noutf8( - struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, - uint64_t hasbits, uint64_t data) { - upb_StringView* dst = (upb_StringView*)data; - FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false); -} - -UPB_FORCEINLINE -static void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size, - int copy, char* data, size_t data_offset, - upb_StringView* dst) { - d->arena.UPB_PRIVATE(ptr) += copy; - dst->data = data + data_offset; - UPB_UNPOISON_MEMORY_REGION(data, copy); - memcpy(data, ptr, copy); - UPB_POISON_MEMORY_REGION(data + data_offset + size, - copy - data_offset - size); -} - -#define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ - card, validate_utf8) \ - upb_StringView* dst; \ - fastdecode_arr farr; \ - int64_t size; \ - size_t arena_has; \ - size_t common_has; \ - char* buf; \ - \ - UPB_ASSERT(!upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0)); \ - UPB_ASSERT(fastdecode_checktag(data, tagbytes)); \ - \ - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ - sizeof(upb_StringView), card); \ - \ - again: \ - if (card == CARD_r) { \ - dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \ - } \ - \ - size = (uint8_t)ptr[tagbytes]; \ - ptr += tagbytes + 1; \ - dst->size = size; \ - \ - buf = d->arena.UPB_PRIVATE(ptr); \ - arena_has = UPB_PRIVATE(_upb_ArenaHas)(&d->arena); \ - common_has = UPB_MIN(arena_has, \ - upb_EpsCopyInputStream_BytesAvailable(&d->input, ptr)); \ - \ - if (UPB_LIKELY(size <= 15 - tagbytes)) { \ - if (arena_has < 16) goto longstr; \ - fastdecode_docopy(d, ptr - tagbytes - 1, size, 16, buf, tagbytes + 1, \ - dst); \ - } else if (UPB_LIKELY(size <= 32)) { \ - if (UPB_UNLIKELY(common_has < 32)) goto longstr; \ - fastdecode_docopy(d, ptr, size, 32, buf, 0, dst); \ - } else if (UPB_LIKELY(size <= 64)) { \ - if (UPB_UNLIKELY(common_has < 64)) goto longstr; \ - fastdecode_docopy(d, ptr, size, 64, buf, 0, dst); \ - } else if (UPB_LIKELY(size < 128)) { \ - if (UPB_UNLIKELY(common_has < 128)) goto longstr; \ - fastdecode_docopy(d, ptr, size, 128, buf, 0, dst); \ - } else { \ - goto longstr; \ - } \ - \ - ptr += size; \ - \ - if (card == CARD_r) { \ - if (validate_utf8 && \ - !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \ - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \ - } \ - fastdecode_nextret ret = fastdecode_nextrepeated( \ - d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \ - switch (ret.next) { \ - case FD_NEXT_SAMEFIELD: \ - dst = ret.dst; \ - goto again; \ - case FD_NEXT_OTHERFIELD: \ - data = ret.tag; \ - UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ - case FD_NEXT_ATLIMIT: \ - return ptr; \ - } \ - } \ - \ - if (card != CARD_r && validate_utf8) { \ - data = (uint64_t)dst; \ - UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ - } \ - \ - UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ - \ - longstr: \ - if (card == CARD_r) { \ - fastdecode_commitarr(dst + 1, &farr, sizeof(upb_StringView)); \ - } \ - ptr--; \ - if (validate_utf8) { \ - UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table, \ - hasbits, (uint64_t)dst); \ - } else { \ - UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table, \ - hasbits, (uint64_t)dst); \ - } - -#define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card, \ - copyfunc, validate_utf8) \ - upb_StringView* dst; \ - fastdecode_arr farr; \ - int64_t size; \ - \ - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ - RETURN_GENERIC("string field tag mismatch\n"); \ - } \ - \ - if (UPB_UNLIKELY( \ - !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0))) { \ - UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS); \ - } \ - \ - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ - sizeof(upb_StringView), card); \ - \ - again: \ - if (card == CARD_r) { \ - dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \ - } \ - \ - size = (int8_t)ptr[tagbytes]; \ - ptr += tagbytes + 1; \ - \ - if (UPB_UNLIKELY( \ - !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, size))) { \ - ptr--; \ - if (validate_utf8) { \ - return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, \ - (uint64_t)dst); \ - } else { \ - return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, \ - (uint64_t)dst); \ - } \ - } \ - \ - dst->data = ptr; \ - dst->size = size; \ - ptr = upb_EpsCopyInputStream_ReadStringAliased(&d->input, &dst->data, \ - dst->size); \ - \ - if (card == CARD_r) { \ - if (validate_utf8 && \ - !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \ - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \ - } \ - fastdecode_nextret ret = fastdecode_nextrepeated( \ - d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \ - switch (ret.next) { \ - case FD_NEXT_SAMEFIELD: \ - dst = ret.dst; \ - goto again; \ - case FD_NEXT_OTHERFIELD: \ - data = ret.tag; \ - UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ - case FD_NEXT_ATLIMIT: \ - return ptr; \ - } \ - } \ - \ - if (card != CARD_r && validate_utf8) { \ - data = (uint64_t)dst; \ - UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ - } \ - \ - UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); - -/* Generate all combinations: - * {p,c} x {s,o,r} x {s, b} x {1bt,2bt} */ - -#define s_VALIDATE true -#define b_VALIDATE false - -#define F(card, tagbytes, type) \ - UPB_NOINLINE \ - const char* upb_c##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ - FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ - CARD_##card, type##_VALIDATE); \ - } \ - const char* upb_p##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ - FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, \ - CARD_##card, upb_c##card##type##_##tagbytes##bt, \ - type##_VALIDATE); \ - } - -#define UTF8(card, tagbytes) \ - F(card, tagbytes, s) \ - F(card, tagbytes, b) - -#define TAGBYTES(card) \ - UTF8(card, 1) \ - UTF8(card, 2) - -TAGBYTES(s) -TAGBYTES(o) -TAGBYTES(r) - -#undef s_VALIDATE -#undef b_VALIDATE -#undef F -#undef TAGBYTES -#undef FASTDECODE_LONGSTRING -#undef FASTDECODE_COPYSTRING -#undef FASTDECODE_STRING - -/* message fields *************************************************************/ - -UPB_INLINE -upb_Message* decode_newmsg_ceil(upb_Decoder* d, const upb_MiniTable* m, - int msg_ceil_bytes) { - size_t size = m->UPB_PRIVATE(size) + sizeof(upb_Message_Internal); - char* msg_data; - if (UPB_LIKELY(msg_ceil_bytes > 0 && - UPB_PRIVATE(_upb_ArenaHas)(&d->arena) >= msg_ceil_bytes)) { - UPB_ASSERT(size <= (size_t)msg_ceil_bytes); - msg_data = d->arena.UPB_PRIVATE(ptr); - d->arena.UPB_PRIVATE(ptr) += size; - UPB_UNPOISON_MEMORY_REGION(msg_data, msg_ceil_bytes); - memset(msg_data, 0, msg_ceil_bytes); - UPB_POISON_MEMORY_REGION(msg_data + size, msg_ceil_bytes - size); - } else { - msg_data = (char*)upb_Arena_Malloc(&d->arena, size); - memset(msg_data, 0, size); - } - return msg_data + sizeof(upb_Message_Internal); -} - -typedef struct { - intptr_t table; - upb_Message* msg; -} fastdecode_submsgdata; - -UPB_FORCEINLINE -static const char* fastdecode_tosubmsg(upb_EpsCopyInputStream* e, - const char* ptr, void* ctx) { - upb_Decoder* d = (upb_Decoder*)e; - fastdecode_submsgdata* submsg = ctx; - ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0, 0); - UPB_ASSUME(ptr != NULL); - return ptr; -} - -#define FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, \ - msg_ceil_bytes, card) \ - \ - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ - RETURN_GENERIC("submessage field tag mismatch\n"); \ - } \ - \ - if (--d->depth == 0) { \ - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_MaxDepthExceeded); \ - } \ - \ - upb_Message** dst; \ - uint32_t submsg_idx = (data >> 16) & 0xff; \ - const upb_MiniTable* tablep = decode_totablep(table); \ - const upb_MiniTable* subtablep = upb_MiniTableSub_Message( \ - *UPB_PRIVATE(_upb_MiniTable_GetSubByIndex)(tablep, submsg_idx)); \ - fastdecode_submsgdata submsg = {decode_totable(subtablep)}; \ - fastdecode_arr farr; \ - \ - if (subtablep->UPB_PRIVATE(table_mask) == (uint8_t)-1) { \ - d->depth++; \ - RETURN_GENERIC("submessage doesn't have fast tables."); \ - } \ - \ - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ - sizeof(upb_Message*), card); \ - \ - if (card == CARD_s) { \ - *(uint32_t*)msg |= hasbits; \ - hasbits = 0; \ - } \ - \ - again: \ - if (card == CARD_r) { \ - dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_Message*)); \ - } \ - \ - submsg.msg = *dst; \ - \ - if (card == CARD_r || UPB_LIKELY(!submsg.msg)) { \ - *dst = submsg.msg = decode_newmsg_ceil(d, subtablep, msg_ceil_bytes); \ - } \ - \ - ptr += tagbytes; \ - ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg); \ - \ - if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) { \ - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ - } \ - \ - if (card == CARD_r) { \ - fastdecode_nextret ret = fastdecode_nextrepeated( \ - d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_Message*)); \ - switch (ret.next) { \ - case FD_NEXT_SAMEFIELD: \ - dst = ret.dst; \ - goto again; \ - case FD_NEXT_OTHERFIELD: \ - d->depth++; \ - data = ret.tag; \ - UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ - case FD_NEXT_ATLIMIT: \ - d->depth++; \ - return ptr; \ - } \ - } \ - \ - d->depth++; \ - UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); - -#define F(card, tagbytes, size_ceil, ceil_arg) \ - const char* upb_p##card##m_##tagbytes##bt_max##size_ceil##b( \ - UPB_PARSE_PARAMS) { \ - FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, ceil_arg, \ - CARD_##card); \ - } - -#define SIZES(card, tagbytes) \ - F(card, tagbytes, 64, 64) \ - F(card, tagbytes, 128, 128) \ - F(card, tagbytes, 192, 192) \ - F(card, tagbytes, 256, 256) \ - F(card, tagbytes, max, -1) - -#define TAGBYTES(card) \ - SIZES(card, 1) \ - SIZES(card, 2) - -TAGBYTES(s) -TAGBYTES(o) -TAGBYTES(r) - -#undef TAGBYTES -#undef SIZES -#undef F -#undef FASTDECODE_SUBMSG - -#endif /* UPB_FASTTABLE */ - - -#include <stddef.h> -#include <stdint.h> - - -// Must be last. - -UPB_NOINLINE UPB_PRIVATE(_upb_WireReader_LongVarint) - UPB_PRIVATE(_upb_WireReader_ReadLongVarint)(const char* ptr, uint64_t val) { - UPB_PRIVATE(_upb_WireReader_LongVarint) ret = {NULL, 0}; - uint64_t byte; - int i; - for (i = 1; i < 10; i++) { - byte = (uint8_t)ptr[i]; - val += (byte - 1) << (i * 7); - if (!(byte & 0x80)) { - ret.ptr = ptr + i + 1; - ret.val = val; - return ret; - } - } - return ret; -} - -const char* UPB_PRIVATE(_upb_WireReader_SkipGroup)( - const char* ptr, uint32_t tag, int depth_limit, - upb_EpsCopyInputStream* stream) { - if (--depth_limit == 0) return NULL; - uint32_t end_group_tag = (tag & ~7ULL) | kUpb_WireType_EndGroup; - while (!upb_EpsCopyInputStream_IsDone(stream, &ptr)) { - uint32_t tag; - ptr = upb_WireReader_ReadTag(ptr, &tag); - if (!ptr) return NULL; - if (tag == end_group_tag) return ptr; - ptr = _upb_WireReader_SkipValue(ptr, tag, depth_limit, stream); - if (!ptr) return NULL; - } - return ptr; -} - - -#include <string.h> - - -// Must be last. - -const struct upb_Extension* _upb_Message_Getext( - const struct upb_Message* msg, const upb_MiniTableExtension* e) { - size_t n; - const struct upb_Extension* ext = UPB_PRIVATE(_upb_Message_Getexts)(msg, &n); - - // For now we use linear search exclusively to find extensions. - // If this becomes an issue due to messages with lots of extensions, - // we can introduce a table of some sort. - for (size_t i = 0; i < n; i++) { - if (ext[i].ext == e) { - return &ext[i]; - } - } - - return NULL; -} - -const struct upb_Extension* UPB_PRIVATE(_upb_Message_Getexts)( - const struct upb_Message* msg, size_t* count) { - upb_Message_InternalData* in = upb_Message_GetInternalData(msg); - if (in) { - *count = (in->size - in->ext_begin) / sizeof(struct upb_Extension); - return UPB_PTR_AT(in, in->ext_begin, void); - } else { - *count = 0; - return NULL; - } -} - -struct upb_Extension* _upb_Message_GetOrCreateExtension( - struct upb_Message* msg, const upb_MiniTableExtension* e, upb_Arena* a) { - struct upb_Extension* ext = - (struct upb_Extension*)_upb_Message_Getext(msg, e); - if (ext) return ext; - if (!UPB_PRIVATE(_upb_Message_Realloc)(msg, sizeof(struct upb_Extension), a)) - return NULL; - upb_Message_InternalData* in = upb_Message_GetInternalData(msg); - in->ext_begin -= sizeof(struct upb_Extension); - ext = UPB_PTR_AT(in, in->ext_begin, void); - memset(ext, 0, sizeof(struct upb_Extension)); - ext->ext = e; - return ext; -} - - -#include <math.h> -#include <string.h> - - -// Must be last. - -const float kUpb_FltInfinity = INFINITY; -const double kUpb_Infinity = INFINITY; -const double kUpb_NaN = NAN; - -bool UPB_PRIVATE(_upb_Message_Realloc)(struct upb_Message* msg, size_t need, - upb_Arena* a) { - const size_t overhead = sizeof(upb_Message_InternalData); - - upb_Message_Internal* owner = upb_Message_Getinternal(msg); - upb_Message_InternalData* in = owner->internal; - if (!in) { - // No internal data, allocate from scratch. - size_t size = UPB_MAX(128, upb_Log2CeilingSize(need + overhead)); - in = upb_Arena_Malloc(a, size); - if (!in) return false; - - in->size = size; - in->unknown_end = overhead; - in->ext_begin = size; - owner->internal = in; - } else if (in->ext_begin - in->unknown_end < need) { - // Internal data is too small, reallocate. - size_t new_size = upb_Log2CeilingSize(in->size + need); - size_t ext_bytes = in->size - in->ext_begin; - size_t new_ext_begin = new_size - ext_bytes; - in = upb_Arena_Realloc(a, in, in->size, new_size); - if (!in) return false; - - if (ext_bytes) { - // Need to move extension data to the end. - char* ptr = (char*)in; - memmove(ptr + new_ext_begin, ptr + in->ext_begin, ext_bytes); - } - in->ext_begin = new_ext_begin; - in->size = new_size; - owner->internal = in; - } - - UPB_ASSERT(in->ext_begin - in->unknown_end >= need); - return true; -} - - -const char _kUpb_ToBase92[] = { - ' ', '!', '#', '$', '%', '&', '(', ')', '*', '+', ',', '-', '.', '/', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', - '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', - 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', - 'Z', '[', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', - 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '{', '|', '}', '~', -}; - -const int8_t _kUpb_FromBase92[] = { - 0, 1, -1, 2, 3, 4, 5, -1, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, -1, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, -}; - - -#include <assert.h> -#include <stddef.h> -#include <stdint.h> - - -// Must be last. - -typedef struct { - uint64_t present_values_mask; - uint32_t last_written_value; -} upb_MtDataEncoderInternal_EnumState; - -typedef struct { - uint64_t msg_modifiers; - uint32_t last_field_num; - enum { - kUpb_OneofState_NotStarted, - kUpb_OneofState_StartedOneof, - kUpb_OneofState_EmittedOneofField, - } oneof_state; -} upb_MtDataEncoderInternal_MsgState; - -typedef struct { - char* buf_start; // Only for checking kUpb_MtDataEncoder_MinSize. - union { - upb_MtDataEncoderInternal_EnumState enum_state; - upb_MtDataEncoderInternal_MsgState msg_state; - } state; -} upb_MtDataEncoderInternal; - -static upb_MtDataEncoderInternal* upb_MtDataEncoder_GetInternal( - upb_MtDataEncoder* e, char* buf_start) { - UPB_ASSERT(sizeof(upb_MtDataEncoderInternal) <= sizeof(e->internal)); - upb_MtDataEncoderInternal* ret = (upb_MtDataEncoderInternal*)e->internal; - ret->buf_start = buf_start; - return ret; -} - -static char* upb_MtDataEncoder_PutRaw(upb_MtDataEncoder* e, char* ptr, - char ch) { - upb_MtDataEncoderInternal* in = (upb_MtDataEncoderInternal*)e->internal; - UPB_ASSERT(ptr - in->buf_start < kUpb_MtDataEncoder_MinSize); - if (ptr == e->end) return NULL; - *ptr++ = ch; - return ptr; -} - -static char* upb_MtDataEncoder_Put(upb_MtDataEncoder* e, char* ptr, char ch) { - return upb_MtDataEncoder_PutRaw(e, ptr, _upb_ToBase92(ch)); -} - -static char* upb_MtDataEncoder_PutBase92Varint(upb_MtDataEncoder* e, char* ptr, - uint32_t val, int min, int max) { - int shift = upb_Log2Ceiling(_upb_FromBase92(max) - _upb_FromBase92(min) + 1); - UPB_ASSERT(shift <= 6); - uint32_t mask = (1 << shift) - 1; - do { - uint32_t bits = val & mask; - ptr = upb_MtDataEncoder_Put(e, ptr, bits + _upb_FromBase92(min)); - if (!ptr) return NULL; - val >>= shift; - } while (val); - return ptr; -} - -char* upb_MtDataEncoder_PutModifier(upb_MtDataEncoder* e, char* ptr, - uint64_t mod) { - if (mod) { - ptr = upb_MtDataEncoder_PutBase92Varint(e, ptr, mod, - kUpb_EncodedValue_MinModifier, - kUpb_EncodedValue_MaxModifier); - } - return ptr; -} - -char* upb_MtDataEncoder_EncodeExtension(upb_MtDataEncoder* e, char* ptr, - upb_FieldType type, uint32_t field_num, - uint64_t field_mod) { - upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); - in->state.msg_state.msg_modifiers = 0; - in->state.msg_state.last_field_num = 0; - in->state.msg_state.oneof_state = kUpb_OneofState_NotStarted; - - ptr = upb_MtDataEncoder_PutRaw(e, ptr, kUpb_EncodedVersion_ExtensionV1); - if (!ptr) return NULL; - - return upb_MtDataEncoder_PutField(e, ptr, type, field_num, field_mod); -} - -char* upb_MtDataEncoder_EncodeMap(upb_MtDataEncoder* e, char* ptr, - upb_FieldType key_type, - upb_FieldType value_type, uint64_t key_mod, - uint64_t value_mod) { - upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); - in->state.msg_state.msg_modifiers = 0; - in->state.msg_state.last_field_num = 0; - in->state.msg_state.oneof_state = kUpb_OneofState_NotStarted; - - ptr = upb_MtDataEncoder_PutRaw(e, ptr, kUpb_EncodedVersion_MapV1); - if (!ptr) return NULL; - - ptr = upb_MtDataEncoder_PutField(e, ptr, key_type, 1, key_mod); - if (!ptr) return NULL; - - return upb_MtDataEncoder_PutField(e, ptr, value_type, 2, value_mod); -} - -char* upb_MtDataEncoder_EncodeMessageSet(upb_MtDataEncoder* e, char* ptr) { - (void)upb_MtDataEncoder_GetInternal(e, ptr); - return upb_MtDataEncoder_PutRaw(e, ptr, kUpb_EncodedVersion_MessageSetV1); -} - -char* upb_MtDataEncoder_StartMessage(upb_MtDataEncoder* e, char* ptr, - uint64_t msg_mod) { - upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); - in->state.msg_state.msg_modifiers = msg_mod; - in->state.msg_state.last_field_num = 0; - in->state.msg_state.oneof_state = kUpb_OneofState_NotStarted; - - ptr = upb_MtDataEncoder_PutRaw(e, ptr, kUpb_EncodedVersion_MessageV1); - if (!ptr) return NULL; - - return upb_MtDataEncoder_PutModifier(e, ptr, msg_mod); -} - -static char* _upb_MtDataEncoder_MaybePutFieldSkip(upb_MtDataEncoder* e, - char* ptr, - uint32_t field_num) { - upb_MtDataEncoderInternal* in = (upb_MtDataEncoderInternal*)e->internal; - if (field_num <= in->state.msg_state.last_field_num) return NULL; - if (in->state.msg_state.last_field_num + 1 != field_num) { - // Put skip. - UPB_ASSERT(field_num > in->state.msg_state.last_field_num); - uint32_t skip = field_num - in->state.msg_state.last_field_num; - ptr = upb_MtDataEncoder_PutBase92Varint( - e, ptr, skip, kUpb_EncodedValue_MinSkip, kUpb_EncodedValue_MaxSkip); - if (!ptr) return NULL; - } - in->state.msg_state.last_field_num = field_num; - return ptr; -} - -static char* _upb_MtDataEncoder_PutFieldType(upb_MtDataEncoder* e, char* ptr, - upb_FieldType type, - uint64_t field_mod) { - static const char kUpb_TypeToEncoded[] = { - [kUpb_FieldType_Double] = kUpb_EncodedType_Double, - [kUpb_FieldType_Float] = kUpb_EncodedType_Float, - [kUpb_FieldType_Int64] = kUpb_EncodedType_Int64, - [kUpb_FieldType_UInt64] = kUpb_EncodedType_UInt64, - [kUpb_FieldType_Int32] = kUpb_EncodedType_Int32, - [kUpb_FieldType_Fixed64] = kUpb_EncodedType_Fixed64, - [kUpb_FieldType_Fixed32] = kUpb_EncodedType_Fixed32, - [kUpb_FieldType_Bool] = kUpb_EncodedType_Bool, - [kUpb_FieldType_String] = kUpb_EncodedType_String, - [kUpb_FieldType_Group] = kUpb_EncodedType_Group, - [kUpb_FieldType_Message] = kUpb_EncodedType_Message, - [kUpb_FieldType_Bytes] = kUpb_EncodedType_Bytes, - [kUpb_FieldType_UInt32] = kUpb_EncodedType_UInt32, - [kUpb_FieldType_Enum] = kUpb_EncodedType_OpenEnum, - [kUpb_FieldType_SFixed32] = kUpb_EncodedType_SFixed32, - [kUpb_FieldType_SFixed64] = kUpb_EncodedType_SFixed64, - [kUpb_FieldType_SInt32] = kUpb_EncodedType_SInt32, - [kUpb_FieldType_SInt64] = kUpb_EncodedType_SInt64, - }; - - int encoded_type = kUpb_TypeToEncoded[type]; - - if (field_mod & kUpb_FieldModifier_IsClosedEnum) { - UPB_ASSERT(type == kUpb_FieldType_Enum); - encoded_type = kUpb_EncodedType_ClosedEnum; - } - - if (field_mod & kUpb_FieldModifier_IsRepeated) { - // Repeated fields shift the type number up (unlike other modifiers which - // are bit flags). - encoded_type += kUpb_EncodedType_RepeatedBase; - } - - return upb_MtDataEncoder_Put(e, ptr, encoded_type); -} - -static char* _upb_MtDataEncoder_MaybePutModifiers(upb_MtDataEncoder* e, - char* ptr, upb_FieldType type, - uint64_t field_mod) { - upb_MtDataEncoderInternal* in = (upb_MtDataEncoderInternal*)e->internal; - uint32_t encoded_modifiers = 0; - if ((field_mod & kUpb_FieldModifier_IsRepeated) && - upb_FieldType_IsPackable(type)) { - bool field_is_packed = field_mod & kUpb_FieldModifier_IsPacked; - bool default_is_packed = in->state.msg_state.msg_modifiers & - kUpb_MessageModifier_DefaultIsPacked; - if (field_is_packed != default_is_packed) { - encoded_modifiers |= kUpb_EncodedFieldModifier_FlipPacked; - } - } - - if (type == kUpb_FieldType_String) { - bool field_validates_utf8 = field_mod & kUpb_FieldModifier_ValidateUtf8; - bool message_validates_utf8 = - in->state.msg_state.msg_modifiers & kUpb_MessageModifier_ValidateUtf8; - if (field_validates_utf8 != message_validates_utf8) { - // Old binaries do not recognize the field modifier. We need the failure - // mode to be too lax rather than too strict. Our caller should have - // handled this (see _upb_MessageDef_ValidateUtf8()). - assert(!message_validates_utf8); - encoded_modifiers |= kUpb_EncodedFieldModifier_FlipValidateUtf8; - } - } - - if (field_mod & kUpb_FieldModifier_IsProto3Singular) { - encoded_modifiers |= kUpb_EncodedFieldModifier_IsProto3Singular; - } - - if (field_mod & kUpb_FieldModifier_IsRequired) { - encoded_modifiers |= kUpb_EncodedFieldModifier_IsRequired; - } - - return upb_MtDataEncoder_PutModifier(e, ptr, encoded_modifiers); -} - -char* upb_MtDataEncoder_PutField(upb_MtDataEncoder* e, char* ptr, - upb_FieldType type, uint32_t field_num, - uint64_t field_mod) { - upb_MtDataEncoder_GetInternal(e, ptr); - - ptr = _upb_MtDataEncoder_MaybePutFieldSkip(e, ptr, field_num); - if (!ptr) return NULL; - - ptr = _upb_MtDataEncoder_PutFieldType(e, ptr, type, field_mod); - if (!ptr) return NULL; - - return _upb_MtDataEncoder_MaybePutModifiers(e, ptr, type, field_mod); -} - -char* upb_MtDataEncoder_StartOneof(upb_MtDataEncoder* e, char* ptr) { - upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); - if (in->state.msg_state.oneof_state == kUpb_OneofState_NotStarted) { - ptr = upb_MtDataEncoder_Put(e, ptr, _upb_FromBase92(kUpb_EncodedValue_End)); - } else { - ptr = upb_MtDataEncoder_Put( - e, ptr, _upb_FromBase92(kUpb_EncodedValue_OneofSeparator)); - } - in->state.msg_state.oneof_state = kUpb_OneofState_StartedOneof; - return ptr; -} - -char* upb_MtDataEncoder_PutOneofField(upb_MtDataEncoder* e, char* ptr, - uint32_t field_num) { - upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); - if (in->state.msg_state.oneof_state == kUpb_OneofState_EmittedOneofField) { - ptr = upb_MtDataEncoder_Put( - e, ptr, _upb_FromBase92(kUpb_EncodedValue_FieldSeparator)); - if (!ptr) return NULL; - } - ptr = upb_MtDataEncoder_PutBase92Varint(e, ptr, field_num, _upb_ToBase92(0), - _upb_ToBase92(63)); - in->state.msg_state.oneof_state = kUpb_OneofState_EmittedOneofField; - return ptr; -} - -char* upb_MtDataEncoder_StartEnum(upb_MtDataEncoder* e, char* ptr) { - upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); - in->state.enum_state.present_values_mask = 0; - in->state.enum_state.last_written_value = 0; - - return upb_MtDataEncoder_PutRaw(e, ptr, kUpb_EncodedVersion_EnumV1); -} - -static char* upb_MtDataEncoder_FlushDenseEnumMask(upb_MtDataEncoder* e, - char* ptr) { - upb_MtDataEncoderInternal* in = (upb_MtDataEncoderInternal*)e->internal; - ptr = upb_MtDataEncoder_Put(e, ptr, in->state.enum_state.present_values_mask); - in->state.enum_state.present_values_mask = 0; - in->state.enum_state.last_written_value += 5; - return ptr; -} - -char* upb_MtDataEncoder_PutEnumValue(upb_MtDataEncoder* e, char* ptr, - uint32_t val) { - // TODO: optimize this encoding. - upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); - UPB_ASSERT(val >= in->state.enum_state.last_written_value); - uint32_t delta = val - in->state.enum_state.last_written_value; - if (delta >= 5 && in->state.enum_state.present_values_mask) { - ptr = upb_MtDataEncoder_FlushDenseEnumMask(e, ptr); - if (!ptr) { - return NULL; - } - delta -= 5; - } - - if (delta >= 5) { - ptr = upb_MtDataEncoder_PutBase92Varint( - e, ptr, delta, kUpb_EncodedValue_MinSkip, kUpb_EncodedValue_MaxSkip); - in->state.enum_state.last_written_value += delta; - delta = 0; - } - - UPB_ASSERT((in->state.enum_state.present_values_mask >> delta) == 0); - in->state.enum_state.present_values_mask |= 1ULL << delta; - return ptr; -} - -char* upb_MtDataEncoder_EndEnum(upb_MtDataEncoder* e, char* ptr) { - upb_MtDataEncoderInternal* in = upb_MtDataEncoder_GetInternal(e, ptr); - if (!in->state.enum_state.present_values_mask) return ptr; - return upb_MtDataEncoder_FlushDenseEnumMask(e, ptr); -} - - -#include <stddef.h> - -// Must be last. - -// A MiniTable for an empty message, used for unlinked sub-messages. -const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty) = { - .UPB_PRIVATE(subs) = NULL, - .UPB_PRIVATE(fields) = NULL, - .UPB_PRIVATE(size) = 0, - .UPB_PRIVATE(field_count) = 0, - .UPB_PRIVATE(ext) = kUpb_ExtMode_NonExtendable, - .UPB_PRIVATE(dense_below) = 0, - .UPB_PRIVATE(table_mask) = -1, - .UPB_PRIVATE(required_count) = 0, -}; - // This should #undef all macros #defined in def.inc #undef UPB_SIZE
diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index e44b7b6..dfa9ffe 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h
@@ -4684,108 +4684,6 @@ #endif // UPB_WIRE_EPS_COPY_INPUT_STREAM_H_ -#ifndef UPB_BASE_INTERNAL_LOG2_H_ -#define UPB_BASE_INTERNAL_LOG2_H_ - -// Must be last. - -#ifdef __cplusplus -extern "C" { -#endif - -UPB_INLINE int upb_Log2Ceiling(int x) { - if (x <= 1) return 0; -#ifdef __GNUC__ - return 32 - __builtin_clz(x - 1); -#else - int lg2 = 0; - while ((1 << lg2) < x) lg2++; - return lg2; -#endif -} - -UPB_INLINE int upb_Log2CeilingSize(int x) { return 1 << upb_Log2Ceiling(x); } - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_BASE_INTERNAL_LOG2_H_ */ - -#ifndef UPB_HASH_INT_TABLE_H_ -#define UPB_HASH_INT_TABLE_H_ - - -// Must be last. - -typedef struct { - upb_table t; // For entries that don't fit in the array part. - const upb_tabval* array; // Array part of the table. See const note above. - size_t array_size; // Array part size. - size_t array_count; // Array part number of elements. -} upb_inttable; - -#ifdef __cplusplus -extern "C" { -#endif - -// Initialize a table. If memory allocation failed, false is returned and -// the table is uninitialized. -bool upb_inttable_init(upb_inttable* table, upb_Arena* a); - -// Returns the number of values in the table. -size_t upb_inttable_count(const upb_inttable* t); - -// Inserts the given key into the hashtable with the given value. -// The key must not already exist in the hash table. -// The value must not be UINTPTR_MAX. -// -// If a table resize was required but memory allocation failed, false is -// returned and the table is unchanged. -bool upb_inttable_insert(upb_inttable* t, uintptr_t key, upb_value val, - upb_Arena* a); - -// Looks up key in this table, returning "true" if the key was found. -// If v is non-NULL, copies the value for this key into *v. -bool upb_inttable_lookup(const upb_inttable* t, uintptr_t key, upb_value* v); - -// Removes an item from the table. Returns true if the remove was successful, -// and stores the removed item in *val if non-NULL. -bool upb_inttable_remove(upb_inttable* t, uintptr_t key, upb_value* val); - -// Updates an existing entry in an inttable. -// If the entry does not exist, returns false and does nothing. -// Unlike insert/remove, this does not invalidate iterators. -bool upb_inttable_replace(upb_inttable* t, uintptr_t key, upb_value val); - -// Optimizes the table for the current set of entries, for both memory use and -// lookup time. Client should call this after all entries have been inserted; -// inserting more entries is legal, but will likely require a table resize. -void upb_inttable_compact(upb_inttable* t, upb_Arena* a); - -// Iteration over inttable: -// -// intptr_t iter = UPB_INTTABLE_BEGIN; -// uintptr_t key; -// upb_value val; -// while (upb_inttable_next(t, &key, &val, &iter)) { -// // ... -// } - -#define UPB_INTTABLE_BEGIN -1 - -bool upb_inttable_next(const upb_inttable* t, uintptr_t* key, upb_value* val, - intptr_t* iter); -void upb_inttable_removeiter(upb_inttable* t, intptr_t* iter); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_HASH_INT_TABLE_H_ */ - #ifndef UPB_JSON_DECODE_H_ #define UPB_JSON_DECODE_H_ @@ -12208,24 +12106,6 @@ #endif // UPB_PORT_VSNPRINTF_COMPAT_H_ -#ifndef UPB_LEX_STRTOD_H_ -#define UPB_LEX_STRTOD_H_ - -// Must be last. - -#ifdef __cplusplus -extern "C" { -#endif - -double _upb_NoLocaleStrtod(const char *str, char **endptr); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_LEX_STRTOD_H_ */ - #ifndef UPB_PORT_ATOMIC_H_ #define UPB_PORT_ATOMIC_H_ @@ -12447,6 +12327,35 @@ #endif /* UPB_MESSAGE_INTERNAL_MAP_SORTER_H_ */ +#ifndef UPB_BASE_INTERNAL_LOG2_H_ +#define UPB_BASE_INTERNAL_LOG2_H_ + +// Must be last. + +#ifdef __cplusplus +extern "C" { +#endif + +UPB_INLINE int upb_Log2Ceiling(int x) { + if (x <= 1) return 0; +#ifdef __GNUC__ + return 32 - __builtin_clz(x - 1); +#else + int lg2 = 0; + while ((1 << lg2) < x) lg2++; + return lg2; +#endif +} + +UPB_INLINE int upb_Log2CeilingSize(int x) { return 1 << upb_Log2Ceiling(x); } + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_BASE_INTERNAL_LOG2_H_ */ + #ifndef UPB_MESSAGE_COMPARE_H_ #define UPB_MESSAGE_COMPARE_H_ @@ -12732,6 +12641,587 @@ #endif /* UPB_MINI_TABLE_COMPAT_H_ */ +#ifndef UPB_HASH_INT_TABLE_H_ +#define UPB_HASH_INT_TABLE_H_ + + +// Must be last. + +typedef struct { + upb_table t; // For entries that don't fit in the array part. + const upb_tabval* array; // Array part of the table. See const note above. + size_t array_size; // Array part size. + size_t array_count; // Array part number of elements. +} upb_inttable; + +#ifdef __cplusplus +extern "C" { +#endif + +// Initialize a table. If memory allocation failed, false is returned and +// the table is uninitialized. +bool upb_inttable_init(upb_inttable* table, upb_Arena* a); + +// Returns the number of values in the table. +size_t upb_inttable_count(const upb_inttable* t); + +// Inserts the given key into the hashtable with the given value. +// The key must not already exist in the hash table. +// The value must not be UINTPTR_MAX. +// +// If a table resize was required but memory allocation failed, false is +// returned and the table is unchanged. +bool upb_inttable_insert(upb_inttable* t, uintptr_t key, upb_value val, + upb_Arena* a); + +// Looks up key in this table, returning "true" if the key was found. +// If v is non-NULL, copies the value for this key into *v. +bool upb_inttable_lookup(const upb_inttable* t, uintptr_t key, upb_value* v); + +// Removes an item from the table. Returns true if the remove was successful, +// and stores the removed item in *val if non-NULL. +bool upb_inttable_remove(upb_inttable* t, uintptr_t key, upb_value* val); + +// Updates an existing entry in an inttable. +// If the entry does not exist, returns false and does nothing. +// Unlike insert/remove, this does not invalidate iterators. +bool upb_inttable_replace(upb_inttable* t, uintptr_t key, upb_value val); + +// Optimizes the table for the current set of entries, for both memory use and +// lookup time. Client should call this after all entries have been inserted; +// inserting more entries is legal, but will likely require a table resize. +void upb_inttable_compact(upb_inttable* t, upb_Arena* a); + +// Iteration over inttable: +// +// intptr_t iter = UPB_INTTABLE_BEGIN; +// uintptr_t key; +// upb_value val; +// while (upb_inttable_next(t, &key, &val, &iter)) { +// // ... +// } + +#define UPB_INTTABLE_BEGIN -1 + +bool upb_inttable_next(const upb_inttable* t, uintptr_t* key, upb_value* val, + intptr_t* iter); +void upb_inttable_removeiter(upb_inttable* t, intptr_t* iter); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_HASH_INT_TABLE_H_ */ + +#ifndef UPB_WIRE_INTERNAL_CONSTANTS_H_ +#define UPB_WIRE_INTERNAL_CONSTANTS_H_ + +#define kUpb_WireFormat_DefaultDepthLimit 100 + +// MessageSet wire format is: +// message MessageSet { +// repeated group Item = 1 { +// required int32 type_id = 2; +// required bytes message = 3; +// } +// } + +enum { + kUpb_MsgSet_Item = 1, + kUpb_MsgSet_TypeId = 2, + kUpb_MsgSet_Message = 3, +}; + +#endif /* UPB_WIRE_INTERNAL_CONSTANTS_H_ */ + +/* + * Internal implementation details of the decoder that are shared between + * decode.c and decode_fast.c. + */ + +#ifndef UPB_WIRE_INTERNAL_DECODER_H_ +#define UPB_WIRE_INTERNAL_DECODER_H_ + +#include "utf8_range.h" + +// Must be last. + +#define DECODE_NOGROUP (uint32_t) - 1 + +typedef struct upb_Decoder { + upb_EpsCopyInputStream input; + const upb_ExtensionRegistry* extreg; + const char* unknown; // Start of unknown data, preserve at buffer flip + upb_Message* unknown_msg; // Pointer to preserve data to + int depth; // Tracks recursion depth to bound stack usage. + uint32_t end_group; // field number of END_GROUP tag, else DECODE_NOGROUP. + uint16_t options; + bool missing_required; + union { + upb_Arena arena; + void* foo[UPB_ARENA_SIZE_HACK]; + }; + upb_DecodeStatus status; + jmp_buf err; + +#ifndef NDEBUG + const char* debug_tagstart; + const char* debug_valstart; +#endif +} upb_Decoder; + +/* Error function that will abort decoding with longjmp(). We can't declare this + * UPB_NORETURN, even though it is appropriate, because if we do then compilers + * will "helpfully" refuse to tailcall to it + * (see: https://stackoverflow.com/a/55657013), which will defeat a major goal + * of our optimizations. That is also why we must declare it in a separate file, + * otherwise the compiler will see that it calls longjmp() and deduce that it is + * noreturn. */ +const char* _upb_FastDecoder_ErrorJmp(upb_Decoder* d, int status); + +extern const uint8_t upb_utf8_offsets[]; + +UPB_INLINE +bool _upb_Decoder_VerifyUtf8Inline(const char* ptr, int len) { + return utf8_range_IsValid(ptr, len); +} + +const char* _upb_Decoder_CheckRequired(upb_Decoder* d, const char* ptr, + const upb_Message* msg, + const upb_MiniTable* m); + +/* x86-64 pointers always have the high 16 bits matching. So we can shift + * left 8 and right 8 without loss of information. */ +UPB_INLINE intptr_t decode_totable(const upb_MiniTable* tablep) { + return ((intptr_t)tablep << 8) | tablep->UPB_PRIVATE(table_mask); +} + +UPB_INLINE const upb_MiniTable* decode_totablep(intptr_t table) { + return (const upb_MiniTable*)(table >> 8); +} + +const char* _upb_Decoder_IsDoneFallback(upb_EpsCopyInputStream* e, + const char* ptr, int overrun); + +UPB_INLINE bool _upb_Decoder_IsDone(upb_Decoder* d, const char** ptr) { + return upb_EpsCopyInputStream_IsDoneWithCallback( + &d->input, ptr, &_upb_Decoder_IsDoneFallback); +} + +UPB_INLINE const char* _upb_Decoder_BufferFlipCallback( + upb_EpsCopyInputStream* e, const char* old_end, const char* new_start) { + upb_Decoder* d = (upb_Decoder*)e; + if (!old_end) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + + if (d->unknown) { + if (!UPB_PRIVATE(_upb_Message_AddUnknown)( + d->unknown_msg, d->unknown, old_end - d->unknown, &d->arena)) { + _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); + } + d->unknown = new_start; + } + return new_start; +} + +#if UPB_FASTTABLE +UPB_INLINE +const char* _upb_FastDecoder_TagDispatch(upb_Decoder* d, const char* ptr, + upb_Message* msg, intptr_t table, + uint64_t hasbits, uint64_t tag) { + const upb_MiniTable* table_p = decode_totablep(table); + uint8_t mask = table; + uint64_t data; + size_t idx = tag & mask; + UPB_ASSUME((idx & 7) == 0); + idx >>= 3; + data = table_p->UPB_PRIVATE(fasttable)[idx].field_data ^ tag; + UPB_MUSTTAIL return table_p->UPB_PRIVATE(fasttable)[idx].field_parser( + d, ptr, msg, table, hasbits, data); +} +#endif + +UPB_INLINE uint32_t _upb_FastDecoder_LoadTag(const char* ptr) { + uint16_t tag; + memcpy(&tag, ptr, 2); + return tag; +} + + +#endif /* UPB_WIRE_INTERNAL_DECODER_H_ */ + +#ifndef UPB_WIRE_INTERNAL_ENDIAN_H_ +#define UPB_WIRE_INTERNAL_ENDIAN_H_ + +#include <stdint.h> + +// Must be last. + +#ifdef __cplusplus +extern "C" { +#endif + +UPB_INLINE bool UPB_PRIVATE(_upb_IsLittleEndian)(void) { + const int x = 1; + return *(char*)&x == 1; +} + +UPB_INLINE uint32_t UPB_PRIVATE(_upb_BigEndian32)(uint32_t val) { + if (UPB_PRIVATE(_upb_IsLittleEndian)()) return val; + + return ((val & 0xff) << 24) | ((val & 0xff00) << 8) | + ((val & 0xff0000) >> 8) | ((val & 0xff000000) >> 24); +} + +UPB_INLINE uint64_t UPB_PRIVATE(_upb_BigEndian64)(uint64_t val) { + if (UPB_PRIVATE(_upb_IsLittleEndian)()) return val; + + return ((uint64_t)UPB_PRIVATE(_upb_BigEndian32)((uint32_t)val) << 32) | + UPB_PRIVATE(_upb_BigEndian32)((uint32_t)(val >> 32)); +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_WIRE_INTERNAL_ENDIAN_H_ */ + +#ifndef UPB_WIRE_READER_H_ +#define UPB_WIRE_READER_H_ + + +#ifndef UPB_WIRE_INTERNAL_READER_H_ +#define UPB_WIRE_INTERNAL_READER_H_ + +// Must be last. + +#define kUpb_WireReader_WireTypeBits 3 +#define kUpb_WireReader_WireTypeMask 7 + +typedef struct { + const char* ptr; + uint64_t val; +} UPB_PRIVATE(_upb_WireReader_LongVarint); + +#ifdef __cplusplus +extern "C" { +#endif + +UPB_PRIVATE(_upb_WireReader_LongVarint) +UPB_PRIVATE(_upb_WireReader_ReadLongVarint)(const char* ptr, uint64_t val); + +static UPB_FORCEINLINE const char* UPB_PRIVATE(_upb_WireReader_ReadVarint)( + const char* ptr, uint64_t* val, int maxlen, uint64_t maxval) { + uint64_t byte = (uint8_t)*ptr; + if (UPB_LIKELY((byte & 0x80) == 0)) { + *val = (uint32_t)byte; + return ptr + 1; + } + const char* start = ptr; + UPB_PRIVATE(_upb_WireReader_LongVarint) + res = UPB_PRIVATE(_upb_WireReader_ReadLongVarint)(ptr, byte); + if (!res.ptr || (maxlen < 10 && res.ptr - start > maxlen) || + res.val > maxval) { + return NULL; // Malformed. + } + *val = res.val; + return res.ptr; +} + +UPB_INLINE uint32_t UPB_PRIVATE(_upb_WireReader_GetFieldNumber)(uint32_t tag) { + return tag >> kUpb_WireReader_WireTypeBits; +} + +UPB_INLINE uint8_t UPB_PRIVATE(_upb_WireReader_GetWireType)(uint32_t tag) { + return tag & kUpb_WireReader_WireTypeMask; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif // UPB_WIRE_INTERNAL_READER_H_ + +#ifndef UPB_WIRE_TYPES_H_ +#define UPB_WIRE_TYPES_H_ + +// A list of types as they are encoded on the wire. +typedef enum { + kUpb_WireType_Varint = 0, + kUpb_WireType_64Bit = 1, + kUpb_WireType_Delimited = 2, + kUpb_WireType_StartGroup = 3, + kUpb_WireType_EndGroup = 4, + kUpb_WireType_32Bit = 5 +} upb_WireType; + +#endif /* UPB_WIRE_TYPES_H_ */ + +// Must be last. + +// The upb_WireReader interface is suitable for general-purpose parsing of +// protobuf binary wire format. It is designed to be used along with +// upb_EpsCopyInputStream for buffering, and all parsing routines in this file +// assume that at least kUpb_EpsCopyInputStream_SlopBytes worth of data is +// available to read without any bounds checks. + +#ifdef __cplusplus +extern "C" { +#endif + +// Parses a tag into `tag`, and returns a pointer past the end of the tag, or +// NULL if there was an error in the tag data. +// +// REQUIRES: there must be at least 10 bytes of data available at `ptr`. +// Bounds checks must be performed before calling this function, preferably +// by calling upb_EpsCopyInputStream_IsDone(). +static UPB_FORCEINLINE const char* upb_WireReader_ReadTag(const char* ptr, + uint32_t* tag) { + uint64_t val; + ptr = UPB_PRIVATE(_upb_WireReader_ReadVarint)(ptr, &val, 5, UINT32_MAX); + if (!ptr) return NULL; + *tag = val; + return ptr; +} + +// Given a tag, returns the field number. +UPB_INLINE uint32_t upb_WireReader_GetFieldNumber(uint32_t tag) { + return UPB_PRIVATE(_upb_WireReader_GetFieldNumber)(tag); +} + +// Given a tag, returns the wire type. +UPB_INLINE uint8_t upb_WireReader_GetWireType(uint32_t tag) { + return UPB_PRIVATE(_upb_WireReader_GetWireType)(tag); +} + +UPB_INLINE const char* upb_WireReader_ReadVarint(const char* ptr, + uint64_t* val) { + return UPB_PRIVATE(_upb_WireReader_ReadVarint)(ptr, val, 10, UINT64_MAX); +} + +// Skips data for a varint, returning a pointer past the end of the varint, or +// NULL if there was an error in the varint data. +// +// REQUIRES: there must be at least 10 bytes of data available at `ptr`. +// Bounds checks must be performed before calling this function, preferably +// by calling upb_EpsCopyInputStream_IsDone(). +UPB_INLINE const char* upb_WireReader_SkipVarint(const char* ptr) { + uint64_t val; + return upb_WireReader_ReadVarint(ptr, &val); +} + +// Reads a varint indicating the size of a delimited field into `size`, or +// NULL if there was an error in the varint data. +// +// REQUIRES: there must be at least 10 bytes of data available at `ptr`. +// Bounds checks must be performed before calling this function, preferably +// by calling upb_EpsCopyInputStream_IsDone(). +UPB_INLINE const char* upb_WireReader_ReadSize(const char* ptr, int* size) { + uint64_t size64; + ptr = upb_WireReader_ReadVarint(ptr, &size64); + if (!ptr || size64 >= INT32_MAX) return NULL; + *size = size64; + return ptr; +} + +// Reads a fixed32 field, performing byte swapping if necessary. +// +// REQUIRES: there must be at least 4 bytes of data available at `ptr`. +// Bounds checks must be performed before calling this function, preferably +// by calling upb_EpsCopyInputStream_IsDone(). +UPB_INLINE const char* upb_WireReader_ReadFixed32(const char* ptr, void* val) { + uint32_t uval; + memcpy(&uval, ptr, 4); + uval = UPB_PRIVATE(_upb_BigEndian32)(uval); + memcpy(val, &uval, 4); + return ptr + 4; +} + +// Reads a fixed64 field, performing byte swapping if necessary. +// +// REQUIRES: there must be at least 4 bytes of data available at `ptr`. +// Bounds checks must be performed before calling this function, preferably +// by calling upb_EpsCopyInputStream_IsDone(). +UPB_INLINE const char* upb_WireReader_ReadFixed64(const char* ptr, void* val) { + uint64_t uval; + memcpy(&uval, ptr, 8); + uval = UPB_PRIVATE(_upb_BigEndian64)(uval); + memcpy(val, &uval, 8); + return ptr + 8; +} + +const char* UPB_PRIVATE(_upb_WireReader_SkipGroup)( + const char* ptr, uint32_t tag, int depth_limit, + upb_EpsCopyInputStream* stream); + +// Skips data for a group, returning a pointer past the end of the group, or +// NULL if there was an error parsing the group. The `tag` argument should be +// the start group tag that begins the group. The `depth_limit` argument +// indicates how many levels of recursion the group is allowed to have before +// reporting a parse error (this limit exists to protect against stack +// overflow). +// +// TODO: evaluate how the depth_limit should be specified. Do users need +// control over this? +UPB_INLINE const char* upb_WireReader_SkipGroup( + const char* ptr, uint32_t tag, upb_EpsCopyInputStream* stream) { + return UPB_PRIVATE(_upb_WireReader_SkipGroup)(ptr, tag, 100, stream); +} + +UPB_INLINE const char* _upb_WireReader_SkipValue( + const char* ptr, uint32_t tag, int depth_limit, + upb_EpsCopyInputStream* stream) { + switch (upb_WireReader_GetWireType(tag)) { + case kUpb_WireType_Varint: + return upb_WireReader_SkipVarint(ptr); + case kUpb_WireType_32Bit: + return ptr + 4; + case kUpb_WireType_64Bit: + return ptr + 8; + case kUpb_WireType_Delimited: { + int size; + ptr = upb_WireReader_ReadSize(ptr, &size); + if (!ptr) return NULL; + ptr += size; + return ptr; + } + case kUpb_WireType_StartGroup: + return UPB_PRIVATE(_upb_WireReader_SkipGroup)(ptr, tag, depth_limit, + stream); + case kUpb_WireType_EndGroup: + return NULL; // Should be handled before now. + default: + return NULL; // Unknown wire type. + } +} + +// Skips data for a wire value of any type, returning a pointer past the end of +// the data, or NULL if there was an error parsing the group. The `tag` argument +// should be the tag that was just parsed. The `depth_limit` argument indicates +// how many levels of recursion a group is allowed to have before reporting a +// parse error (this limit exists to protect against stack overflow). +// +// REQUIRES: there must be at least 10 bytes of data available at `ptr`. +// Bounds checks must be performed before calling this function, preferably +// by calling upb_EpsCopyInputStream_IsDone(). +// +// TODO: evaluate how the depth_limit should be specified. Do users need +// control over this? +UPB_INLINE const char* upb_WireReader_SkipValue( + const char* ptr, uint32_t tag, upb_EpsCopyInputStream* stream) { + return _upb_WireReader_SkipValue(ptr, tag, 100, stream); +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif // UPB_WIRE_READER_H_ + +#ifndef UPB_LEX_STRTOD_H_ +#define UPB_LEX_STRTOD_H_ + +// Must be last. + +#ifdef __cplusplus +extern "C" { +#endif + +double _upb_NoLocaleStrtod(const char *str, char **endptr); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_LEX_STRTOD_H_ */ + +#ifndef UPB_MINI_DESCRIPTOR_INTERNAL_ENCODE_H_ +#define UPB_MINI_DESCRIPTOR_INTERNAL_ENCODE_H_ + +#include <stdint.h> + + +// Must be last. + +// If the input buffer has at least this many bytes available, the encoder call +// is guaranteed to succeed (as long as field number order is maintained). +#define kUpb_MtDataEncoder_MinSize 16 + +typedef struct { + char* end; // Limit of the buffer passed as a parameter. + // Aliased to internal-only members in .cc. + char internal[32]; +} upb_MtDataEncoder; + +#ifdef __cplusplus +extern "C" { +#endif + +// Encodes field/oneof information for a given message. The sequence of calls +// should look like: +// +// upb_MtDataEncoder e; +// char buf[256]; +// char* ptr = buf; +// e.end = ptr + sizeof(buf); +// unit64_t msg_mod = ...; // bitwise & of kUpb_MessageModifiers or zero +// ptr = upb_MtDataEncoder_StartMessage(&e, ptr, msg_mod); +// // Fields *must* be in field number order. +// ptr = upb_MtDataEncoder_PutField(&e, ptr, ...); +// ptr = upb_MtDataEncoder_PutField(&e, ptr, ...); +// ptr = upb_MtDataEncoder_PutField(&e, ptr, ...); +// +// // If oneofs are present. Oneofs must be encoded after regular fields. +// ptr = upb_MiniTable_StartOneof(&e, ptr) +// ptr = upb_MiniTable_PutOneofField(&e, ptr, ...); +// ptr = upb_MiniTable_PutOneofField(&e, ptr, ...); +// +// ptr = upb_MiniTable_StartOneof(&e, ptr); +// ptr = upb_MiniTable_PutOneofField(&e, ptr, ...); +// ptr = upb_MiniTable_PutOneofField(&e, ptr, ...); +// +// Oneofs must be encoded after all regular fields. +char* upb_MtDataEncoder_StartMessage(upb_MtDataEncoder* e, char* ptr, + uint64_t msg_mod); +char* upb_MtDataEncoder_PutField(upb_MtDataEncoder* e, char* ptr, + upb_FieldType type, uint32_t field_num, + uint64_t field_mod); +char* upb_MtDataEncoder_StartOneof(upb_MtDataEncoder* e, char* ptr); +char* upb_MtDataEncoder_PutOneofField(upb_MtDataEncoder* e, char* ptr, + uint32_t field_num); + +// Encodes the set of values for a given enum. The values must be given in +// order (after casting to uint32_t), and repeats are not allowed. +char* upb_MtDataEncoder_StartEnum(upb_MtDataEncoder* e, char* ptr); +char* upb_MtDataEncoder_PutEnumValue(upb_MtDataEncoder* e, char* ptr, + uint32_t val); +char* upb_MtDataEncoder_EndEnum(upb_MtDataEncoder* e, char* ptr); + +// Encodes an entire mini descriptor for an extension. +char* upb_MtDataEncoder_EncodeExtension(upb_MtDataEncoder* e, char* ptr, + upb_FieldType type, uint32_t field_num, + uint64_t field_mod); + +// Encodes an entire mini descriptor for a map. +char* upb_MtDataEncoder_EncodeMap(upb_MtDataEncoder* e, char* ptr, + upb_FieldType key_type, + upb_FieldType value_type, uint64_t key_mod, + uint64_t value_mod); + +// Encodes an entire mini descriptor for a message set. +char* upb_MtDataEncoder_EncodeMessageSet(upb_MtDataEncoder* e, char* ptr); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_MINI_DESCRIPTOR_INTERNAL_ENCODE_H_ */ + #ifndef UPB_REFLECTION_DEF_POOL_INTERNAL_H_ #define UPB_REFLECTION_DEF_POOL_INTERNAL_H_ @@ -13144,89 +13634,6 @@ #define UPB_REFLECTION_DESC_STATE_INTERNAL_H_ -#ifndef UPB_MINI_DESCRIPTOR_INTERNAL_ENCODE_H_ -#define UPB_MINI_DESCRIPTOR_INTERNAL_ENCODE_H_ - -#include <stdint.h> - - -// Must be last. - -// If the input buffer has at least this many bytes available, the encoder call -// is guaranteed to succeed (as long as field number order is maintained). -#define kUpb_MtDataEncoder_MinSize 16 - -typedef struct { - char* end; // Limit of the buffer passed as a parameter. - // Aliased to internal-only members in .cc. - char internal[32]; -} upb_MtDataEncoder; - -#ifdef __cplusplus -extern "C" { -#endif - -// Encodes field/oneof information for a given message. The sequence of calls -// should look like: -// -// upb_MtDataEncoder e; -// char buf[256]; -// char* ptr = buf; -// e.end = ptr + sizeof(buf); -// unit64_t msg_mod = ...; // bitwise & of kUpb_MessageModifiers or zero -// ptr = upb_MtDataEncoder_StartMessage(&e, ptr, msg_mod); -// // Fields *must* be in field number order. -// ptr = upb_MtDataEncoder_PutField(&e, ptr, ...); -// ptr = upb_MtDataEncoder_PutField(&e, ptr, ...); -// ptr = upb_MtDataEncoder_PutField(&e, ptr, ...); -// -// // If oneofs are present. Oneofs must be encoded after regular fields. -// ptr = upb_MiniTable_StartOneof(&e, ptr) -// ptr = upb_MiniTable_PutOneofField(&e, ptr, ...); -// ptr = upb_MiniTable_PutOneofField(&e, ptr, ...); -// -// ptr = upb_MiniTable_StartOneof(&e, ptr); -// ptr = upb_MiniTable_PutOneofField(&e, ptr, ...); -// ptr = upb_MiniTable_PutOneofField(&e, ptr, ...); -// -// Oneofs must be encoded after all regular fields. -char* upb_MtDataEncoder_StartMessage(upb_MtDataEncoder* e, char* ptr, - uint64_t msg_mod); -char* upb_MtDataEncoder_PutField(upb_MtDataEncoder* e, char* ptr, - upb_FieldType type, uint32_t field_num, - uint64_t field_mod); -char* upb_MtDataEncoder_StartOneof(upb_MtDataEncoder* e, char* ptr); -char* upb_MtDataEncoder_PutOneofField(upb_MtDataEncoder* e, char* ptr, - uint32_t field_num); - -// Encodes the set of values for a given enum. The values must be given in -// order (after casting to uint32_t), and repeats are not allowed. -char* upb_MtDataEncoder_StartEnum(upb_MtDataEncoder* e, char* ptr); -char* upb_MtDataEncoder_PutEnumValue(upb_MtDataEncoder* e, char* ptr, - uint32_t val); -char* upb_MtDataEncoder_EndEnum(upb_MtDataEncoder* e, char* ptr); - -// Encodes an entire mini descriptor for an extension. -char* upb_MtDataEncoder_EncodeExtension(upb_MtDataEncoder* e, char* ptr, - upb_FieldType type, uint32_t field_num, - uint64_t field_mod); - -// Encodes an entire mini descriptor for a map. -char* upb_MtDataEncoder_EncodeMap(upb_MtDataEncoder* e, char* ptr, - upb_FieldType key_type, - upb_FieldType value_type, uint64_t key_mod, - uint64_t value_mod); - -// Encodes an entire mini descriptor for a message set. -char* upb_MtDataEncoder_EncodeMessageSet(upb_MtDataEncoder* e, char* ptr); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_MINI_DESCRIPTOR_INTERNAL_ENCODE_H_ */ - // Must be last. // Manages the storage for mini descriptor strings as they are being encoded. @@ -13457,413 +13864,6 @@ #endif /* UPB_REFLECTION_METHOD_DEF_INTERNAL_H_ */ -#ifndef UPB_WIRE_INTERNAL_CONSTANTS_H_ -#define UPB_WIRE_INTERNAL_CONSTANTS_H_ - -#define kUpb_WireFormat_DefaultDepthLimit 100 - -// MessageSet wire format is: -// message MessageSet { -// repeated group Item = 1 { -// required int32 type_id = 2; -// required bytes message = 3; -// } -// } - -enum { - kUpb_MsgSet_Item = 1, - kUpb_MsgSet_TypeId = 2, - kUpb_MsgSet_Message = 3, -}; - -#endif /* UPB_WIRE_INTERNAL_CONSTANTS_H_ */ - -/* - * Internal implementation details of the decoder that are shared between - * decode.c and decode_fast.c. - */ - -#ifndef UPB_WIRE_INTERNAL_DECODER_H_ -#define UPB_WIRE_INTERNAL_DECODER_H_ - -#include "utf8_range.h" - -// Must be last. - -#define DECODE_NOGROUP (uint32_t) - 1 - -typedef struct upb_Decoder { - upb_EpsCopyInputStream input; - const upb_ExtensionRegistry* extreg; - const char* unknown; // Start of unknown data, preserve at buffer flip - upb_Message* unknown_msg; // Pointer to preserve data to - int depth; // Tracks recursion depth to bound stack usage. - uint32_t end_group; // field number of END_GROUP tag, else DECODE_NOGROUP. - uint16_t options; - bool missing_required; - union { - upb_Arena arena; - void* foo[UPB_ARENA_SIZE_HACK]; - }; - upb_DecodeStatus status; - jmp_buf err; - -#ifndef NDEBUG - const char* debug_tagstart; - const char* debug_valstart; -#endif -} upb_Decoder; - -/* Error function that will abort decoding with longjmp(). We can't declare this - * UPB_NORETURN, even though it is appropriate, because if we do then compilers - * will "helpfully" refuse to tailcall to it - * (see: https://stackoverflow.com/a/55657013), which will defeat a major goal - * of our optimizations. That is also why we must declare it in a separate file, - * otherwise the compiler will see that it calls longjmp() and deduce that it is - * noreturn. */ -const char* _upb_FastDecoder_ErrorJmp(upb_Decoder* d, int status); - -extern const uint8_t upb_utf8_offsets[]; - -UPB_INLINE -bool _upb_Decoder_VerifyUtf8Inline(const char* ptr, int len) { - return utf8_range_IsValid(ptr, len); -} - -const char* _upb_Decoder_CheckRequired(upb_Decoder* d, const char* ptr, - const upb_Message* msg, - const upb_MiniTable* m); - -/* x86-64 pointers always have the high 16 bits matching. So we can shift - * left 8 and right 8 without loss of information. */ -UPB_INLINE intptr_t decode_totable(const upb_MiniTable* tablep) { - return ((intptr_t)tablep << 8) | tablep->UPB_PRIVATE(table_mask); -} - -UPB_INLINE const upb_MiniTable* decode_totablep(intptr_t table) { - return (const upb_MiniTable*)(table >> 8); -} - -const char* _upb_Decoder_IsDoneFallback(upb_EpsCopyInputStream* e, - const char* ptr, int overrun); - -UPB_INLINE bool _upb_Decoder_IsDone(upb_Decoder* d, const char** ptr) { - return upb_EpsCopyInputStream_IsDoneWithCallback( - &d->input, ptr, &_upb_Decoder_IsDoneFallback); -} - -UPB_INLINE const char* _upb_Decoder_BufferFlipCallback( - upb_EpsCopyInputStream* e, const char* old_end, const char* new_start) { - upb_Decoder* d = (upb_Decoder*)e; - if (!old_end) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); - - if (d->unknown) { - if (!UPB_PRIVATE(_upb_Message_AddUnknown)( - d->unknown_msg, d->unknown, old_end - d->unknown, &d->arena)) { - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); - } - d->unknown = new_start; - } - return new_start; -} - -#if UPB_FASTTABLE -UPB_INLINE -const char* _upb_FastDecoder_TagDispatch(upb_Decoder* d, const char* ptr, - upb_Message* msg, intptr_t table, - uint64_t hasbits, uint64_t tag) { - const upb_MiniTable* table_p = decode_totablep(table); - uint8_t mask = table; - uint64_t data; - size_t idx = tag & mask; - UPB_ASSUME((idx & 7) == 0); - idx >>= 3; - data = table_p->UPB_PRIVATE(fasttable)[idx].field_data ^ tag; - UPB_MUSTTAIL return table_p->UPB_PRIVATE(fasttable)[idx].field_parser( - d, ptr, msg, table, hasbits, data); -} -#endif - -UPB_INLINE uint32_t _upb_FastDecoder_LoadTag(const char* ptr) { - uint16_t tag; - memcpy(&tag, ptr, 2); - return tag; -} - - -#endif /* UPB_WIRE_INTERNAL_DECODER_H_ */ - -#ifndef UPB_WIRE_INTERNAL_ENDIAN_H_ -#define UPB_WIRE_INTERNAL_ENDIAN_H_ - -#include <stdint.h> - -// Must be last. - -#ifdef __cplusplus -extern "C" { -#endif - -UPB_INLINE bool UPB_PRIVATE(_upb_IsLittleEndian)(void) { - const int x = 1; - return *(char*)&x == 1; -} - -UPB_INLINE uint32_t UPB_PRIVATE(_upb_BigEndian32)(uint32_t val) { - if (UPB_PRIVATE(_upb_IsLittleEndian)()) return val; - - return ((val & 0xff) << 24) | ((val & 0xff00) << 8) | - ((val & 0xff0000) >> 8) | ((val & 0xff000000) >> 24); -} - -UPB_INLINE uint64_t UPB_PRIVATE(_upb_BigEndian64)(uint64_t val) { - if (UPB_PRIVATE(_upb_IsLittleEndian)()) return val; - - return ((uint64_t)UPB_PRIVATE(_upb_BigEndian32)((uint32_t)val) << 32) | - UPB_PRIVATE(_upb_BigEndian32)((uint32_t)(val >> 32)); -} - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_WIRE_INTERNAL_ENDIAN_H_ */ - -#ifndef UPB_WIRE_READER_H_ -#define UPB_WIRE_READER_H_ - - -#ifndef UPB_WIRE_INTERNAL_READER_H_ -#define UPB_WIRE_INTERNAL_READER_H_ - -// Must be last. - -#define kUpb_WireReader_WireTypeBits 3 -#define kUpb_WireReader_WireTypeMask 7 - -typedef struct { - const char* ptr; - uint64_t val; -} UPB_PRIVATE(_upb_WireReader_LongVarint); - -#ifdef __cplusplus -extern "C" { -#endif - -UPB_PRIVATE(_upb_WireReader_LongVarint) -UPB_PRIVATE(_upb_WireReader_ReadLongVarint)(const char* ptr, uint64_t val); - -static UPB_FORCEINLINE const char* UPB_PRIVATE(_upb_WireReader_ReadVarint)( - const char* ptr, uint64_t* val, int maxlen, uint64_t maxval) { - uint64_t byte = (uint8_t)*ptr; - if (UPB_LIKELY((byte & 0x80) == 0)) { - *val = (uint32_t)byte; - return ptr + 1; - } - const char* start = ptr; - UPB_PRIVATE(_upb_WireReader_LongVarint) - res = UPB_PRIVATE(_upb_WireReader_ReadLongVarint)(ptr, byte); - if (!res.ptr || (maxlen < 10 && res.ptr - start > maxlen) || - res.val > maxval) { - return NULL; // Malformed. - } - *val = res.val; - return res.ptr; -} - -UPB_INLINE uint32_t UPB_PRIVATE(_upb_WireReader_GetFieldNumber)(uint32_t tag) { - return tag >> kUpb_WireReader_WireTypeBits; -} - -UPB_INLINE uint8_t UPB_PRIVATE(_upb_WireReader_GetWireType)(uint32_t tag) { - return tag & kUpb_WireReader_WireTypeMask; -} - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif // UPB_WIRE_INTERNAL_READER_H_ - -#ifndef UPB_WIRE_TYPES_H_ -#define UPB_WIRE_TYPES_H_ - -// A list of types as they are encoded on the wire. -typedef enum { - kUpb_WireType_Varint = 0, - kUpb_WireType_64Bit = 1, - kUpb_WireType_Delimited = 2, - kUpb_WireType_StartGroup = 3, - kUpb_WireType_EndGroup = 4, - kUpb_WireType_32Bit = 5 -} upb_WireType; - -#endif /* UPB_WIRE_TYPES_H_ */ - -// Must be last. - -// The upb_WireReader interface is suitable for general-purpose parsing of -// protobuf binary wire format. It is designed to be used along with -// upb_EpsCopyInputStream for buffering, and all parsing routines in this file -// assume that at least kUpb_EpsCopyInputStream_SlopBytes worth of data is -// available to read without any bounds checks. - -#ifdef __cplusplus -extern "C" { -#endif - -// Parses a tag into `tag`, and returns a pointer past the end of the tag, or -// NULL if there was an error in the tag data. -// -// REQUIRES: there must be at least 10 bytes of data available at `ptr`. -// Bounds checks must be performed before calling this function, preferably -// by calling upb_EpsCopyInputStream_IsDone(). -static UPB_FORCEINLINE const char* upb_WireReader_ReadTag(const char* ptr, - uint32_t* tag) { - uint64_t val; - ptr = UPB_PRIVATE(_upb_WireReader_ReadVarint)(ptr, &val, 5, UINT32_MAX); - if (!ptr) return NULL; - *tag = val; - return ptr; -} - -// Given a tag, returns the field number. -UPB_INLINE uint32_t upb_WireReader_GetFieldNumber(uint32_t tag) { - return UPB_PRIVATE(_upb_WireReader_GetFieldNumber)(tag); -} - -// Given a tag, returns the wire type. -UPB_INLINE uint8_t upb_WireReader_GetWireType(uint32_t tag) { - return UPB_PRIVATE(_upb_WireReader_GetWireType)(tag); -} - -UPB_INLINE const char* upb_WireReader_ReadVarint(const char* ptr, - uint64_t* val) { - return UPB_PRIVATE(_upb_WireReader_ReadVarint)(ptr, val, 10, UINT64_MAX); -} - -// Skips data for a varint, returning a pointer past the end of the varint, or -// NULL if there was an error in the varint data. -// -// REQUIRES: there must be at least 10 bytes of data available at `ptr`. -// Bounds checks must be performed before calling this function, preferably -// by calling upb_EpsCopyInputStream_IsDone(). -UPB_INLINE const char* upb_WireReader_SkipVarint(const char* ptr) { - uint64_t val; - return upb_WireReader_ReadVarint(ptr, &val); -} - -// Reads a varint indicating the size of a delimited field into `size`, or -// NULL if there was an error in the varint data. -// -// REQUIRES: there must be at least 10 bytes of data available at `ptr`. -// Bounds checks must be performed before calling this function, preferably -// by calling upb_EpsCopyInputStream_IsDone(). -UPB_INLINE const char* upb_WireReader_ReadSize(const char* ptr, int* size) { - uint64_t size64; - ptr = upb_WireReader_ReadVarint(ptr, &size64); - if (!ptr || size64 >= INT32_MAX) return NULL; - *size = size64; - return ptr; -} - -// Reads a fixed32 field, performing byte swapping if necessary. -// -// REQUIRES: there must be at least 4 bytes of data available at `ptr`. -// Bounds checks must be performed before calling this function, preferably -// by calling upb_EpsCopyInputStream_IsDone(). -UPB_INLINE const char* upb_WireReader_ReadFixed32(const char* ptr, void* val) { - uint32_t uval; - memcpy(&uval, ptr, 4); - uval = UPB_PRIVATE(_upb_BigEndian32)(uval); - memcpy(val, &uval, 4); - return ptr + 4; -} - -// Reads a fixed64 field, performing byte swapping if necessary. -// -// REQUIRES: there must be at least 4 bytes of data available at `ptr`. -// Bounds checks must be performed before calling this function, preferably -// by calling upb_EpsCopyInputStream_IsDone(). -UPB_INLINE const char* upb_WireReader_ReadFixed64(const char* ptr, void* val) { - uint64_t uval; - memcpy(&uval, ptr, 8); - uval = UPB_PRIVATE(_upb_BigEndian64)(uval); - memcpy(val, &uval, 8); - return ptr + 8; -} - -const char* UPB_PRIVATE(_upb_WireReader_SkipGroup)( - const char* ptr, uint32_t tag, int depth_limit, - upb_EpsCopyInputStream* stream); - -// Skips data for a group, returning a pointer past the end of the group, or -// NULL if there was an error parsing the group. The `tag` argument should be -// the start group tag that begins the group. The `depth_limit` argument -// indicates how many levels of recursion the group is allowed to have before -// reporting a parse error (this limit exists to protect against stack -// overflow). -// -// TODO: evaluate how the depth_limit should be specified. Do users need -// control over this? -UPB_INLINE const char* upb_WireReader_SkipGroup( - const char* ptr, uint32_t tag, upb_EpsCopyInputStream* stream) { - return UPB_PRIVATE(_upb_WireReader_SkipGroup)(ptr, tag, 100, stream); -} - -UPB_INLINE const char* _upb_WireReader_SkipValue( - const char* ptr, uint32_t tag, int depth_limit, - upb_EpsCopyInputStream* stream) { - switch (upb_WireReader_GetWireType(tag)) { - case kUpb_WireType_Varint: - return upb_WireReader_SkipVarint(ptr); - case kUpb_WireType_32Bit: - return ptr + 4; - case kUpb_WireType_64Bit: - return ptr + 8; - case kUpb_WireType_Delimited: { - int size; - ptr = upb_WireReader_ReadSize(ptr, &size); - if (!ptr) return NULL; - ptr += size; - return ptr; - } - case kUpb_WireType_StartGroup: - return UPB_PRIVATE(_upb_WireReader_SkipGroup)(ptr, tag, depth_limit, - stream); - case kUpb_WireType_EndGroup: - return NULL; // Should be handled before now. - default: - return NULL; // Unknown wire type. - } -} - -// Skips data for a wire value of any type, returning a pointer past the end of -// the data, or NULL if there was an error parsing the group. The `tag` argument -// should be the tag that was just parsed. The `depth_limit` argument indicates -// how many levels of recursion a group is allowed to have before reporting a -// parse error (this limit exists to protect against stack overflow). -// -// REQUIRES: there must be at least 10 bytes of data available at `ptr`. -// Bounds checks must be performed before calling this function, preferably -// by calling upb_EpsCopyInputStream_IsDone(). -// -// TODO: evaluate how the depth_limit should be specified. Do users need -// control over this? -UPB_INLINE const char* upb_WireReader_SkipValue( - const char* ptr, uint32_t tag, upb_EpsCopyInputStream* stream) { - return _upb_WireReader_SkipValue(ptr, tag, 100, stream); -} - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif // UPB_WIRE_READER_H_ - // This should #undef all macros #defined in def.inc #undef UPB_SIZE