| // Protocol Buffers - Google's data interchange format | |
| // Copyright 2023 Google LLC. All rights reserved. | |
| // | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file or at | |
| // https://developers.google.com/open-source/licenses/bsd | |
| // Fast decoder: ~3x the speed of decode.c, but requires x86-64/ARM64. | |
| // Also the table size grows by 2x. | |
| // | |
| // Could potentially be ported to other 64-bit archs that pass at least six | |
| // arguments in registers and have 8 unused high bits in pointers. | |
| // | |
| // The overall design is to create specialized functions for every possible | |
| // field type (eg. oneof boolean field with a 1 byte tag) and then dispatch | |
| // to the specialized function as quickly as possible. | |
| #include "upb/wire/internal/decode_fast.h" | |
| #include "upb/message/array.h" | |
| #include "upb/message/internal/array.h" | |
| #include "upb/mini_table/sub.h" | |
| #include "upb/wire/internal/decoder.h" | |
| // Must be last. | |
| #include "upb/port/def.inc" | |
| #if UPB_FASTTABLE | |
| // The standard set of arguments passed to each parsing function. | |
| // Thanks to x86-64 calling conventions, these will stay in registers. | |
| #define UPB_PARSE_PARAMS \ | |
| upb_Decoder *d, const char *ptr, upb_Message *msg, intptr_t table, \ | |
| uint64_t hasbits, uint64_t data | |
| #define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data | |
| #define RETURN_GENERIC(m) \ | |
| /* Uncomment either of these for debugging purposes. */ \ | |
| /* fprintf(stderr, m); */ \ | |
| /*__builtin_trap(); */ \ | |
| return _upb_FastDecoder_DecodeGeneric(d, ptr, msg, table, hasbits, 0); | |
| typedef enum { | |
| CARD_s = 0, /* Singular (optional, non-repeated) */ | |
| CARD_o = 1, /* Oneof */ | |
| CARD_r = 2, /* Repeated */ | |
| CARD_p = 3 /* Packed Repeated */ | |
| } upb_card; | |
| UPB_NOINLINE | |
| static const char* fastdecode_isdonefallback(UPB_PARSE_PARAMS) { | |
| int overrun = data; | |
| ptr = _upb_EpsCopyInputStream_IsDoneFallbackInline( | |
| &d->input, ptr, overrun, _upb_Decoder_BufferFlipCallback); | |
| data = _upb_FastDecoder_LoadTag(ptr); | |
| UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); | |
| } | |
| UPB_FORCEINLINE | |
| const char* fastdecode_dispatch(UPB_PARSE_PARAMS) { | |
| int overrun; | |
| switch (upb_EpsCopyInputStream_IsDoneStatus(&d->input, ptr, &overrun)) { | |
| case kUpb_IsDoneStatus_Done: | |
| ((uint32_t*)msg)[2] |= hasbits; // Sync hasbits. | |
| const upb_MiniTable* m = decode_totablep(table); | |
| return UPB_UNLIKELY(m->UPB_PRIVATE(required_count)) | |
| ? _upb_Decoder_CheckRequired(d, ptr, msg, m) | |
| : ptr; | |
| case kUpb_IsDoneStatus_NotDone: | |
| break; | |
| case kUpb_IsDoneStatus_NeedFallback: | |
| data = overrun; | |
| UPB_MUSTTAIL return fastdecode_isdonefallback(UPB_PARSE_ARGS); | |
| } | |
| // Read two bytes of tag data (for a one-byte tag, the high byte is junk). | |
| data = _upb_FastDecoder_LoadTag(ptr); | |
| UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); | |
| } | |
| UPB_FORCEINLINE | |
| bool fastdecode_checktag(uint16_t data, int tagbytes) { | |
| if (tagbytes == 1) { | |
| return (data & 0xff) == 0; | |
| } else { | |
| return data == 0; | |
| } | |
| } | |
| UPB_FORCEINLINE | |
| const char* fastdecode_longsize(const char* ptr, int* size) { | |
| int i; | |
| UPB_ASSERT(*size & 0x80); | |
| *size &= 0xff; | |
| for (i = 0; i < 3; i++) { | |
| ptr++; | |
| size_t byte = (uint8_t)ptr[-1]; | |
| *size += (byte - 1) << (7 + 7 * i); | |
| if (UPB_LIKELY((byte & 0x80) == 0)) return ptr; | |
| } | |
| ptr++; | |
| size_t byte = (uint8_t)ptr[-1]; | |
| // len is limited by 2gb not 4gb, hence 8 and not 16 as normally expected | |
| // for a 32 bit varint. | |
| if (UPB_UNLIKELY(byte >= 8)) return NULL; | |
| *size += (byte - 1) << 28; | |
| return ptr; | |
| } | |
| UPB_FORCEINLINE | |
| const char* fastdecode_delimited( | |
| upb_Decoder* d, const char* ptr, | |
| upb_EpsCopyInputStream_ParseDelimitedFunc* func, void* ctx) { | |
| ptr++; | |
| // Sign-extend so varint greater than one byte becomes negative, causing | |
| // fast delimited parse to fail. | |
| int len = (int8_t)ptr[-1]; | |
| if (!upb_EpsCopyInputStream_TryParseDelimitedFast(&d->input, &ptr, len, func, | |
| ctx)) { | |
| // Slow case: Sub-message is >=128 bytes and/or exceeds the current buffer. | |
| // If it exceeds the buffer limit, limit/limit_ptr will change during | |
| // sub-message parsing, so we need to preserve delta, not limit. | |
| if (UPB_UNLIKELY(len & 0x80)) { | |
| // Size varint >1 byte (length >= 128). | |
| ptr = fastdecode_longsize(ptr, &len); | |
| if (!ptr) { | |
| // Corrupt wire format: size exceeded INT_MAX. | |
| return NULL; | |
| } | |
| } | |
| if (!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, len)) { | |
| // Corrupt wire format: invalid limit. | |
| return NULL; | |
| } | |
| int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, len); | |
| ptr = func(&d->input, ptr, ctx); | |
| upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta); | |
| } | |
| return ptr; | |
| } | |
| /* singular, oneof, repeated field handling ***********************************/ | |
| typedef struct { | |
| upb_Array* arr; | |
| void* end; | |
| } fastdecode_arr; | |
| typedef enum { | |
| FD_NEXT_ATLIMIT, | |
| FD_NEXT_SAMEFIELD, | |
| FD_NEXT_OTHERFIELD | |
| } fastdecode_next; | |
| typedef struct { | |
| void* dst; | |
| fastdecode_next next; | |
| uint32_t tag; | |
| } fastdecode_nextret; | |
| UPB_FORCEINLINE | |
| void* fastdecode_resizearr(upb_Decoder* d, void* dst, fastdecode_arr* farr, | |
| int valbytes) { | |
| if (UPB_UNLIKELY(dst == farr->end)) { | |
| size_t old_capacity = farr->arr->UPB_PRIVATE(capacity); | |
| size_t old_bytes = old_capacity * valbytes; | |
| size_t new_capacity = old_capacity * 2; | |
| size_t new_bytes = new_capacity * valbytes; | |
| char* old_ptr = upb_Array_MutableDataPtr(farr->arr); | |
| char* new_ptr = upb_Arena_Realloc(&d->arena, old_ptr, old_bytes, new_bytes); | |
| uint8_t elem_size_lg2 = __builtin_ctz(valbytes); | |
| UPB_PRIVATE(_upb_Array_SetTaggedPtr)(farr->arr, new_ptr, elem_size_lg2); | |
| farr->arr->UPB_PRIVATE(capacity) = new_capacity; | |
| dst = (void*)(new_ptr + (old_capacity * valbytes)); | |
| farr->end = (void*)(new_ptr + (new_capacity * valbytes)); | |
| } | |
| return dst; | |
| } | |
| UPB_FORCEINLINE | |
| bool fastdecode_tagmatch(uint32_t tag, uint64_t data, int tagbytes) { | |
| if (tagbytes == 1) { | |
| return (uint8_t)tag == (uint8_t)data; | |
| } else { | |
| return (uint16_t)tag == (uint16_t)data; | |
| } | |
| } | |
| UPB_FORCEINLINE | |
| void fastdecode_commitarr(void* dst, fastdecode_arr* farr, int valbytes) { | |
| farr->arr->UPB_PRIVATE(size) = | |
| (size_t)((char*)dst - (char*)upb_Array_MutableDataPtr(farr->arr)) / | |
| valbytes; | |
| } | |
| UPB_FORCEINLINE | |
| fastdecode_nextret fastdecode_nextrepeated(upb_Decoder* d, void* dst, | |
| const char** ptr, | |
| fastdecode_arr* farr, uint64_t data, | |
| int tagbytes, int valbytes) { | |
| fastdecode_nextret ret; | |
| dst = (char*)dst + valbytes; | |
| if (UPB_LIKELY(!_upb_Decoder_IsDone(d, ptr))) { | |
| ret.tag = _upb_FastDecoder_LoadTag(*ptr); | |
| if (fastdecode_tagmatch(ret.tag, data, tagbytes)) { | |
| ret.next = FD_NEXT_SAMEFIELD; | |
| } else { | |
| fastdecode_commitarr(dst, farr, valbytes); | |
| ret.next = FD_NEXT_OTHERFIELD; | |
| } | |
| } else { | |
| fastdecode_commitarr(dst, farr, valbytes); | |
| ret.next = FD_NEXT_ATLIMIT; | |
| } | |
| ret.dst = dst; | |
| return ret; | |
| } | |
| UPB_FORCEINLINE | |
| void* fastdecode_fieldmem(upb_Message* msg, uint64_t data) { | |
| size_t ofs = data >> 48; | |
| return (char*)msg + ofs; | |
| } | |
| UPB_FORCEINLINE | |
| void* fastdecode_getfield(upb_Decoder* d, const char* ptr, upb_Message* msg, | |
| uint64_t* data, uint64_t* hasbits, | |
| fastdecode_arr* farr, int valbytes, upb_card card) { | |
| switch (card) { | |
| case CARD_s: { | |
| uint8_t hasbit_index = *data >> 24; | |
| // Set hasbit and return pointer to scalar field. | |
| *hasbits |= 1ull << hasbit_index; | |
| return fastdecode_fieldmem(msg, *data); | |
| } | |
| case CARD_o: { | |
| uint16_t case_ofs = *data >> 32; | |
| uint32_t* oneof_case = UPB_PTR_AT(msg, case_ofs, uint32_t); | |
| uint8_t field_number = *data >> 24; | |
| *oneof_case = field_number; | |
| return fastdecode_fieldmem(msg, *data); | |
| } | |
| case CARD_r: { | |
| // Get pointer to upb_Array and allocate/expand if necessary. | |
| uint8_t elem_size_lg2 = __builtin_ctz(valbytes); | |
| upb_Array** arr_p = fastdecode_fieldmem(msg, *data); | |
| char* begin; | |
| ((uint32_t*)msg)[2] |= *hasbits; | |
| *hasbits = 0; | |
| if (UPB_LIKELY(!*arr_p)) { | |
| farr->arr = UPB_PRIVATE(_upb_Array_New)(&d->arena, 8, elem_size_lg2); | |
| *arr_p = farr->arr; | |
| } else { | |
| farr->arr = *arr_p; | |
| } | |
| begin = upb_Array_MutableDataPtr(farr->arr); | |
| farr->end = begin + (farr->arr->UPB_PRIVATE(capacity) * valbytes); | |
| *data = _upb_FastDecoder_LoadTag(ptr); | |
| return begin + (farr->arr->UPB_PRIVATE(size) * valbytes); | |
| } | |
| default: | |
| UPB_UNREACHABLE(); | |
| } | |
| } | |
| UPB_FORCEINLINE | |
| bool fastdecode_flippacked(uint64_t* data, int tagbytes) { | |
| *data ^= (0x2 ^ 0x0); // Patch data to match packed wiretype. | |
| return fastdecode_checktag(*data, tagbytes); | |
| } | |
| #define FASTDECODE_CHECKPACKED(tagbytes, card, func) \ | |
| if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ | |
| if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \ | |
| UPB_MUSTTAIL return func(UPB_PARSE_ARGS); \ | |
| } \ | |
| RETURN_GENERIC("packed check tag mismatch\n"); \ | |
| } | |
| /* varint fields **************************************************************/ | |
| UPB_FORCEINLINE | |
| uint64_t fastdecode_munge(uint64_t val, int valbytes, bool zigzag) { | |
| if (valbytes == 1) { | |
| return val != 0; | |
| } else if (zigzag) { | |
| if (valbytes == 4) { | |
| uint32_t n = val; | |
| return (n >> 1) ^ -(int32_t)(n & 1); | |
| } else if (valbytes == 8) { | |
| return (val >> 1) ^ -(int64_t)(val & 1); | |
| } | |
| UPB_UNREACHABLE(); | |
| } | |
| return val; | |
| } | |
| UPB_FORCEINLINE | |
| const char* fastdecode_varint64(const char* ptr, uint64_t* val) { | |
| ptr++; | |
| *val = (uint8_t)ptr[-1]; | |
| if (UPB_UNLIKELY(*val & 0x80)) { | |
| int i; | |
| for (i = 0; i < 8; i++) { | |
| ptr++; | |
| uint64_t byte = (uint8_t)ptr[-1]; | |
| *val += (byte - 1) << (7 + 7 * i); | |
| if (UPB_LIKELY((byte & 0x80) == 0)) goto done; | |
| } | |
| ptr++; | |
| uint64_t byte = (uint8_t)ptr[-1]; | |
| if (byte > 1) { | |
| return NULL; | |
| } | |
| *val += (byte - 1) << 63; | |
| } | |
| done: | |
| UPB_ASSUME(ptr != NULL); | |
| return ptr; | |
| } | |
| #define FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| valbytes, card, zigzag, packed) \ | |
| uint64_t val; \ | |
| void* dst; \ | |
| fastdecode_arr farr; \ | |
| \ | |
| FASTDECODE_CHECKPACKED(tagbytes, card, packed); \ | |
| \ | |
| dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ | |
| card); \ | |
| if (card == CARD_r) { \ | |
| if (UPB_UNLIKELY(!dst)) { \ | |
| RETURN_GENERIC("need array resize\n"); \ | |
| } \ | |
| } \ | |
| \ | |
| again: \ | |
| if (card == CARD_r) { \ | |
| dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ | |
| } \ | |
| \ | |
| ptr += tagbytes; \ | |
| ptr = fastdecode_varint64(ptr, &val); \ | |
| if (ptr == NULL) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ | |
| val = fastdecode_munge(val, valbytes, zigzag); \ | |
| memcpy(dst, &val, valbytes); \ | |
| \ | |
| if (card == CARD_r) { \ | |
| fastdecode_nextret ret = fastdecode_nextrepeated( \ | |
| d, dst, &ptr, &farr, data, tagbytes, valbytes); \ | |
| switch (ret.next) { \ | |
| case FD_NEXT_SAMEFIELD: \ | |
| dst = ret.dst; \ | |
| goto again; \ | |
| case FD_NEXT_OTHERFIELD: \ | |
| data = ret.tag; \ | |
| UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ | |
| case FD_NEXT_ATLIMIT: \ | |
| return ptr; \ | |
| } \ | |
| } \ | |
| \ | |
| UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); | |
| typedef struct { | |
| uint8_t valbytes; | |
| bool zigzag; | |
| void* dst; | |
| fastdecode_arr farr; | |
| } fastdecode_varintdata; | |
| UPB_FORCEINLINE | |
| const char* fastdecode_topackedvarint(upb_EpsCopyInputStream* e, | |
| const char* ptr, void* ctx) { | |
| upb_Decoder* d = (upb_Decoder*)e; | |
| fastdecode_varintdata* data = ctx; | |
| void* dst = data->dst; | |
| uint64_t val; | |
| while (!_upb_Decoder_IsDone(d, &ptr)) { | |
| dst = fastdecode_resizearr(d, dst, &data->farr, data->valbytes); | |
| ptr = fastdecode_varint64(ptr, &val); | |
| if (ptr == NULL) return NULL; | |
| val = fastdecode_munge(val, data->valbytes, data->zigzag); | |
| memcpy(dst, &val, data->valbytes); | |
| dst = (char*)dst + data->valbytes; | |
| } | |
| fastdecode_commitarr(dst, &data->farr, data->valbytes); | |
| return ptr; | |
| } | |
| #define FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| valbytes, zigzag, unpacked) \ | |
| fastdecode_varintdata ctx = {valbytes, zigzag}; \ | |
| \ | |
| FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked); \ | |
| \ | |
| ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \ | |
| valbytes, CARD_r); \ | |
| if (UPB_UNLIKELY(!ctx.dst)) { \ | |
| RETURN_GENERIC("need array resize\n"); \ | |
| } \ | |
| \ | |
| ptr += tagbytes; \ | |
| ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \ | |
| \ | |
| if (UPB_UNLIKELY(ptr == NULL)) { \ | |
| _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ | |
| } \ | |
| \ | |
| UPB_MUSTTAIL return fastdecode_dispatch(d, ptr, msg, table, hasbits, 0); | |
| #define FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| valbytes, card, zigzag, unpacked, packed) \ | |
| if (card == CARD_p) { \ | |
| FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| valbytes, zigzag, unpacked); \ | |
| } else { \ | |
| FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| valbytes, card, zigzag, packed); \ | |
| } | |
| #define z_ZZ true | |
| #define b_ZZ false | |
| #define v_ZZ false | |
| /* Generate all combinations: | |
| * {s,o,r,p} x {b1,v4,z4,v8,z8} x {1bt,2bt} */ | |
| #define F(card, type, valbytes, tagbytes) \ | |
| UPB_NOINLINE \ | |
| const char* upb_p##card##type##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ | |
| FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \ | |
| CARD_##card, type##_ZZ, \ | |
| upb_pr##type##valbytes##_##tagbytes##bt, \ | |
| upb_pp##type##valbytes##_##tagbytes##bt); \ | |
| } | |
| #define TYPES(card, tagbytes) \ | |
| F(card, b, 1, tagbytes) \ | |
| F(card, v, 4, tagbytes) \ | |
| F(card, v, 8, tagbytes) \ | |
| F(card, z, 4, tagbytes) \ | |
| F(card, z, 8, tagbytes) | |
| #define TAGBYTES(card) \ | |
| TYPES(card, 1) \ | |
| TYPES(card, 2) | |
| TAGBYTES(s) | |
| TAGBYTES(o) | |
| TAGBYTES(r) | |
| TAGBYTES(p) | |
| #undef z_ZZ | |
| #undef b_ZZ | |
| #undef v_ZZ | |
| #undef o_ONEOF | |
| #undef s_ONEOF | |
| #undef r_ONEOF | |
| #undef F | |
| #undef TYPES | |
| #undef TAGBYTES | |
| #undef FASTDECODE_UNPACKEDVARINT | |
| #undef FASTDECODE_PACKEDVARINT | |
| #undef FASTDECODE_VARINT | |
| /* fixed fields ***************************************************************/ | |
| #define FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| valbytes, card, packed) \ | |
| void* dst; \ | |
| fastdecode_arr farr; \ | |
| \ | |
| FASTDECODE_CHECKPACKED(tagbytes, card, packed) \ | |
| \ | |
| dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ | |
| card); \ | |
| if (card == CARD_r) { \ | |
| if (UPB_UNLIKELY(!dst)) { \ | |
| RETURN_GENERIC("couldn't allocate array in arena\n"); \ | |
| } \ | |
| } \ | |
| \ | |
| again: \ | |
| if (card == CARD_r) { \ | |
| dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ | |
| } \ | |
| \ | |
| ptr += tagbytes; \ | |
| memcpy(dst, ptr, valbytes); \ | |
| ptr += valbytes; \ | |
| \ | |
| if (card == CARD_r) { \ | |
| fastdecode_nextret ret = fastdecode_nextrepeated( \ | |
| d, dst, &ptr, &farr, data, tagbytes, valbytes); \ | |
| switch (ret.next) { \ | |
| case FD_NEXT_SAMEFIELD: \ | |
| dst = ret.dst; \ | |
| goto again; \ | |
| case FD_NEXT_OTHERFIELD: \ | |
| data = ret.tag; \ | |
| UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ | |
| case FD_NEXT_ATLIMIT: \ | |
| return ptr; \ | |
| } \ | |
| } \ | |
| \ | |
| UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); | |
| #define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| valbytes, unpacked) \ | |
| FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked) \ | |
| \ | |
| ptr += tagbytes; \ | |
| int size = (uint8_t)ptr[0]; \ | |
| ptr++; \ | |
| if (size & 0x80) { \ | |
| ptr = fastdecode_longsize(ptr, &size); \ | |
| } \ | |
| \ | |
| if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckDataSizeAvailable( \ | |
| &d->input, ptr, size) || \ | |
| (size % valbytes) != 0)) { \ | |
| _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ | |
| } \ | |
| \ | |
| upb_Array** arr_p = fastdecode_fieldmem(msg, data); \ | |
| upb_Array* arr = *arr_p; \ | |
| uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \ | |
| int elems = size / valbytes; \ | |
| \ | |
| if (UPB_LIKELY(!arr)) { \ | |
| *arr_p = arr = \ | |
| UPB_PRIVATE(_upb_Array_New)(&d->arena, elems, elem_size_lg2); \ | |
| if (!arr) { \ | |
| _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ | |
| } \ | |
| } else { \ | |
| UPB_PRIVATE(_upb_Array_ResizeUninitialized)(arr, elems, &d->arena); \ | |
| } \ | |
| \ | |
| char* dst = upb_Array_MutableDataPtr(arr); \ | |
| memcpy(dst, ptr, size); \ | |
| arr->UPB_PRIVATE(size) = elems; \ | |
| \ | |
| ptr += size; \ | |
| UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); | |
| #define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| valbytes, card, unpacked, packed) \ | |
| if (card == CARD_p) { \ | |
| FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| valbytes, unpacked); \ | |
| } else { \ | |
| FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| valbytes, card, packed); \ | |
| } | |
| /* Generate all combinations: | |
| * {s,o,r,p} x {f4,f8} x {1bt,2bt} */ | |
| #define F(card, valbytes, tagbytes) \ | |
| UPB_NOINLINE \ | |
| const char* upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ | |
| FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \ | |
| CARD_##card, upb_ppf##valbytes##_##tagbytes##bt, \ | |
| upb_prf##valbytes##_##tagbytes##bt); \ | |
| } | |
| #define TYPES(card, tagbytes) \ | |
| F(card, 4, tagbytes) \ | |
| F(card, 8, tagbytes) | |
| #define TAGBYTES(card) \ | |
| TYPES(card, 1) \ | |
| TYPES(card, 2) | |
| TAGBYTES(s) | |
| TAGBYTES(o) | |
| TAGBYTES(r) | |
| TAGBYTES(p) | |
| #undef F | |
| #undef TYPES | |
| #undef TAGBYTES | |
| #undef FASTDECODE_UNPACKEDFIXED | |
| #undef FASTDECODE_PACKEDFIXED | |
| /* string fields **************************************************************/ | |
| typedef const char* fastdecode_copystr_func(struct upb_Decoder* d, | |
| const char* ptr, upb_Message* msg, | |
| const upb_MiniTable* table, | |
| uint64_t hasbits, | |
| upb_StringView* dst); | |
| UPB_NOINLINE | |
| static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr, | |
| upb_Message* msg, intptr_t table, | |
| uint64_t hasbits, uint64_t data) { | |
| upb_StringView* dst = (upb_StringView*)data; | |
| if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { | |
| _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); | |
| } | |
| UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); | |
| } | |
| #define FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, validate_utf8) \ | |
| int size = (uint8_t)ptr[0]; /* Could plumb through hasbits. */ \ | |
| ptr++; \ | |
| if (size & 0x80) { \ | |
| ptr = fastdecode_longsize(ptr, &size); \ | |
| } \ | |
| \ | |
| if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, size))) { \ | |
| dst->size = 0; \ | |
| _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ | |
| } \ | |
| \ | |
| const char* s_ptr = ptr; \ | |
| ptr = upb_EpsCopyInputStream_ReadString(&d->input, &s_ptr, size, &d->arena); \ | |
| if (!ptr) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); \ | |
| dst->data = s_ptr; \ | |
| dst->size = size; \ | |
| \ | |
| if (validate_utf8) { \ | |
| data = (uint64_t)dst; \ | |
| UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ | |
| } else { \ | |
| UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ | |
| } | |
| UPB_NOINLINE | |
| static const char* fastdecode_longstring_utf8(struct upb_Decoder* d, | |
| const char* ptr, upb_Message* msg, | |
| intptr_t table, uint64_t hasbits, | |
| uint64_t data) { | |
| upb_StringView* dst = (upb_StringView*)data; | |
| FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, true); | |
| } | |
| UPB_NOINLINE | |
| static const char* fastdecode_longstring_noutf8( | |
| struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, | |
| uint64_t hasbits, uint64_t data) { | |
| upb_StringView* dst = (upb_StringView*)data; | |
| FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false); | |
| } | |
| UPB_FORCEINLINE | |
| void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size, int copy, | |
| char* data, size_t data_offset, upb_StringView* dst) { | |
| d->arena.UPB_PRIVATE(ptr) += copy; | |
| dst->data = data + data_offset; | |
| UPB_UNPOISON_MEMORY_REGION(data, copy); | |
| memcpy(data, ptr, copy); | |
| UPB_POISON_MEMORY_REGION(data + data_offset + size, | |
| copy - data_offset - size); | |
| } | |
| #define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| card, validate_utf8) \ | |
| upb_StringView* dst; \ | |
| fastdecode_arr farr; \ | |
| int64_t size; \ | |
| size_t arena_has; \ | |
| size_t common_has; \ | |
| char* buf; \ | |
| \ | |
| UPB_ASSERT(!upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0)); \ | |
| UPB_ASSERT(fastdecode_checktag(data, tagbytes)); \ | |
| \ | |
| dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ | |
| sizeof(upb_StringView), card); \ | |
| \ | |
| again: \ | |
| if (card == CARD_r) { \ | |
| dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \ | |
| } \ | |
| \ | |
| size = (uint8_t)ptr[tagbytes]; \ | |
| ptr += tagbytes + 1; \ | |
| dst->size = size; \ | |
| \ | |
| buf = d->arena.UPB_PRIVATE(ptr); \ | |
| arena_has = UPB_PRIVATE(_upb_ArenaHas)(&d->arena); \ | |
| common_has = UPB_MIN(arena_has, \ | |
| upb_EpsCopyInputStream_BytesAvailable(&d->input, ptr)); \ | |
| \ | |
| if (UPB_LIKELY(size <= 15 - tagbytes)) { \ | |
| if (arena_has < 16) goto longstr; \ | |
| fastdecode_docopy(d, ptr - tagbytes - 1, size, 16, buf, tagbytes + 1, \ | |
| dst); \ | |
| } else if (UPB_LIKELY(size <= 32)) { \ | |
| if (UPB_UNLIKELY(common_has < 32)) goto longstr; \ | |
| fastdecode_docopy(d, ptr, size, 32, buf, 0, dst); \ | |
| } else if (UPB_LIKELY(size <= 64)) { \ | |
| if (UPB_UNLIKELY(common_has < 64)) goto longstr; \ | |
| fastdecode_docopy(d, ptr, size, 64, buf, 0, dst); \ | |
| } else if (UPB_LIKELY(size < 128)) { \ | |
| if (UPB_UNLIKELY(common_has < 128)) goto longstr; \ | |
| fastdecode_docopy(d, ptr, size, 128, buf, 0, dst); \ | |
| } else { \ | |
| goto longstr; \ | |
| } \ | |
| \ | |
| ptr += size; \ | |
| \ | |
| if (card == CARD_r) { \ | |
| if (validate_utf8 && \ | |
| !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \ | |
| _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \ | |
| } \ | |
| fastdecode_nextret ret = fastdecode_nextrepeated( \ | |
| d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \ | |
| switch (ret.next) { \ | |
| case FD_NEXT_SAMEFIELD: \ | |
| dst = ret.dst; \ | |
| goto again; \ | |
| case FD_NEXT_OTHERFIELD: \ | |
| data = ret.tag; \ | |
| UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ | |
| case FD_NEXT_ATLIMIT: \ | |
| return ptr; \ | |
| } \ | |
| } \ | |
| \ | |
| if (card != CARD_r && validate_utf8) { \ | |
| data = (uint64_t)dst; \ | |
| UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ | |
| } \ | |
| \ | |
| UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ | |
| \ | |
| longstr: \ | |
| if (card == CARD_r) { \ | |
| fastdecode_commitarr(dst + 1, &farr, sizeof(upb_StringView)); \ | |
| } \ | |
| ptr--; \ | |
| if (validate_utf8) { \ | |
| UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table, \ | |
| hasbits, (uint64_t)dst); \ | |
| } else { \ | |
| UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table, \ | |
| hasbits, (uint64_t)dst); \ | |
| } | |
| #define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card, \ | |
| copyfunc, validate_utf8) \ | |
| upb_StringView* dst; \ | |
| fastdecode_arr farr; \ | |
| int64_t size; \ | |
| \ | |
| if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ | |
| RETURN_GENERIC("string field tag mismatch\n"); \ | |
| } \ | |
| \ | |
| if (UPB_UNLIKELY( \ | |
| !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0))) { \ | |
| UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS); \ | |
| } \ | |
| \ | |
| dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ | |
| sizeof(upb_StringView), card); \ | |
| \ | |
| again: \ | |
| if (card == CARD_r) { \ | |
| dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \ | |
| } \ | |
| \ | |
| size = (int8_t)ptr[tagbytes]; \ | |
| ptr += tagbytes + 1; \ | |
| \ | |
| if (UPB_UNLIKELY( \ | |
| !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, size))) { \ | |
| ptr--; \ | |
| if (validate_utf8) { \ | |
| return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, \ | |
| (uint64_t)dst); \ | |
| } else { \ | |
| return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, \ | |
| (uint64_t)dst); \ | |
| } \ | |
| } \ | |
| \ | |
| dst->data = ptr; \ | |
| dst->size = size; \ | |
| ptr = upb_EpsCopyInputStream_ReadStringAliased(&d->input, &dst->data, \ | |
| dst->size); \ | |
| \ | |
| if (card == CARD_r) { \ | |
| if (validate_utf8 && \ | |
| !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \ | |
| _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \ | |
| } \ | |
| fastdecode_nextret ret = fastdecode_nextrepeated( \ | |
| d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \ | |
| switch (ret.next) { \ | |
| case FD_NEXT_SAMEFIELD: \ | |
| dst = ret.dst; \ | |
| goto again; \ | |
| case FD_NEXT_OTHERFIELD: \ | |
| data = ret.tag; \ | |
| UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ | |
| case FD_NEXT_ATLIMIT: \ | |
| return ptr; \ | |
| } \ | |
| } \ | |
| \ | |
| if (card != CARD_r && validate_utf8) { \ | |
| data = (uint64_t)dst; \ | |
| UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ | |
| } \ | |
| \ | |
| UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); | |
| /* Generate all combinations: | |
| * {p,c} x {s,o,r} x {s, b} x {1bt,2bt} */ | |
| #define s_VALIDATE true | |
| #define b_VALIDATE false | |
| #define F(card, tagbytes, type) \ | |
| UPB_NOINLINE \ | |
| const char* upb_c##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ | |
| FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| CARD_##card, type##_VALIDATE); \ | |
| } \ | |
| const char* upb_p##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ | |
| FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| CARD_##card, upb_c##card##type##_##tagbytes##bt, \ | |
| type##_VALIDATE); \ | |
| } | |
| #define UTF8(card, tagbytes) \ | |
| F(card, tagbytes, s) \ | |
| F(card, tagbytes, b) | |
| #define TAGBYTES(card) \ | |
| UTF8(card, 1) \ | |
| UTF8(card, 2) | |
| TAGBYTES(s) | |
| TAGBYTES(o) | |
| TAGBYTES(r) | |
| #undef s_VALIDATE | |
| #undef b_VALIDATE | |
| #undef F | |
| #undef TAGBYTES | |
| #undef FASTDECODE_LONGSTRING | |
| #undef FASTDECODE_COPYSTRING | |
| #undef FASTDECODE_STRING | |
| /* message fields *************************************************************/ | |
| UPB_INLINE | |
| upb_Message* decode_newmsg_ceil(upb_Decoder* d, const upb_MiniTable* m, | |
| int msg_ceil_bytes) { | |
| size_t size = m->UPB_PRIVATE(size); | |
| char* msg_data; | |
| if (UPB_LIKELY(msg_ceil_bytes > 0 && | |
| UPB_PRIVATE(_upb_ArenaHas)(&d->arena) >= msg_ceil_bytes)) { | |
| UPB_ASSERT(size <= (size_t)msg_ceil_bytes); | |
| msg_data = d->arena.UPB_PRIVATE(ptr); | |
| d->arena.UPB_PRIVATE(ptr) += size; | |
| UPB_UNPOISON_MEMORY_REGION(msg_data, msg_ceil_bytes); | |
| memset(msg_data, 0, msg_ceil_bytes); | |
| UPB_POISON_MEMORY_REGION(msg_data + size, msg_ceil_bytes - size); | |
| } else { | |
| msg_data = (char*)upb_Arena_Malloc(&d->arena, size); | |
| memset(msg_data, 0, size); | |
| } | |
| return (upb_Message*)msg_data; | |
| } | |
| typedef struct { | |
| intptr_t table; | |
| upb_Message* msg; | |
| } fastdecode_submsgdata; | |
| UPB_FORCEINLINE | |
| const char* fastdecode_tosubmsg(upb_EpsCopyInputStream* e, const char* ptr, | |
| void* ctx) { | |
| upb_Decoder* d = (upb_Decoder*)e; | |
| fastdecode_submsgdata* submsg = ctx; | |
| ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0, 0); | |
| UPB_ASSUME(ptr != NULL); | |
| return ptr; | |
| } | |
| #define FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, \ | |
| msg_ceil_bytes, card) \ | |
| \ | |
| if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ | |
| RETURN_GENERIC("submessage field tag mismatch\n"); \ | |
| } \ | |
| \ | |
| if (--d->depth == 0) { \ | |
| _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_MaxDepthExceeded); \ | |
| } \ | |
| \ | |
| upb_Message** dst; \ | |
| uint32_t submsg_idx = (data >> 16) & 0xff; \ | |
| const upb_MiniTable* tablep = decode_totablep(table); \ | |
| const upb_MiniTable* subtablep = upb_MiniTableSub_Message( \ | |
| *UPB_PRIVATE(_upb_MiniTable_GetSubByIndex)(tablep, submsg_idx)); \ | |
| fastdecode_submsgdata submsg = {decode_totable(subtablep)}; \ | |
| fastdecode_arr farr; \ | |
| \ | |
| if (subtablep->UPB_PRIVATE(table_mask) == (uint8_t)-1) { \ | |
| d->depth++; \ | |
| RETURN_GENERIC("submessage doesn't have fast tables."); \ | |
| } \ | |
| \ | |
| dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ | |
| sizeof(upb_Message*), card); \ | |
| \ | |
| if (card == CARD_s) { \ | |
| ((uint32_t*)msg)[2] |= hasbits; \ | |
| hasbits = 0; \ | |
| } \ | |
| \ | |
| again: \ | |
| if (card == CARD_r) { \ | |
| dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_Message*)); \ | |
| } \ | |
| \ | |
| submsg.msg = *dst; \ | |
| \ | |
| if (card == CARD_r || UPB_LIKELY(!submsg.msg)) { \ | |
| *dst = submsg.msg = decode_newmsg_ceil(d, subtablep, msg_ceil_bytes); \ | |
| } \ | |
| \ | |
| ptr += tagbytes; \ | |
| ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg); \ | |
| \ | |
| if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) { \ | |
| _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ | |
| } \ | |
| \ | |
| if (card == CARD_r) { \ | |
| fastdecode_nextret ret = fastdecode_nextrepeated( \ | |
| d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_Message*)); \ | |
| switch (ret.next) { \ | |
| case FD_NEXT_SAMEFIELD: \ | |
| dst = ret.dst; \ | |
| goto again; \ | |
| case FD_NEXT_OTHERFIELD: \ | |
| d->depth++; \ | |
| data = ret.tag; \ | |
| UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ | |
| case FD_NEXT_ATLIMIT: \ | |
| d->depth++; \ | |
| return ptr; \ | |
| } \ | |
| } \ | |
| \ | |
| d->depth++; \ | |
| UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); | |
| #define F(card, tagbytes, size_ceil, ceil_arg) \ | |
| const char* upb_p##card##m_##tagbytes##bt_max##size_ceil##b( \ | |
| UPB_PARSE_PARAMS) { \ | |
| FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, ceil_arg, \ | |
| CARD_##card); \ | |
| } | |
| #define SIZES(card, tagbytes) \ | |
| F(card, tagbytes, 64, 64) \ | |
| F(card, tagbytes, 128, 128) \ | |
| F(card, tagbytes, 192, 192) \ | |
| F(card, tagbytes, 256, 256) \ | |
| F(card, tagbytes, max, -1) | |
| #define TAGBYTES(card) \ | |
| SIZES(card, 1) \ | |
| SIZES(card, 2) | |
| TAGBYTES(s) | |
| TAGBYTES(o) | |
| TAGBYTES(r) | |
| #undef TAGBYTES | |
| #undef SIZES | |
| #undef F | |
| #undef FASTDECODE_SUBMSG | |
| #endif /* UPB_FASTTABLE */ |