src/perfetto_cmd/pbtxt_to_pb.cc - third_party/perfetto - Git at Google

 /*
  * Copyright (C) 2018 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #include "src/perfetto_cmd/pbtxt_to_pb.h"

 #include <ctype.h>
 #include <limits>
 #include <map>
 #include <optional>
 #include <set>
 #include <stack>
 #include <string>

 #include "perfetto/base/logging.h"
 #include "perfetto/ext/base/file_utils.h"
 #include "perfetto/ext/base/string_utils.h"
 #include "perfetto/ext/base/string_view.h"
 #include "perfetto/ext/base/utils.h"
 #include "perfetto/protozero/message.h"
 #include "perfetto/protozero/message_handle.h"
 #include "perfetto/protozero/scattered_heap_buffer.h"
 #include "src/perfetto_cmd/config.descriptor.h"

 #include "protos/perfetto/common/descriptor.gen.h"

 namespace perfetto {
 constexpr char kConfigProtoName[] = ".perfetto.protos.TraceConfig";

 using protos::gen::DescriptorProto;
 using protos::gen::EnumDescriptorProto;
 using protos::gen::EnumValueDescriptorProto;
 using protos::gen::FieldDescriptorProto;
 using protos::gen::FileDescriptorSet;

 namespace {

 constexpr bool IsOct(char c) {
   return (c >= '0' && c <= '7');
 }

 constexpr bool IsDigit(char c) {
   return (c >= '0' && c <= '9');
 }

 constexpr bool IsIdentifierStart(char c) {
   return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || c == '_';
 }

 constexpr bool IsIdentifierBody(char c) {
   return IsIdentifierStart(c) || IsDigit(c);
 }

 const char* FieldToTypeName(const FieldDescriptorProto* field) {
   switch (field->type()) {
     case FieldDescriptorProto::TYPE_UINT64:
       return "uint64";
     case FieldDescriptorProto::TYPE_UINT32:
       return "uint32";
     case FieldDescriptorProto::TYPE_INT64:
       return "int64";
     case FieldDescriptorProto::TYPE_SINT64:
       return "sint64";
     case FieldDescriptorProto::TYPE_INT32:
       return "int32";
     case FieldDescriptorProto::TYPE_SINT32:
       return "sint32";
     case FieldDescriptorProto::TYPE_FIXED64:
       return "fixed64";
     case FieldDescriptorProto::TYPE_SFIXED64:
       return "sfixed64";
     case FieldDescriptorProto::TYPE_FIXED32:
       return "fixed32";
     case FieldDescriptorProto::TYPE_SFIXED32:
       return "sfixed32";
     case FieldDescriptorProto::TYPE_DOUBLE:
       return "double";
     case FieldDescriptorProto::TYPE_FLOAT:
       return "float";
     case FieldDescriptorProto::TYPE_BOOL:
       return "bool";
     case FieldDescriptorProto::TYPE_STRING:
       return "string";
     case FieldDescriptorProto::TYPE_BYTES:
       return "bytes";
     case FieldDescriptorProto::TYPE_GROUP:
       return "group";
     case FieldDescriptorProto::TYPE_MESSAGE:
       return "message";
     case FieldDescriptorProto::TYPE_ENUM:
       return "enum";
   }
   // For gcc
   PERFETTO_FATAL("Non complete switch");
 }

 std::string Format(const char* fmt, std::map<std::string, std::string> args) {
   std::string result(fmt);
   for (const auto& key_value : args) {
     size_t start = result.find(key_value.first);
     PERFETTO_CHECK(start != std::string::npos);
     result.replace(start, key_value.first.size(), key_value.second);
     PERFETTO_CHECK(result.find(key_value.first) == std::string::npos);
   }
   return result;
 }

 enum ParseState {
   kWaitingForKey,
   kReadingKey,
   kWaitingForValue,
   kReadingStringValue,
   kReadingStringEscape,
   kReadingNumericValue,
   kReadingIdentifierValue,
 };

 struct Token {
   size_t offset;
   size_t column;
   size_t row;
   base::StringView txt;

   size_t size() const { return txt.size(); }
   std::string ToStdString() const { return txt.ToStdString(); }
 };

 struct ParserDelegateContext {
   const DescriptorProto* descriptor;
   protozero::Message* message;
   std::set<std::string> seen_fields;
 };

 class ParserDelegate {
  public:
   ParserDelegate(
       const DescriptorProto* descriptor,
       protozero::Message* message,
       ErrorReporter* reporter,
       std::map<std::string, const DescriptorProto*> name_to_descriptor,
       std::map<std::string, const EnumDescriptorProto*> name_to_enum)
       : reporter_(reporter),
         name_to_descriptor_(std::move(name_to_descriptor)),
         name_to_enum_(std::move(name_to_enum)) {
     ctx_.push(ParserDelegateContext{descriptor, message, {}});
   }

   void NumericField(Token key, Token value) {
     const FieldDescriptorProto* field =
         FindFieldByName(key, value,
                         {
                             FieldDescriptorProto::TYPE_UINT64,
                             FieldDescriptorProto::TYPE_UINT32,
                             FieldDescriptorProto::TYPE_INT64,
                             FieldDescriptorProto::TYPE_SINT64,
                             FieldDescriptorProto::TYPE_INT32,
                             FieldDescriptorProto::TYPE_SINT32,
                             FieldDescriptorProto::TYPE_FIXED64,
                             FieldDescriptorProto::TYPE_SFIXED64,
                             FieldDescriptorProto::TYPE_FIXED32,
                             FieldDescriptorProto::TYPE_SFIXED32,
                             FieldDescriptorProto::TYPE_DOUBLE,
                             FieldDescriptorProto::TYPE_FLOAT,
                         });
     if (!field)
       return;
     const auto& field_type = field->type();
     switch (field_type) {
       case FieldDescriptorProto::TYPE_UINT64:
         return VarIntField<uint64_t>(field, value);
       case FieldDescriptorProto::TYPE_UINT32:
         return VarIntField<uint32_t>(field, value);
       case FieldDescriptorProto::TYPE_INT64:
       case FieldDescriptorProto::TYPE_SINT64:
         return VarIntField<int64_t>(field, value);
       case FieldDescriptorProto::TYPE_INT32:
       case FieldDescriptorProto::TYPE_SINT32:
         return VarIntField<int32_t>(field, value);

       case FieldDescriptorProto::TYPE_FIXED64:
       case FieldDescriptorProto::TYPE_SFIXED64:
         return FixedField<int64_t>(field, value);

       case FieldDescriptorProto::TYPE_FIXED32:
       case FieldDescriptorProto::TYPE_SFIXED32:
         return FixedField<int32_t>(field, value);

       case FieldDescriptorProto::TYPE_DOUBLE:
         return FixedFloatField<double>(field, value);
       case FieldDescriptorProto::TYPE_FLOAT:
         return FixedFloatField<float>(field, value);

       case FieldDescriptorProto::TYPE_BOOL:
       case FieldDescriptorProto::TYPE_STRING:
       case FieldDescriptorProto::TYPE_BYTES:
       case FieldDescriptorProto::TYPE_GROUP:
       case FieldDescriptorProto::TYPE_MESSAGE:
       case FieldDescriptorProto::TYPE_ENUM:
         PERFETTO_FATAL("Invalid type");
     }
   }

   void StringField(Token key, Token value) {
     const FieldDescriptorProto* field =
         FindFieldByName(key, value,
                         {
                             FieldDescriptorProto::TYPE_STRING,
                             FieldDescriptorProto::TYPE_BYTES,
                         });
     if (!field)
       return;
     uint32_t field_id = static_cast<uint32_t>(field->number());
     const auto& field_type = field->type();
     PERFETTO_CHECK(field_type == FieldDescriptorProto::TYPE_STRING ||
                    field_type == FieldDescriptorProto::TYPE_BYTES);

     std::unique_ptr<char, base::FreeDeleter> s(
         static_cast<char*>(malloc(value.size())));
     size_t j = 0;
     const char* const txt = value.txt.data();
     for (size_t i = 0; i < value.size(); i++) {
       char c = txt[i];
       if (c == '\\') {
         if (i + 1 >= value.size()) {
           // This should be caught by the lexer.
           PERFETTO_FATAL("Escape at end of string.");
           return;
         }
         char next = txt[++i];
         switch (next) {
           case '\\':
           case '\'':
           case '"':
           case '?':
             s.get()[j++] = next;
             break;
           case 'a':
             s.get()[j++] = '\a';
             break;
           case 'b':
             s.get()[j++] = '\b';
             break;
           case 'f':
             s.get()[j++] = '\f';
             break;
           case 'n':
             s.get()[j++] = '\n';
             break;
           case 'r':
             s.get()[j++] = '\r';
             break;
           case 't':
             s.get()[j++] = '\t';
             break;
           case 'v':
             s.get()[j++] = '\v';
             break;
           case '0':
           case '1':
           case '2':
           case '3':
           case '4':
           case '5':
           case '6':
           case '7':
           case '8':
           case '9': {
             // Cases 8 and 9 are not really required and are only added for the
             // sake of error reporting.
             bool oct_err = false;
             if (i + 2 >= value.size() || !IsOct(txt[i + 1]) ||
                 !IsOct(txt[i + 2])) {
               oct_err = true;
             } else {
               char buf[4]{next, txt[++i], txt[++i], '\0'};
               auto octval = base::CStringToUInt32(buf, 8);
               if (!octval.has_value() || *octval > 0xff) {
                 oct_err = true;
               } else {
                 s.get()[j++] = static_cast<char>(static_cast<uint8_t>(*octval));
               }
             }
             if (oct_err) {
               AddError(value,
                        "Malformed string escape in $k in proto $n on '$v'. "
                        "\\NNN escapes must be exactly three octal digits <= "
                        "\\377 (0xff).",
                        std::map<std::string, std::string>{
                            {"$k", key.ToStdString()},
                            {"$n", descriptor_name()},
                            {"$v", value.ToStdString()},
                        });
             }
             break;
           }
           default:
             AddError(value,
                      "Unknown string escape in $k in "
                      "proto $n: '$v'",
                      std::map<std::string, std::string>{
                          {"$k", key.ToStdString()},
                          {"$n", descriptor_name()},
                          {"$v", value.ToStdString()},
                      });
             return;
         }
       } else {
         s.get()[j++] = c;
       }
     }
     msg()->AppendBytes(field_id, s.get(), j);
   }

   void IdentifierField(Token key, Token value) {
     const FieldDescriptorProto* field =
         FindFieldByName(key, value,
                         {
                             FieldDescriptorProto::TYPE_BOOL,
                             FieldDescriptorProto::TYPE_ENUM,
                         });
     if (!field)
       return;
     uint32_t field_id = static_cast<uint32_t>(field->number());
     const auto& field_type = field->type();
     if (field_type == FieldDescriptorProto::TYPE_BOOL) {
       if (value.txt != "true" && value.txt != "false") {
         AddError(value,
                  "Expected 'true' or 'false' for boolean field $k in "
                  "proto $n instead saw '$v'",
                  std::map<std::string, std::string>{
                      {"$k", key.ToStdString()},
                      {"$n", descriptor_name()},
                      {"$v", value.ToStdString()},
                  });
         return;
       }
       msg()->AppendTinyVarInt(field_id, value.txt == "true" ? 1 : 0);
     } else if (field_type == FieldDescriptorProto::TYPE_ENUM) {
       const std::string& type_name = field->type_name();
       const EnumDescriptorProto* enum_descriptor = name_to_enum_[type_name];
       PERFETTO_CHECK(enum_descriptor);
       bool found_value = false;
       int32_t enum_value_number = 0;
       for (const EnumValueDescriptorProto& enum_value :
            enum_descriptor->value()) {
         if (value.ToStdString() != enum_value.name())
           continue;
         found_value = true;
         enum_value_number = enum_value.number();
         break;
       }
       if (!found_value) {
         AddError(value,
                  "Unexpected value '$v' for enum field $k in "
                  "proto $n",
                  std::map<std::string, std::string>{
                      {"$v", value.ToStdString()},
                      {"$k", key.ToStdString()},
                      {"$n", descriptor_name()},
                  });
         return;
       }
       msg()->AppendVarInt<int32_t>(field_id, enum_value_number);
     }
   }

   bool BeginNestedMessage(Token key, Token value) {
     const FieldDescriptorProto* field =
         FindFieldByName(key, value,
                         {
                             FieldDescriptorProto::TYPE_MESSAGE,
                         });
     if (!field) {
       // FindFieldByName adds an error.
       return false;
     }
     uint32_t field_id = static_cast<uint32_t>(field->number());
     const std::string& type_name = field->type_name();
     const DescriptorProto* nested_descriptor = name_to_descriptor_[type_name];
     PERFETTO_CHECK(nested_descriptor);
     auto* nested_msg = msg()->BeginNestedMessage<protozero::Message>(field_id);
     ctx_.push(ParserDelegateContext{nested_descriptor, nested_msg, {}});
     return true;
   }

   void EndNestedMessage() {
     msg()->Finalize();
     ctx_.pop();
   }

   void Eof() {}

   void AddError(size_t row,
                 size_t column,
                 const char* fmt,
                 const std::map<std::string, std::string>& args) {
     reporter_->AddError(row, column, 0, Format(fmt, args));
   }

   void AddError(Token token,
                 const char* fmt,
                 const std::map<std::string, std::string>& args) {
     reporter_->AddError(token.row, token.column, token.size(),
                         Format(fmt, args));
   }

  private:
   template <typename T>
   void VarIntField(const FieldDescriptorProto* field, Token t) {
     uint32_t field_id = static_cast<uint32_t>(field->number());
     uint64_t n = 0;
     PERFETTO_CHECK(ParseInteger(t.txt, &n));
     if (field->type() == FieldDescriptorProto::TYPE_SINT64 ||
         field->type() == FieldDescriptorProto::TYPE_SINT32) {
       msg()->AppendSignedVarInt<T>(field_id, static_cast<T>(n));
     } else {
       msg()->AppendVarInt<T>(field_id, static_cast<T>(n));
     }
   }

   template <typename T>
   void FixedField(const FieldDescriptorProto* field, Token t) {
     uint32_t field_id = static_cast<uint32_t>(field->number());
     uint64_t n = 0;
     PERFETTO_CHECK(ParseInteger(t.txt, &n));
     msg()->AppendFixed<T>(field_id, static_cast<T>(n));
   }

   template <typename T>
   void FixedFloatField(const FieldDescriptorProto* field, Token t) {
     uint32_t field_id = static_cast<uint32_t>(field->number());
     std::optional<double> opt_n = base::StringToDouble(t.ToStdString());
     msg()->AppendFixed<T>(field_id, static_cast<T>(opt_n.value_or(0l)));
   }

   template <typename T>
   bool ParseInteger(base::StringView s, T* number_ptr) {
     uint64_t n = 0;
     PERFETTO_CHECK(sscanf(s.ToStdString().c_str(), "%" PRIu64, &n) == 1);
     PERFETTO_CHECK(n <= std::numeric_limits<T>::max());
     *number_ptr = static_cast<T>(n);
     return true;
   }

   const FieldDescriptorProto* FindFieldByName(
       Token key,
       Token value,
       std::set<FieldDescriptorProto::Type> valid_field_types) {
     const std::string field_name = key.ToStdString();
     const FieldDescriptorProto* field_descriptor = nullptr;
     for (const auto& f : descriptor()->field()) {
       if (f.name() == field_name) {
         field_descriptor = &f;
         break;
       }
     }

     if (!field_descriptor) {
       AddError(key, "No field named \"$n\" in proto $p",
                {
                    {"$n", field_name},
                    {"$p", descriptor_name()},
                });
       return nullptr;
     }

     bool is_repeated =
         field_descriptor->label() == FieldDescriptorProto::LABEL_REPEATED;
     auto it_and_inserted = ctx_.top().seen_fields.emplace(field_name);
     if (!it_and_inserted.second && !is_repeated) {
       AddError(key, "Saw non-repeating field '$f' more than once",
                {
                    {"$f", field_name},
                });
     }

     if (!valid_field_types.count(field_descriptor->type())) {
       AddError(value,
                "Expected value of type $t for field $k in proto $n "
                "instead saw '$v'",
                {
                    {"$t", FieldToTypeName(field_descriptor)},
                    {"$k", field_name},
                    {"$n", descriptor_name()},
                    {"$v", value.ToStdString()},
                });
       return nullptr;
     }

     return field_descriptor;
   }

   const DescriptorProto* descriptor() {
     PERFETTO_CHECK(!ctx_.empty());
     return ctx_.top().descriptor;
   }

   const std::string& descriptor_name() { return descriptor()->name(); }

   protozero::Message* msg() {
     PERFETTO_CHECK(!ctx_.empty());
     return ctx_.top().message;
   }

   std::stack<ParserDelegateContext> ctx_;
   ErrorReporter* reporter_;
   std::map<std::string, const DescriptorProto*> name_to_descriptor_;
   std::map<std::string, const EnumDescriptorProto*> name_to_enum_;
 };

 void Parse(const std::string& input, ParserDelegate* delegate) {
   ParseState state = kWaitingForKey;
   size_t column = 0;
   size_t row = 1;
   size_t depth = 0;
   bool saw_colon_for_this_key = false;
   bool saw_semicolon_for_this_value = true;
   bool comment_till_eol = false;
   Token key{};
   Token value{};

   for (size_t i = 0; i < input.size(); i++, column++) {
     bool last_character = i + 1 == input.size();
     char c = input.at(i);
     if (c == '\n') {
       column = 0;
       row++;
       if (comment_till_eol) {
         comment_till_eol = false;
         continue;
       }
     }
     if (comment_till_eol)
       continue;

     switch (state) {
       case kWaitingForKey:
         if (isspace(c))
           continue;
         if (c == '#') {
           comment_till_eol = true;
           continue;
         }
         if (c == '}') {
           if (depth == 0) {
             delegate->AddError(row, column, "Unmatched closing brace", {});
             return;
           }
           saw_semicolon_for_this_value = false;
           depth--;
           delegate->EndNestedMessage();
           continue;
         }
         if (!saw_semicolon_for_this_value && c == ';') {
           saw_semicolon_for_this_value = true;
           continue;
         }
         if (IsIdentifierStart(c)) {
           saw_colon_for_this_key = false;
           state = kReadingKey;
           key.offset = i;
           key.row = row;
           key.column = column;
           continue;
         }
         break;

       case kReadingKey:
         if (IsIdentifierBody(c))
           continue;
         key.txt = base::StringView(input.data() + key.offset, i - key.offset);
         state = kWaitingForValue;
         if (c == '#')
           comment_till_eol = true;
         continue;

       case kWaitingForValue:
         if (isspace(c))
           continue;
         if (c == '#') {
           comment_till_eol = true;
           continue;
         }
         value.offset = i;
         value.row = row;
         value.column = column;

         if (c == ':' && !saw_colon_for_this_key) {
           saw_colon_for_this_key = true;
           continue;
         }
         if (c == '"') {
           state = kReadingStringValue;
           continue;
         }
         if (c == '-' || IsDigit(c) || c == '.') {
           state = kReadingNumericValue;
           continue;
         }
         if (IsIdentifierStart(c)) {
           state = kReadingIdentifierValue;
           continue;
         }
         if (c == '{') {
           state = kWaitingForKey;
           depth++;
           value.txt = base::StringView(input.data() + value.offset, 1);
           if (!delegate->BeginNestedMessage(key, value)) {
             return;
           }
           continue;
         }
         break;

       case kReadingNumericValue:
         if (isspace(c) || c == ';' || last_character) {
           bool keep_last = last_character && !(isspace(c) || c == ';');
           size_t size = i - value.offset + (keep_last ? 1 : 0);
           value.txt = base::StringView(input.data() + value.offset, size);
           saw_semicolon_for_this_value = c == ';';
           state = kWaitingForKey;
           delegate->NumericField(key, value);
           continue;
         }
         if (IsDigit(c) || c == '.')
           continue;
         break;

       case kReadingStringValue:
         if (c == '\\') {
           state = kReadingStringEscape;
         } else if (c == '"') {
           size_t size = i - value.offset - 1;
           value.column++;
           value.txt = base::StringView(input.data() + value.offset + 1, size);
           saw_semicolon_for_this_value = false;
           state = kWaitingForKey;
           delegate->StringField(key, value);
         }
         continue;

       case kReadingStringEscape:
         state = kReadingStringValue;
         continue;

       case kReadingIdentifierValue:
         if (isspace(c) || c == ';' || c == '#' || last_character) {
           bool keep_last =
               last_character && !(isspace(c) || c == ';' || c == '#');
           size_t size = i - value.offset + (keep_last ? 1 : 0);
           value.txt = base::StringView(input.data() + value.offset, size);
           comment_till_eol = c == '#';
           saw_semicolon_for_this_value = c == ';';
           state = kWaitingForKey;
           delegate->IdentifierField(key, value);
           continue;
         }
         if (IsIdentifierBody(c)) {
           continue;
         }
         break;
     }
     delegate->AddError(row, column, "Unexpected character '$c'",
                        std::map<std::string, std::string>{
                            {"$c", std::string(1, c)},
                        });
     return;
   }  // for
   if (depth > 0)
     delegate->AddError(row, column, "Nested message not closed", {});
   if (state != kWaitingForKey)
     delegate->AddError(row, column, "Unexpected end of input", {});
   delegate->Eof();
 }

 void AddNestedDescriptors(
     const std::string& prefix,
     const DescriptorProto* descriptor,
     std::map<std::string, const DescriptorProto*>* name_to_descriptor,
     std::map<std::string, const EnumDescriptorProto*>* name_to_enum) {
   for (const EnumDescriptorProto& enum_descriptor : descriptor->enum_type()) {
     const std::string name = prefix + "." + enum_descriptor.name();
     (*name_to_enum)[name] = &enum_descriptor;
   }
   for (const DescriptorProto& nested_descriptor : descriptor->nested_type()) {
     const std::string name = prefix + "." + nested_descriptor.name();
     (*name_to_descriptor)[name] = &nested_descriptor;
     AddNestedDescriptors(name, &nested_descriptor, name_to_descriptor,
                          name_to_enum);
   }
 }

 }  // namespace

 ErrorReporter::ErrorReporter() = default;
 ErrorReporter::~ErrorReporter() = default;

 std::vector<uint8_t> PbtxtToPb(const std::string& input,
                                ErrorReporter* reporter) {
   std::map<std::string, const DescriptorProto*> name_to_descriptor;
   std::map<std::string, const EnumDescriptorProto*> name_to_enum;
   FileDescriptorSet file_descriptor_set;

   {
     file_descriptor_set.ParseFromArray(
         kConfigDescriptor.data(), static_cast<int>(kConfigDescriptor.size()));
     for (const auto& file_descriptor : file_descriptor_set.file()) {
       for (const auto& enum_descriptor : file_descriptor.enum_type()) {
         const std::string name =
             "." + file_descriptor.package() + "." + enum_descriptor.name();
         name_to_enum[name] = &enum_descriptor;
       }
       for (const auto& descriptor : file_descriptor.message_type()) {
         const std::string name =
             "." + file_descriptor.package() + "." + descriptor.name();
         name_to_descriptor[name] = &descriptor;
         AddNestedDescriptors(name, &descriptor, &name_to_descriptor,
                              &name_to_enum);
       }
     }
   }

   const DescriptorProto* descriptor = name_to_descriptor[kConfigProtoName];
   PERFETTO_CHECK(descriptor);

   protozero::HeapBuffered<protozero::Message> message;
   ParserDelegate delegate(descriptor, message.get(), reporter,
                           std::move(name_to_descriptor),
                           std::move(name_to_enum));
   Parse(input, &delegate);
   return message.SerializeAsArray();
 }

 }  // namespace perfetto
	/*
	* Copyright (C) 2018 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#include "src/perfetto_cmd/pbtxt_to_pb.h"

	#include <ctype.h>
	#include <limits>
	#include <map>
	#include <optional>
	#include <set>
	#include <stack>
	#include <string>

	#include "perfetto/base/logging.h"
	#include "perfetto/ext/base/file_utils.h"
	#include "perfetto/ext/base/string_utils.h"
	#include "perfetto/ext/base/string_view.h"
	#include "perfetto/ext/base/utils.h"
	#include "perfetto/protozero/message.h"
	#include "perfetto/protozero/message_handle.h"
	#include "perfetto/protozero/scattered_heap_buffer.h"
	#include "src/perfetto_cmd/config.descriptor.h"

	#include "protos/perfetto/common/descriptor.gen.h"

	namespace perfetto {
	constexpr char kConfigProtoName[] = ".perfetto.protos.TraceConfig";

	using protos::gen::DescriptorProto;
	using protos::gen::EnumDescriptorProto;
	using protos::gen::EnumValueDescriptorProto;
	using protos::gen::FieldDescriptorProto;
	using protos::gen::FileDescriptorSet;

	namespace {

	constexpr bool IsOct(char c) {
	return (c >= '0' && c <= '7');
	}

	constexpr bool IsDigit(char c) {
	return (c >= '0' && c <= '9');
	}

	constexpr bool IsIdentifierStart(char c) {
	return ('A' <= c && c <= 'Z') \|\| ('a' <= c && c <= 'z') \|\| c == '_';
	}

	constexpr bool IsIdentifierBody(char c) {
	return IsIdentifierStart(c) \|\| IsDigit(c);
	}

	const char* FieldToTypeName(const FieldDescriptorProto* field) {
	switch (field->type()) {
	case FieldDescriptorProto::TYPE_UINT64:
	return "uint64";
	case FieldDescriptorProto::TYPE_UINT32:
	return "uint32";
	case FieldDescriptorProto::TYPE_INT64:
	return "int64";
	case FieldDescriptorProto::TYPE_SINT64:
	return "sint64";
	case FieldDescriptorProto::TYPE_INT32:
	return "int32";
	case FieldDescriptorProto::TYPE_SINT32:
	return "sint32";
	case FieldDescriptorProto::TYPE_FIXED64:
	return "fixed64";
	case FieldDescriptorProto::TYPE_SFIXED64:
	return "sfixed64";
	case FieldDescriptorProto::TYPE_FIXED32:
	return "fixed32";
	case FieldDescriptorProto::TYPE_SFIXED32:
	return "sfixed32";
	case FieldDescriptorProto::TYPE_DOUBLE:
	return "double";
	case FieldDescriptorProto::TYPE_FLOAT:
	return "float";
	case FieldDescriptorProto::TYPE_BOOL:
	return "bool";
	case FieldDescriptorProto::TYPE_STRING:
	return "string";
	case FieldDescriptorProto::TYPE_BYTES:
	return "bytes";
	case FieldDescriptorProto::TYPE_GROUP:
	return "group";
	case FieldDescriptorProto::TYPE_MESSAGE:
	return "message";
	case FieldDescriptorProto::TYPE_ENUM:
	return "enum";
	}
	// For gcc
	PERFETTO_FATAL("Non complete switch");
	}

	std::string Format(const char* fmt, std::map<std::string, std::string> args) {
	std::string result(fmt);
	for (const auto& key_value : args) {
	size_t start = result.find(key_value.first);
	PERFETTO_CHECK(start != std::string::npos);
	result.replace(start, key_value.first.size(), key_value.second);
	PERFETTO_CHECK(result.find(key_value.first) == std::string::npos);
	}
	return result;
	}

	enum ParseState {
	kWaitingForKey,
	kReadingKey,
	kWaitingForValue,
	kReadingStringValue,
	kReadingStringEscape,
	kReadingNumericValue,
	kReadingIdentifierValue,
	};

	struct Token {
	size_t offset;
	size_t column;
	size_t row;
	base::StringView txt;

	size_t size() const { return txt.size(); }
	std::string ToStdString() const { return txt.ToStdString(); }
	};

	struct ParserDelegateContext {
	const DescriptorProto* descriptor;
	protozero::Message* message;
	std::set<std::string> seen_fields;
	};

	class ParserDelegate {
	public:
	ParserDelegate(
	const DescriptorProto* descriptor,
	protozero::Message* message,
	ErrorReporter* reporter,
	std::map<std::string, const DescriptorProto*> name_to_descriptor,
	std::map<std::string, const EnumDescriptorProto*> name_to_enum)
	: reporter_(reporter),
	name_to_descriptor_(std::move(name_to_descriptor)),
	name_to_enum_(std::move(name_to_enum)) {
	ctx_.push(ParserDelegateContext{descriptor, message, {}});
	}

	void NumericField(Token key, Token value) {
	const FieldDescriptorProto* field =
	FindFieldByName(key, value,
	{
	FieldDescriptorProto::TYPE_UINT64,
	FieldDescriptorProto::TYPE_UINT32,
	FieldDescriptorProto::TYPE_INT64,
	FieldDescriptorProto::TYPE_SINT64,
	FieldDescriptorProto::TYPE_INT32,
	FieldDescriptorProto::TYPE_SINT32,
	FieldDescriptorProto::TYPE_FIXED64,
	FieldDescriptorProto::TYPE_SFIXED64,
	FieldDescriptorProto::TYPE_FIXED32,
	FieldDescriptorProto::TYPE_SFIXED32,
	FieldDescriptorProto::TYPE_DOUBLE,
	FieldDescriptorProto::TYPE_FLOAT,
	});
	if (!field)
	return;
	const auto& field_type = field->type();
	switch (field_type) {
	case FieldDescriptorProto::TYPE_UINT64:
	return VarIntField<uint64_t>(field, value);
	case FieldDescriptorProto::TYPE_UINT32:
	return VarIntField<uint32_t>(field, value);
	case FieldDescriptorProto::TYPE_INT64:
	case FieldDescriptorProto::TYPE_SINT64:
	return VarIntField<int64_t>(field, value);
	case FieldDescriptorProto::TYPE_INT32:
	case FieldDescriptorProto::TYPE_SINT32:
	return VarIntField<int32_t>(field, value);

	case FieldDescriptorProto::TYPE_FIXED64:
	case FieldDescriptorProto::TYPE_SFIXED64:
	return FixedField<int64_t>(field, value);

	case FieldDescriptorProto::TYPE_FIXED32:
	case FieldDescriptorProto::TYPE_SFIXED32:
	return FixedField<int32_t>(field, value);

	case FieldDescriptorProto::TYPE_DOUBLE:
	return FixedFloatField<double>(field, value);
	case FieldDescriptorProto::TYPE_FLOAT:
	return FixedFloatField<float>(field, value);

	case FieldDescriptorProto::TYPE_BOOL:
	case FieldDescriptorProto::TYPE_STRING:
	case FieldDescriptorProto::TYPE_BYTES:
	case FieldDescriptorProto::TYPE_GROUP:
	case FieldDescriptorProto::TYPE_MESSAGE:
	case FieldDescriptorProto::TYPE_ENUM:
	PERFETTO_FATAL("Invalid type");
	}
	}

	void StringField(Token key, Token value) {
	const FieldDescriptorProto* field =
	FindFieldByName(key, value,
	{
	FieldDescriptorProto::TYPE_STRING,
	FieldDescriptorProto::TYPE_BYTES,
	});
	if (!field)
	return;
	uint32_t field_id = static_cast<uint32_t>(field->number());
	const auto& field_type = field->type();
	PERFETTO_CHECK(field_type == FieldDescriptorProto::TYPE_STRING \|\|
	field_type == FieldDescriptorProto::TYPE_BYTES);

	std::unique_ptr<char, base::FreeDeleter> s(
	static_cast<char*>(malloc(value.size())));
	size_t j = 0;
	const char* const txt = value.txt.data();
	for (size_t i = 0; i < value.size(); i++) {
	char c = txt[i];
	if (c == '\\') {
	if (i + 1 >= value.size()) {
	// This should be caught by the lexer.
	PERFETTO_FATAL("Escape at end of string.");
	return;
	}
	char next = txt[++i];
	switch (next) {
	case '\\':
	case '\'':
	case '"':
	case '?':
	s.get()[j++] = next;
	break;
	case 'a':
	s.get()[j++] = '\a';
	break;
	case 'b':
	s.get()[j++] = '\b';
	break;
	case 'f':
	s.get()[j++] = '\f';
	break;
	case 'n':
	s.get()[j++] = '\n';
	break;
	case 'r':
	s.get()[j++] = '\r';
	break;
	case 't':
	s.get()[j++] = '\t';
	break;
	case 'v':
	s.get()[j++] = '\v';
	break;
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9': {
	// Cases 8 and 9 are not really required and are only added for the
	// sake of error reporting.
	bool oct_err = false;
	if (i + 2 >= value.size() \|\| !IsOct(txt[i + 1]) \|\|
	!IsOct(txt[i + 2])) {
	oct_err = true;
	} else {
	char buf[4]{next, txt[++i], txt[++i], '\0'};
	auto octval = base::CStringToUInt32(buf, 8);
	if (!octval.has_value() \|\| *octval > 0xff) {
	oct_err = true;
	} else {
	s.get()[j++] = static_cast<char>(static_cast<uint8_t>(*octval));
	}
	}
	if (oct_err) {
	AddError(value,
	"Malformed string escape in $k in proto $n on '$v'. "
	"\\NNN escapes must be exactly three octal digits <= "
	"\\377 (0xff).",
	std::map<std::string, std::string>{
	{"$k", key.ToStdString()},
	{"$n", descriptor_name()},
	{"$v", value.ToStdString()},
	});
	}
	break;
	}
	default:
	AddError(value,
	"Unknown string escape in $k in "
	"proto $n: '$v'",
	std::map<std::string, std::string>{
	{"$k", key.ToStdString()},
	{"$n", descriptor_name()},
	{"$v", value.ToStdString()},
	});
	return;
	}
	} else {
	s.get()[j++] = c;
	}
	}
	msg()->AppendBytes(field_id, s.get(), j);
	}

	void IdentifierField(Token key, Token value) {
	const FieldDescriptorProto* field =
	FindFieldByName(key, value,
	{
	FieldDescriptorProto::TYPE_BOOL,
	FieldDescriptorProto::TYPE_ENUM,
	});
	if (!field)
	return;
	uint32_t field_id = static_cast<uint32_t>(field->number());
	const auto& field_type = field->type();
	if (field_type == FieldDescriptorProto::TYPE_BOOL) {
	if (value.txt != "true" && value.txt != "false") {
	AddError(value,
	"Expected 'true' or 'false' for boolean field $k in "
	"proto $n instead saw '$v'",
	std::map<std::string, std::string>{
	{"$k", key.ToStdString()},
	{"$n", descriptor_name()},
	{"$v", value.ToStdString()},
	});
	return;
	}
	msg()->AppendTinyVarInt(field_id, value.txt == "true" ? 1 : 0);
	} else if (field_type == FieldDescriptorProto::TYPE_ENUM) {
	const std::string& type_name = field->type_name();
	const EnumDescriptorProto* enum_descriptor = name_to_enum_[type_name];
	PERFETTO_CHECK(enum_descriptor);
	bool found_value = false;
	int32_t enum_value_number = 0;
	for (const EnumValueDescriptorProto& enum_value :
	enum_descriptor->value()) {
	if (value.ToStdString() != enum_value.name())
	continue;
	found_value = true;
	enum_value_number = enum_value.number();
	break;
	}
	if (!found_value) {
	AddError(value,
	"Unexpected value '$v' for enum field $k in "
	"proto $n",
	std::map<std::string, std::string>{
	{"$v", value.ToStdString()},
	{"$k", key.ToStdString()},
	{"$n", descriptor_name()},
	});
	return;
	}
	msg()->AppendVarInt<int32_t>(field_id, enum_value_number);
	}
	}

	bool BeginNestedMessage(Token key, Token value) {
	const FieldDescriptorProto* field =
	FindFieldByName(key, value,
	{
	FieldDescriptorProto::TYPE_MESSAGE,
	});
	if (!field) {
	// FindFieldByName adds an error.
	return false;
	}
	uint32_t field_id = static_cast<uint32_t>(field->number());
	const std::string& type_name = field->type_name();
	const DescriptorProto* nested_descriptor = name_to_descriptor_[type_name];
	PERFETTO_CHECK(nested_descriptor);
	auto* nested_msg = msg()->BeginNestedMessage<protozero::Message>(field_id);
	ctx_.push(ParserDelegateContext{nested_descriptor, nested_msg, {}});
	return true;
	}

	void EndNestedMessage() {
	msg()->Finalize();
	ctx_.pop();
	}

	void Eof() {}

	void AddError(size_t row,
	size_t column,
	const char* fmt,
	const std::map<std::string, std::string>& args) {
	reporter_->AddError(row, column, 0, Format(fmt, args));
	}

	void AddError(Token token,
	const char* fmt,
	const std::map<std::string, std::string>& args) {
	reporter_->AddError(token.row, token.column, token.size(),
	Format(fmt, args));
	}

	private:
	template <typename T>
	void VarIntField(const FieldDescriptorProto* field, Token t) {
	uint32_t field_id = static_cast<uint32_t>(field->number());
	uint64_t n = 0;
	PERFETTO_CHECK(ParseInteger(t.txt, &n));
	if (field->type() == FieldDescriptorProto::TYPE_SINT64 \|\|
	field->type() == FieldDescriptorProto::TYPE_SINT32) {
	msg()->AppendSignedVarInt<T>(field_id, static_cast<T>(n));
	} else {
	msg()->AppendVarInt<T>(field_id, static_cast<T>(n));
	}
	}

	template <typename T>
	void FixedField(const FieldDescriptorProto* field, Token t) {
	uint32_t field_id = static_cast<uint32_t>(field->number());
	uint64_t n = 0;
	PERFETTO_CHECK(ParseInteger(t.txt, &n));
	msg()->AppendFixed<T>(field_id, static_cast<T>(n));
	}

	template <typename T>
	void FixedFloatField(const FieldDescriptorProto* field, Token t) {
	uint32_t field_id = static_cast<uint32_t>(field->number());
	std::optional<double> opt_n = base::StringToDouble(t.ToStdString());
	msg()->AppendFixed<T>(field_id, static_cast<T>(opt_n.value_or(0l)));
	}

	template <typename T>
	bool ParseInteger(base::StringView s, T* number_ptr) {
	uint64_t n = 0;
	PERFETTO_CHECK(sscanf(s.ToStdString().c_str(), "%" PRIu64, &n) == 1);
	PERFETTO_CHECK(n <= std::numeric_limits<T>::max());
	*number_ptr = static_cast<T>(n);
	return true;
	}

	const FieldDescriptorProto* FindFieldByName(
	Token key,
	Token value,
	std::set<FieldDescriptorProto::Type> valid_field_types) {
	const std::string field_name = key.ToStdString();
	const FieldDescriptorProto* field_descriptor = nullptr;
	for (const auto& f : descriptor()->field()) {
	if (f.name() == field_name) {
	field_descriptor = &f;
	break;
	}
	}

	if (!field_descriptor) {
	AddError(key, "No field named \"$n\" in proto $p",
	{
	{"$n", field_name},
	{"$p", descriptor_name()},
	});
	return nullptr;
	}

	bool is_repeated =
	field_descriptor->label() == FieldDescriptorProto::LABEL_REPEATED;
	auto it_and_inserted = ctx_.top().seen_fields.emplace(field_name);
	if (!it_and_inserted.second && !is_repeated) {
	AddError(key, "Saw non-repeating field '$f' more than once",
	{
	{"$f", field_name},
	});
	}

	if (!valid_field_types.count(field_descriptor->type())) {
	AddError(value,
	"Expected value of type $t for field $k in proto $n "
	"instead saw '$v'",
	{
	{"$t", FieldToTypeName(field_descriptor)},
	{"$k", field_name},
	{"$n", descriptor_name()},
	{"$v", value.ToStdString()},
	});
	return nullptr;
	}

	return field_descriptor;
	}

	const DescriptorProto* descriptor() {
	PERFETTO_CHECK(!ctx_.empty());
	return ctx_.top().descriptor;
	}

	const std::string& descriptor_name() { return descriptor()->name(); }

	protozero::Message* msg() {
	PERFETTO_CHECK(!ctx_.empty());
	return ctx_.top().message;
	}

	std::stack<ParserDelegateContext> ctx_;
	ErrorReporter* reporter_;
	std::map<std::string, const DescriptorProto*> name_to_descriptor_;
	std::map<std::string, const EnumDescriptorProto*> name_to_enum_;
	};

	void Parse(const std::string& input, ParserDelegate* delegate) {
	ParseState state = kWaitingForKey;
	size_t column = 0;
	size_t row = 1;
	size_t depth = 0;
	bool saw_colon_for_this_key = false;
	bool saw_semicolon_for_this_value = true;
	bool comment_till_eol = false;
	Token key{};
	Token value{};

	for (size_t i = 0; i < input.size(); i++, column++) {
	bool last_character = i + 1 == input.size();
	char c = input.at(i);
	if (c == '\n') {
	column = 0;
	row++;
	if (comment_till_eol) {
	comment_till_eol = false;
	continue;
	}
	}
	if (comment_till_eol)
	continue;

	switch (state) {
	case kWaitingForKey:
	if (isspace(c))
	continue;
	if (c == '#') {
	comment_till_eol = true;
	continue;
	}
	if (c == '}') {
	if (depth == 0) {
	delegate->AddError(row, column, "Unmatched closing brace", {});
	return;
	}
	saw_semicolon_for_this_value = false;
	depth--;
	delegate->EndNestedMessage();
	continue;
	}
	if (!saw_semicolon_for_this_value && c == ';') {
	saw_semicolon_for_this_value = true;
	continue;
	}
	if (IsIdentifierStart(c)) {
	saw_colon_for_this_key = false;
	state = kReadingKey;
	key.offset = i;
	key.row = row;
	key.column = column;
	continue;
	}
	break;

	case kReadingKey:
	if (IsIdentifierBody(c))
	continue;
	key.txt = base::StringView(input.data() + key.offset, i - key.offset);
	state = kWaitingForValue;
	if (c == '#')
	comment_till_eol = true;
	continue;

	case kWaitingForValue:
	if (isspace(c))
	continue;
	if (c == '#') {
	comment_till_eol = true;
	continue;
	}
	value.offset = i;
	value.row = row;
	value.column = column;

	if (c == ':' && !saw_colon_for_this_key) {
	saw_colon_for_this_key = true;
	continue;
	}
	if (c == '"') {
	state = kReadingStringValue;
	continue;
	}
	if (c == '-' \|\| IsDigit(c) \|\| c == '.') {
	state = kReadingNumericValue;
	continue;
	}
	if (IsIdentifierStart(c)) {
	state = kReadingIdentifierValue;
	continue;
	}
	if (c == '{') {
	state = kWaitingForKey;
	depth++;
	value.txt = base::StringView(input.data() + value.offset, 1);
	if (!delegate->BeginNestedMessage(key, value)) {
	return;
	}
	continue;
	}
	break;

	case kReadingNumericValue:
	if (isspace(c) \|\| c == ';' \|\| last_character) {
	bool keep_last = last_character && !(isspace(c) \|\| c == ';');
	size_t size = i - value.offset + (keep_last ? 1 : 0);
	value.txt = base::StringView(input.data() + value.offset, size);
	saw_semicolon_for_this_value = c == ';';
	state = kWaitingForKey;
	delegate->NumericField(key, value);
	continue;
	}
	if (IsDigit(c) \|\| c == '.')
	continue;
	break;

	case kReadingStringValue:
	if (c == '\\') {
	state = kReadingStringEscape;
	} else if (c == '"') {
	size_t size = i - value.offset - 1;
	value.column++;
	value.txt = base::StringView(input.data() + value.offset + 1, size);
	saw_semicolon_for_this_value = false;
	state = kWaitingForKey;
	delegate->StringField(key, value);
	}
	continue;

	case kReadingStringEscape:
	state = kReadingStringValue;
	continue;

	case kReadingIdentifierValue:
	if (isspace(c) \|\| c == ';' \|\| c == '#' \|\| last_character) {
	bool keep_last =
	last_character && !(isspace(c) \|\| c == ';' \|\| c == '#');
	size_t size = i - value.offset + (keep_last ? 1 : 0);
	value.txt = base::StringView(input.data() + value.offset, size);
	comment_till_eol = c == '#';
	saw_semicolon_for_this_value = c == ';';
	state = kWaitingForKey;
	delegate->IdentifierField(key, value);
	continue;
	}
	if (IsIdentifierBody(c)) {
	continue;
	}
	break;
	}
	delegate->AddError(row, column, "Unexpected character '$c'",
	std::map<std::string, std::string>{
	{"$c", std::string(1, c)},
	});
	return;
	} // for
	if (depth > 0)
	delegate->AddError(row, column, "Nested message not closed", {});
	if (state != kWaitingForKey)
	delegate->AddError(row, column, "Unexpected end of input", {});
	delegate->Eof();
	}

	void AddNestedDescriptors(
	const std::string& prefix,
	const DescriptorProto* descriptor,
	std::map<std::string, const DescriptorProto> name_to_descriptor,
	std::map<std::string, const EnumDescriptorProto> name_to_enum) {
	for (const EnumDescriptorProto& enum_descriptor : descriptor->enum_type()) {
	const std::string name = prefix + "." + enum_descriptor.name();
	(*name_to_enum)[name] = &enum_descriptor;
	}
	for (const DescriptorProto& nested_descriptor : descriptor->nested_type()) {
	const std::string name = prefix + "." + nested_descriptor.name();
	(*name_to_descriptor)[name] = &nested_descriptor;
	AddNestedDescriptors(name, &nested_descriptor, name_to_descriptor,
	name_to_enum);
	}
	}

	} // namespace

	ErrorReporter::ErrorReporter() = default;
	ErrorReporter::~ErrorReporter() = default;

	std::vector<uint8_t> PbtxtToPb(const std::string& input,
	ErrorReporter* reporter) {
	std::map<std::string, const DescriptorProto*> name_to_descriptor;
	std::map<std::string, const EnumDescriptorProto*> name_to_enum;
	FileDescriptorSet file_descriptor_set;

	{
	file_descriptor_set.ParseFromArray(
	kConfigDescriptor.data(), static_cast<int>(kConfigDescriptor.size()));
	for (const auto& file_descriptor : file_descriptor_set.file()) {
	for (const auto& enum_descriptor : file_descriptor.enum_type()) {
	const std::string name =
	"." + file_descriptor.package() + "." + enum_descriptor.name();
	name_to_enum[name] = &enum_descriptor;
	}
	for (const auto& descriptor : file_descriptor.message_type()) {
	const std::string name =
	"." + file_descriptor.package() + "." + descriptor.name();
	name_to_descriptor[name] = &descriptor;
	AddNestedDescriptors(name, &descriptor, &name_to_descriptor,
	&name_to_enum);
	}
	}
	}

	const DescriptorProto* descriptor = name_to_descriptor[kConfigProtoName];
	PERFETTO_CHECK(descriptor);

	protozero::HeapBuffered<protozero::Message> message;
	ParserDelegate delegate(descriptor, message.get(), reporter,
	std::move(name_to_descriptor),
	std::move(name_to_enum));
	Parse(input, &delegate);
	return message.SerializeAsArray();
	}

	} // namespace perfetto