parquet.schema.MessageType.<init>()方法的使用及代码示例

x33g5p2x  于2022-01-25 转载在 其他  
字(10.4k)|赞(0)|评价(0)|浏览(97)

本文整理了Java中parquet.schema.MessageType.<init>()方法的一些代码示例，展示了MessageType.<init>()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。MessageType.<init>()方法的具体详情如下：
包路径：parquet.schema.MessageType
类名称：MessageType
方法名：<init>

MessageType.<init>介绍

暂无

代码示例

代码示例来源：origin: prestodb/presto

public static MessageType convert(final List<String> columnNames, final List<TypeInfo> columnTypes)
{
  return new MessageType("hive_schema", convertTypes(columnNames, columnTypes));
}

代码示例来源：origin: prestodb/presto

public static MessageType convert(final List<String> columnNames, final List<TypeInfo> columnTypes)
{
  return new MessageType("hive_schema", convertTypes(columnNames, columnTypes));
}

代码示例来源：origin: prestodb/presto

public static MessageType convert(final List<String> columnNames, final List<TypeInfo> columnTypes)
{
  return new MessageType("hive_schema", convertTypes(columnNames, columnTypes));
}

代码示例来源：origin: prestodb/presto

.collect(toList());
MessageType requestedSchema = new MessageType(fileSchema.getName(), fields);

代码示例来源：origin: apache/incubator-gobblin

private Type buildSchema() {
 JsonArray inputSchema = this.jsonSchema.getDataTypeValues();
 List<Type> parquetTypes = new ArrayList<>();
 for (JsonElement element : inputSchema) {
  JsonObject map = (JsonObject) element;
  JsonSchema elementSchema = new JsonSchema(map);
  String columnName = elementSchema.getColumnName();
  JsonElementConverter converter = JsonElementConversionFactory.getConverter(elementSchema, false);
  Type schemaType = converter.schema();
  this.converters.put(columnName, converter);
  parquetTypes.add(schemaType);
 }
 String docName = this.jsonSchema.getColumnName();
 switch (recordType) {
  case ROOT:
   return new MessageType(docName, parquetTypes);
  case CHILD:
   return new GroupType(this.jsonSchema.optionalOrRequired(), docName, parquetTypes);
  default:
   throw new RuntimeException("Unsupported Record type");
 }
}

代码示例来源：origin: prestodb/presto

@Test
public void testParquetTupleDomainPrimitive()
{
  HiveColumnHandle columnHandle = new HiveColumnHandle("my_primitive", HiveType.valueOf("bigint"), parseTypeSignature(StandardTypes.BIGINT), 0, REGULAR, Optional.empty());
  Domain singleValueDomain = Domain.singleValue(BIGINT, 123L);
  TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, singleValueDomain));
  MessageType fileSchema = new MessageType("hive_schema", new PrimitiveType(OPTIONAL, INT64, "my_primitive"));
  Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema);
  TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain);
  assertEquals(tupleDomain.getDomains().get().size(), 1);
  ColumnDescriptor descriptor = tupleDomain.getDomains().get().keySet().iterator().next();
  assertEquals(descriptor.getPath().length, 1);
  assertEquals(descriptor.getPath()[0], "my_primitive");
  Domain predicateDomain = Iterables.getOnlyElement(tupleDomain.getDomains().get().values());
  assertEquals(predicateDomain, singleValueDomain);
}

代码示例来源：origin: prestodb/presto

@Test
public void testParquetTupleDomainStruct()
{
  HiveColumnHandle columnHandle = new HiveColumnHandle("my_struct", HiveType.valueOf("struct<a:int,b:int>"), parseTypeSignature(StandardTypes.ROW), 0, REGULAR, Optional.empty());
  RowType.Field rowField = new RowType.Field(Optional.of("my_struct"), INTEGER);
  RowType rowType = RowType.from(ImmutableList.of(rowField));
  TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(rowType)));
  MessageType fileSchema = new MessageType("hive_schema",
      new GroupType(OPTIONAL, "my_struct",
          new PrimitiveType(OPTIONAL, INT32, "a"),
          new PrimitiveType(OPTIONAL, INT32, "b")));
  Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema);
  TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain);
  assertTrue(tupleDomain.getDomains().get().isEmpty());
}

代码示例来源：origin: prestodb/presto

@Test
public void testParquetTupleDomainPrimitiveArray()
{
  HiveColumnHandle columnHandle = new HiveColumnHandle("my_array", HiveType.valueOf("array<int>"), parseTypeSignature(StandardTypes.ARRAY), 0, REGULAR, Optional.empty());
  TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(INTEGER))));
  MessageType fileSchema = new MessageType("hive_schema",
      new GroupType(OPTIONAL, "my_array",
          new GroupType(REPEATED, "bag", new PrimitiveType(OPTIONAL, INT32, "array_element"))));
  Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema);
  TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain);
  assertTrue(tupleDomain.getDomains().get().isEmpty());
}

代码示例来源：origin: prestodb/presto

@Test
public void testParquetTupleDomainStructArray()
{
  HiveColumnHandle columnHandle = new HiveColumnHandle("my_array_struct", HiveType.valueOf("array<struct<a:int>>"), parseTypeSignature(StandardTypes.ARRAY), 0, REGULAR, Optional.empty());
  RowType.Field rowField = new RowType.Field(Optional.of("a"), INTEGER);
  RowType rowType = RowType.from(ImmutableList.of(rowField));
  TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(rowType))));
  MessageType fileSchema = new MessageType("hive_schema",
      new GroupType(OPTIONAL, "my_array_struct",
          new GroupType(REPEATED, "bag",
              new GroupType(OPTIONAL, "array_element", new PrimitiveType(OPTIONAL, INT32, "a")))));
  Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema);
  TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain);
  assertTrue(tupleDomain.getDomains().get().isEmpty());
}

代码示例来源：origin: prestodb/presto

@Test
public void testParquetTupleDomainMap()
{
  HiveColumnHandle columnHandle = new HiveColumnHandle("my_map", HiveType.valueOf("map<int,int>"), parseTypeSignature(StandardTypes.MAP), 0, REGULAR, Optional.empty());
  MapType mapType = new MapType(
      INTEGER,
      INTEGER,
      methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"),
      methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"),
      methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"),
      methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"));
  TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(mapType)));
  MessageType fileSchema = new MessageType("hive_schema",
      new GroupType(OPTIONAL, "my_map",
          new GroupType(REPEATED, "map",
              new PrimitiveType(REQUIRED, INT32, "key"),
              new PrimitiveType(OPTIONAL, INT32, "value"))));
  Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema);
  TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain);
  assertTrue(tupleDomain.getDomains().get().isEmpty());
}

代码示例来源：origin: com.twitter/parquet-thrift

public MessageType getConvertedMessageType() {
 // the root should be a GroupType
 if (currentType == null)
  return new MessageType(currentName, new ArrayList<Type>());
 GroupType rootType = currentType.asGroupType();
 return new MessageType(currentName, rootType.getFields());
}

代码示例来源：origin: com.twitter/parquet-pig

/**
 *
 * @param pigSchema the pig schema
 * @return the resulting Parquet schema
 */
public MessageType convert(Schema pigSchema) {
 return new MessageType("pig_schema", convertTypes(pigSchema));
}

代码示例来源：origin: com.facebook.presto.hive/hive-apache

public static MessageType convert(final List<String> columnNames, final List<TypeInfo> columnTypes) {
 final MessageType schema = new MessageType("hive_schema", convertTypes(columnNames, columnTypes));
 return schema;
}

代码示例来源：origin: asakusafw/asakusafw

private MessageType computeSchema(DataModelDescriptor descriptor) {
  List<Type> fields = new ArrayList<>();
  for (PropertyDescriptor property : descriptor.getPropertyDescriptors()) {
    Type field = computeParquetType(property);
    fields.add(field);
  }
  return new MessageType(
      descriptor.getDataModelClass().getName(),
      fields);
}

代码示例来源：origin: com.facebook.presto.hive/hive-apache

public MessageType union(MessageType toMerge, boolean strict) {
 return new MessageType(this.getName(), mergeFields(toMerge, strict));
}

代码示例来源：origin: com.facebook.presto/presto-hive

public static MessageType convert(final List<String> columnNames, final List<TypeInfo> columnTypes)
{
  return new MessageType("hive_schema", convertTypes(columnNames, columnTypes));
}

代码示例来源：origin: com.facebook.presto/presto-hive

public static MessageType convert(final List<String> columnNames, final List<TypeInfo> columnTypes)
{
  return new MessageType("hive_schema", convertTypes(columnNames, columnTypes));
}

代码示例来源：origin: com.facebook.presto/presto-hive

public static MessageType convert(final List<String> columnNames, final List<TypeInfo> columnTypes)
{
  return new MessageType("hive_schema", convertTypes(columnNames, columnTypes));
}

代码示例来源：origin: julienledem/redelm

private static MessageType parse(String schemaString) {
 Tokenizer st = new Tokenizer(schemaString, " ;{}()\n\t");
 String t = st.nextToken();
 check(t, "message", "start with 'message'", st);
 String name = st.nextToken();
 Type[] fields = readGroupTypeFields(st.nextToken(), st);
 return new MessageType(name, fields);
}

代码示例来源：origin: com.facebook.presto/presto-hive

@Test
public void testParquetTupleDomainPrimitiveArray()
{
  HiveColumnHandle columnHandle = new HiveColumnHandle("my_array", HiveType.valueOf("array<int>"), parseTypeSignature(StandardTypes.ARRAY), 0, REGULAR, Optional.empty());
  TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(INTEGER))));
  MessageType fileSchema = new MessageType("hive_schema",
      new GroupType(OPTIONAL, "my_array",
          new GroupType(REPEATED, "bag", new PrimitiveType(OPTIONAL, INT32, "array_element"))));
  Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema);
  TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain);
  assertTrue(tupleDomain.getDomains().get().isEmpty());
}

内容来源于网络，如有侵权，请联系作者删除！

MessageType

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com