parquet.schema.Type.getOriginalType()方法的使用及代码示例

x33g5p2x  于2022-01-30 转载在 其他  
字(8.2k)|赞(0)|评价(0)|浏览(133)

本文整理了Java中parquet.schema.Type.getOriginalType()方法的一些代码示例,展示了Type.getOriginalType()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Type.getOriginalType()方法的具体详情如下:
包路径:parquet.schema.Type
类名称:Type
方法名:getOriginalType

Type.getOriginalType介绍

暂无

代码示例

代码示例来源:origin: prestodb/presto

public static ColumnIO getArrayElementColumn(ColumnIO columnIO)
{
  while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) {
    columnIO = ((GroupColumnIO) columnIO).getChild(0);
  }
  /* If array has a standard 3-level structure with middle level repeated group with a single field:
   *  optional group my_list (LIST) {
   *     repeated group element {
   *        required binary str (UTF8);
   *     };
   *  }
   */
  if (columnIO instanceof GroupColumnIO &&
      columnIO.getType().getOriginalType() == null &&
      ((GroupColumnIO) columnIO).getChildrenCount() == 1 &&
      !columnIO.getName().equals("array") &&
      !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) {
    return ((GroupColumnIO) columnIO).getChild(0);
  }
  /* Backward-compatibility support for 2-level arrays where a repeated field is not a group:
   *   optional group my_list (LIST) {
   *      repeated int32 element;
   *   }
   */
  return columnIO;
}

代码示例来源:origin: prestodb/presto

OriginalType originalType = type.getOriginalType();

代码示例来源:origin: asakusafw/asakusafw

private String getSourceTypeDescription(Mapping mapping) {
  OriginalType originalType = mapping.sourceType.getOriginalType();
  if (originalType != null) {
    return String.valueOf(originalType);
  }
  return String.valueOf(mapping.source.getType());
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

private SchemaCompatibilityValidator(MessageType schema) {
 for (ColumnDescriptor cd : schema.getColumns()) {
  ColumnPath columnPath = ColumnPath.get(cd.getPath());
  columnsAccordingToSchema.put(columnPath, cd);
  OriginalType ot = schema.getType(cd.getPath()).getOriginalType();
  if (ot != null) {
   originalTypes.put(columnPath, ot);
  }
 }
}

代码示例来源:origin: com.twitter/parquet-pig

private Type filterBag(GroupType bagType, FieldSchema bagFieldSchema) throws FrontendException {
  if (DEBUG) LOG.debug("filtering BAG schema:\n" + bagType + "\nwith:\n " + bagFieldSchema);
  if (bagType.getFieldCount() != 1) {
   throw new RuntimeException("not unwrapping the right type, this should be a Bag: " + bagType);
  }
  Type nested = bagType.getType(0);
  FieldSchema innerField = bagFieldSchema.schema.getField(0);
  if (nested.isPrimitive() || nested.getOriginalType() == OriginalType.MAP || nested.getOriginalType() == OriginalType.LIST) {
   // Bags always contain tuples => we skip the extra tuple that was inserted in that case.
   innerField = innerField.schema.getField(0);
  }
  return bagType.withNewFields(filter(nested, innerField));
 }
}

代码示例来源:origin: uk.co.nichesolutions.presto/presto-hive

private static GroupedConverter createGroupConverter(Type prestoType, String columnName, parquet.schema.Type parquetType, int fieldIndex)
{
  GroupType groupType = parquetType.asGroupType();
  switch (prestoType.getTypeSignature().getBase()) {
    case ARRAY:
      return new ParquetListConverter(prestoType, columnName, groupType, fieldIndex);
    case MAP:
      return new ParquetMapConverter(prestoType, columnName, groupType, fieldIndex);
    case ROW:
      return new ParquetStructConverter(prestoType, columnName, groupType, fieldIndex);
    default:
      throw new IllegalArgumentException("Column " + columnName + " type " + parquetType.getOriginalType() + " not supported");
  }
}

代码示例来源:origin: asakusafw/asakusafw

private boolean isCompatible(Type sourceType, PropertyDescriptor target) {
  ParquetValueDriver driver = ParquetValueDrivers.of(
      target.getTypeInfo(),
      target.getValueClass());
  Type targetType = driver.getType(target.getFieldName());
  if (sourceType.getOriginalType() != null) {
    if (sourceType.getOriginalType() == targetType.getOriginalType()) {
      return true;
    }
  }
  if (sourceType.isPrimitive()) {
    if (targetType.isPrimitive() == false) {
      return false;
    }
    return sourceType.asPrimitiveType().getPrimitiveTypeName()
        == targetType.asPrimitiveType().getPrimitiveTypeName();
  }
  return false;
}

代码示例来源:origin: org.apache.tajo/tajo-storage

final PrimitiveTypeName parquetPrimitiveTypeName =
  fieldType.asPrimitiveType().getPrimitiveTypeName();
final OriginalType originalType = fieldType.getOriginalType();
return parquetPrimitiveTypeName.convert(
  new PrimitiveType.PrimitiveTypeNameConverter<Column, RuntimeException>() {

代码示例来源:origin: com.twitter/parquet-pig

BagConverter(GroupType parquetSchema, FieldSchema pigSchema, ParentValueContainer parent, boolean numbersDefaultToZero, boolean columnIndexAccess) throws FrontendException {
 this.parent = parent;
 if (parquetSchema.getFieldCount() != 1) {
  throw new IllegalArgumentException("bags have only one field. " + parquetSchema + " size = " + parquetSchema.getFieldCount());
 }
 Type nestedType = parquetSchema.getType(0);
 ParentValueContainer childsParent;
 FieldSchema pigField;
 if (nestedType.isPrimitive() || nestedType.getOriginalType() == OriginalType.MAP || nestedType.getOriginalType() == OriginalType.LIST) {
  // Pig bags always contain tuples
  // In that case we need to wrap the value in an extra tuple
  childsParent = new ParentValueContainer() {
   @Override
   void add(Object value) {
    buffer.add(TF.newTuple(value));
   }};
  pigField = pigSchema.schema.getField(0).schema.getField(0);
 } else {
  childsParent = new ParentValueContainer() {
   @Override
   void add(Object value) {
    buffer.add((Tuple)value);
   }};
  pigField = pigSchema.schema.getField(0);
 }
 child = newConverter(pigField, nestedType, childsParent, numbersDefaultToZero, columnIndexAccess);
}

代码示例来源:origin: com.twitter/parquet-pig

final PrimitiveTypeName parquetPrimitiveTypeName =
  parquetType.asPrimitiveType().getPrimitiveTypeName();
final OriginalType originalType = parquetType.getOriginalType();
return parquetPrimitiveTypeName.convert(
  new PrimitiveTypeNameConverter<Schema.FieldSchema, FrontendException>() {

代码示例来源:origin: uber/hudi

final PrimitiveType.PrimitiveTypeName parquetPrimitiveTypeName = parquetType.asPrimitiveType()
  .getPrimitiveTypeName();
final OriginalType originalType = parquetType.getOriginalType();
if (originalType == OriginalType.DECIMAL) {
 final DecimalMetadata decimalMetadata = parquetType.asPrimitiveType().getDecimalMetadata();
   if (!keyType.isPrimitive() || !keyType.asPrimitiveType().getPrimitiveTypeName()
     .equals(PrimitiveType.PrimitiveTypeName.BINARY)
     || !keyType.getOriginalType().equals(OriginalType.UTF8)) {
    throw new UnsupportedOperationException(
      "Map key type must be binary (UTF8): " + keyType);

代码示例来源:origin: com.facebook.presto.hive/hive-apache

/**
 * It writes the field value to the Parquet RecordConsumer. It detects the field type, and calls
 * the correct write function.
 * @param value The writable object that contains the value.
 * @param inspector The object inspector used to get the correct value type.
 * @param type Type that contains information about the type schema.
 */
private void writeValue(final Object value, final ObjectInspector inspector, final Type type) {
 if (type.isPrimitive()) {
  checkInspectorCategory(inspector, ObjectInspector.Category.PRIMITIVE);
  writePrimitive(value, (PrimitiveObjectInspector)inspector);
 } else {
  GroupType groupType = type.asGroupType();
  OriginalType originalType = type.getOriginalType();
  if (originalType != null && originalType.equals(OriginalType.LIST)) {
   checkInspectorCategory(inspector, ObjectInspector.Category.LIST);
   writeArray(value, (ListObjectInspector)inspector, groupType);
  } else if (originalType != null && originalType.equals(OriginalType.MAP)) {
   checkInspectorCategory(inspector, ObjectInspector.Category.MAP);
   writeMap(value, (MapObjectInspector)inspector, groupType);
  } else {
   checkInspectorCategory(inspector, ObjectInspector.Category.STRUCT);
   writeGroup(value, (StructObjectInspector)inspector, groupType);
  }
 }
}

代码示例来源:origin: com.twitter/parquet-tools

private Converter createConverter(Type field) {
 OriginalType otype = field.getOriginalType();
 if (field.isPrimitive()) {
  if (otype != null) {
   switch (otype) {
    case MAP: break;
    case LIST: break;
    case UTF8: return new StringConverter(field.getName());
    case MAP_KEY_VALUE: break;
    case ENUM: break;
   }
  }
  return new SimplePrimitiveConverter(field.getName());
 }
 GroupType groupType = field.asGroupType();
 if (otype != null) {
  switch (otype) {
   case MAP: return new SimpleMapRecordConverter(groupType, field.getName(), this);
   case LIST: return new SimpleListRecordConverter(groupType, field.getName(), this);
  }
 }
 return new SimpleRecordConverter(groupType, field.getName(), this);
}

代码示例来源:origin: com.twitter/parquet-pig

return new FieldStringConverter(parent, type.getOriginalType() == OriginalType.UTF8);
case DataType.BYTEARRAY:
 return new FieldByteArrayConverter(parent);

代码示例来源:origin: com.facebook.presto/presto-hive

OriginalType originalType = type.getOriginalType();

相关文章