parquet.schema.Type.getRepetition()方法的使用及代码示例

x33g5p2x  于2022-01-30 转载在 其他  
字(7.9k)|赞(0)|评价(0)|浏览(110)

本文整理了Java中parquet.schema.Type.getRepetition()方法的一些代码示例,展示了Type.getRepetition()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Type.getRepetition()方法的具体详情如下:
包路径:parquet.schema.Type
类名称:Type
方法名:getRepetition

Type.getRepetition介绍

暂无

代码示例

代码示例来源:origin: prestodb/presto

return Optional.empty();
boolean required = columnIO.getType().getRepetition() != OPTIONAL;
int repetitionLevel = columnIO.getRepetitionLevel();
int definitionLevel = columnIO.getDefinitionLevel();

代码示例来源:origin: julienledem/redelm

private void add(int fieldIndex, Primitive value) {
 Type type = schema.getType(fieldIndex);
 List<Object> list = data[fieldIndex];
 if (type.getRepetition() != Type.Repetition.REPEATED
   && !list.isEmpty()) {
  throw new IllegalStateException("field "+fieldIndex+" (" + type.getName() + ") can not have more than one value: " + list);
 }
 list.add(value);
}

代码示例来源:origin: julienledem/redelm

private void validateMissingFields(int index) {
 for (int i = previousField.peek() + 1; i < index; i++) {
  Type type = types.peek().asGroupType().getType(i);
  if (type.getRepetition() == Repetition.REQUIRED) {
   throw new InvalidRecordException("required field is missing " + type);
  }
 }
}

代码示例来源:origin: julienledem/redelm

private void validate(PrimitiveTypeName p) {
 Type currentType = types.peek().asGroupType().getType(fields.peek());
 int c = fieldValueCount.pop() + 1;
 fieldValueCount.push(c);
 if (DEBUG) LOG.debug("validate " + p + " for " + currentType.getName());
 switch (currentType.getRepetition()) {
  case OPTIONAL:
  case REQUIRED:
   if (c > 1) {
    throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType);
   }
   break;
  case REPEATED:
   break;
  default:
   throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType);
 }
 if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) {
  throw new InvalidRecordException("expected type " + currentType + " but got "+ p);
 }
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

private void validate(PrimitiveTypeName p) {
 Type currentType = types.peek().asGroupType().getType(fields.peek());
 int c = fieldValueCount.pop() + 1;
 fieldValueCount.push(c);
 if (DEBUG) LOG.debug("validate " + p + " for " + currentType.getName());
 switch (currentType.getRepetition()) {
  case OPTIONAL:
  case REQUIRED:
   if (c > 1) {
    throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType);
   }
   break;
  case REPEATED:
   break;
  default:
   throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType);
 }
 if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) {
  throw new InvalidRecordException("expected type " + p + " but got "+ currentType);
 }
}

代码示例来源:origin: julienledem/redelm

@Override
void setLevels(int r, int d, String[] fieldPath, int[] indexFieldPath, List<ColumnIO> repetition, List<ColumnIO> path) {
 super.setLevels(r, d, fieldPath, indexFieldPath, repetition, path);
 for (ColumnIO child : this.children) {
  String[] newFieldPath = Arrays.copyOf(fieldPath, fieldPath.length + 1);
  int[] newIndexFieldPath = Arrays.copyOf(indexFieldPath, indexFieldPath.length + 1);
  newFieldPath[fieldPath.length] =  child.getType().getName();
  newIndexFieldPath[indexFieldPath.length] =  this.getType().asGroupType().getFieldIndex(child.getType().getName());
  List<ColumnIO> newRepetition;
  if (child.getType().getRepetition() == REPEATED) {
   newRepetition = new ArrayList<ColumnIO>(repetition);
   newRepetition.add(child);
  } else {
   newRepetition = repetition;
  }
  List<ColumnIO> newPath = new ArrayList<ColumnIO>(path);
  newPath.add(child);
  child.setLevels(
    // the type repetition level increases whenever there's a possible repetition
    child.getType().getRepetition() == REPEATED ? r + 1 : r,
    // the type definition level increases whenever a field can be missing (not required)
    child.getType().getRepetition() != REQUIRED ? d + 1 : d,
    newFieldPath,
    newIndexFieldPath,
    newRepetition,
    newPath
    );
 }
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

private void validate(PrimitiveTypeName... ptypes) {
 Type currentType = types.peek().asGroupType().getType(fields.peek());
 int c = fieldValueCount.pop() + 1;
 fieldValueCount.push(c);
 if (DEBUG) LOG.debug("validate " + Arrays.toString(ptypes) + " for " + currentType.getName());
 switch (currentType.getRepetition()) {
  case OPTIONAL:
  case REQUIRED:
   if (c > 1) {
    throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType);
   }
   break;
  case REPEATED:
   break;
  default:
   throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType);
 }
 if (!currentType.isPrimitive()) {
  throw new InvalidRecordException(
    "expected type in " + Arrays.toString(ptypes) + " but got " + currentType);
 }
 for (PrimitiveTypeName p : ptypes) {
  if (currentType.asPrimitiveType().getPrimitiveTypeName() == p) {
   return; // type is valid
  }
 }
 throw new InvalidRecordException(
   "expected type in " + Arrays.toString(ptypes) + " but got " + currentType);
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

private void visitChildren(GroupColumnIO newIO, GroupType groupType, GroupType requestedGroupType) {
 GroupColumnIO oldIO = current;
 current = newIO;
 for (Type type : groupType.getFields()) {
  // if the file schema does not contain the field it will just stay null
  if (requestedGroupType.containsField(type.getName())) {
   currentRequestedIndex = requestedGroupType.getFieldIndex(type.getName());
   currentRequestedType = requestedGroupType.getType(currentRequestedIndex);
   if (currentRequestedType.getRepetition().isMoreRestrictiveThan(type.getRepetition())) {
    incompatibleSchema(type, currentRequestedType);
   }
   type.accept(this);
  }
 }
 current = oldIO;
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

if (toMerge.containsField(type.getName())) {
 Type fieldToMerge = toMerge.getType(type.getName());
 if (fieldToMerge.getRepetition().isMoreRestrictiveThan(type.getRepetition())) {
  throw new IncompatibleSchemaModificationException("repetition constraint is more restrictive: can not merge type " + fieldToMerge + " into " + type);

代码示例来源:origin: com.facebook.presto.hive/hive-apache

@Override
protected Type union(Type toMerge, boolean strict) {
 if (toMerge.isPrimitive()) {
  throw new IncompatibleSchemaModificationException("can not merge primitive type " + toMerge + " into group type " + this);
 }
 return new GroupType(toMerge.getRepetition(), getName(), mergeFields(toMerge.asGroupType()));
}

代码示例来源:origin: julienledem/redelm

ColumnIO getParent(int r) {
 if (getRepetitionLevel() == r && getType().getRepetition() == Repetition.REPEATED) {
  return this;
 } else  if (getParent()!=null && getParent().getDefinitionLevel()>=r) {
  return getParent().getParent(r);
 } else {
  throw new InvalidRecordException("no parent("+r+") for "+Arrays.toString(this.getFieldPath()));
 }
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

@Override
 protected Type union(Type toMerge, boolean strict) {
  if (!toMerge.isPrimitive() || (strict && !primitive.equals(toMerge.asPrimitiveType().getPrimitiveTypeName()))) {
   throw new IncompatibleSchemaModificationException("can not merge type " + toMerge + " into " + this);
  }
  Types.PrimitiveBuilder<PrimitiveType> builder = Types.primitive(
    primitive, toMerge.getRepetition());
  if (PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY == primitive) {
   builder.length(length);
  }
  return builder.named(getName());
 }
}

代码示例来源:origin: com.twitter/parquet-thrift

private boolean hasMissingRequiredFieldInGroupType(GroupType requested, GroupType fullSchema) {
 for (Type field : fullSchema.getFields()) {
  if (requested.containsField(field.getName())) {
   Type requestedType = requested.getType(field.getName());
   // if a field is in requested schema and the type of it is a group type, then do recursive check
   if (!field.isPrimitive()) {
    if (hasMissingRequiredFieldInGroupType(requestedType.asGroupType(), field.asGroupType())) {
     return true;
    } else {
     continue;// check next field
    }
   }
  } else {
   if (field.getRepetition() == Type.Repetition.REQUIRED) {
    return true; // if a field is missing in requested schema and it's required
   } else {
    continue; // the missing field is not required, then continue checking next field
   }
  }
 }
 return false;
}

相关文章