本文整理了Java中parquet.schema.Type.getRepetition()
方法的一些代码示例,展示了Type.getRepetition()
的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Type.getRepetition()
方法的具体详情如下:
包路径:parquet.schema.Type
类名称:Type
方法名:getRepetition
暂无
代码示例来源:origin: prestodb/presto
return Optional.empty();
boolean required = columnIO.getType().getRepetition() != OPTIONAL;
int repetitionLevel = columnIO.getRepetitionLevel();
int definitionLevel = columnIO.getDefinitionLevel();
代码示例来源:origin: julienledem/redelm
private void add(int fieldIndex, Primitive value) {
Type type = schema.getType(fieldIndex);
List<Object> list = data[fieldIndex];
if (type.getRepetition() != Type.Repetition.REPEATED
&& !list.isEmpty()) {
throw new IllegalStateException("field "+fieldIndex+" (" + type.getName() + ") can not have more than one value: " + list);
}
list.add(value);
}
代码示例来源:origin: julienledem/redelm
private void validateMissingFields(int index) {
for (int i = previousField.peek() + 1; i < index; i++) {
Type type = types.peek().asGroupType().getType(i);
if (type.getRepetition() == Repetition.REQUIRED) {
throw new InvalidRecordException("required field is missing " + type);
}
}
}
代码示例来源:origin: julienledem/redelm
private void validate(PrimitiveTypeName p) {
Type currentType = types.peek().asGroupType().getType(fields.peek());
int c = fieldValueCount.pop() + 1;
fieldValueCount.push(c);
if (DEBUG) LOG.debug("validate " + p + " for " + currentType.getName());
switch (currentType.getRepetition()) {
case OPTIONAL:
case REQUIRED:
if (c > 1) {
throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType);
}
break;
case REPEATED:
break;
default:
throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType);
}
if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) {
throw new InvalidRecordException("expected type " + currentType + " but got "+ p);
}
}
代码示例来源:origin: com.facebook.presto.hive/hive-apache
private void validate(PrimitiveTypeName p) {
Type currentType = types.peek().asGroupType().getType(fields.peek());
int c = fieldValueCount.pop() + 1;
fieldValueCount.push(c);
if (DEBUG) LOG.debug("validate " + p + " for " + currentType.getName());
switch (currentType.getRepetition()) {
case OPTIONAL:
case REQUIRED:
if (c > 1) {
throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType);
}
break;
case REPEATED:
break;
default:
throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType);
}
if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) {
throw new InvalidRecordException("expected type " + p + " but got "+ currentType);
}
}
代码示例来源:origin: julienledem/redelm
@Override
void setLevels(int r, int d, String[] fieldPath, int[] indexFieldPath, List<ColumnIO> repetition, List<ColumnIO> path) {
super.setLevels(r, d, fieldPath, indexFieldPath, repetition, path);
for (ColumnIO child : this.children) {
String[] newFieldPath = Arrays.copyOf(fieldPath, fieldPath.length + 1);
int[] newIndexFieldPath = Arrays.copyOf(indexFieldPath, indexFieldPath.length + 1);
newFieldPath[fieldPath.length] = child.getType().getName();
newIndexFieldPath[indexFieldPath.length] = this.getType().asGroupType().getFieldIndex(child.getType().getName());
List<ColumnIO> newRepetition;
if (child.getType().getRepetition() == REPEATED) {
newRepetition = new ArrayList<ColumnIO>(repetition);
newRepetition.add(child);
} else {
newRepetition = repetition;
}
List<ColumnIO> newPath = new ArrayList<ColumnIO>(path);
newPath.add(child);
child.setLevels(
// the type repetition level increases whenever there's a possible repetition
child.getType().getRepetition() == REPEATED ? r + 1 : r,
// the type definition level increases whenever a field can be missing (not required)
child.getType().getRepetition() != REQUIRED ? d + 1 : d,
newFieldPath,
newIndexFieldPath,
newRepetition,
newPath
);
}
}
代码示例来源:origin: com.facebook.presto.hive/hive-apache
private void validate(PrimitiveTypeName... ptypes) {
Type currentType = types.peek().asGroupType().getType(fields.peek());
int c = fieldValueCount.pop() + 1;
fieldValueCount.push(c);
if (DEBUG) LOG.debug("validate " + Arrays.toString(ptypes) + " for " + currentType.getName());
switch (currentType.getRepetition()) {
case OPTIONAL:
case REQUIRED:
if (c > 1) {
throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType);
}
break;
case REPEATED:
break;
default:
throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType);
}
if (!currentType.isPrimitive()) {
throw new InvalidRecordException(
"expected type in " + Arrays.toString(ptypes) + " but got " + currentType);
}
for (PrimitiveTypeName p : ptypes) {
if (currentType.asPrimitiveType().getPrimitiveTypeName() == p) {
return; // type is valid
}
}
throw new InvalidRecordException(
"expected type in " + Arrays.toString(ptypes) + " but got " + currentType);
}
代码示例来源:origin: com.facebook.presto.hive/hive-apache
private void visitChildren(GroupColumnIO newIO, GroupType groupType, GroupType requestedGroupType) {
GroupColumnIO oldIO = current;
current = newIO;
for (Type type : groupType.getFields()) {
// if the file schema does not contain the field it will just stay null
if (requestedGroupType.containsField(type.getName())) {
currentRequestedIndex = requestedGroupType.getFieldIndex(type.getName());
currentRequestedType = requestedGroupType.getType(currentRequestedIndex);
if (currentRequestedType.getRepetition().isMoreRestrictiveThan(type.getRepetition())) {
incompatibleSchema(type, currentRequestedType);
}
type.accept(this);
}
}
current = oldIO;
}
代码示例来源:origin: com.facebook.presto.hive/hive-apache
if (toMerge.containsField(type.getName())) {
Type fieldToMerge = toMerge.getType(type.getName());
if (fieldToMerge.getRepetition().isMoreRestrictiveThan(type.getRepetition())) {
throw new IncompatibleSchemaModificationException("repetition constraint is more restrictive: can not merge type " + fieldToMerge + " into " + type);
代码示例来源:origin: com.facebook.presto.hive/hive-apache
@Override
protected Type union(Type toMerge, boolean strict) {
if (toMerge.isPrimitive()) {
throw new IncompatibleSchemaModificationException("can not merge primitive type " + toMerge + " into group type " + this);
}
return new GroupType(toMerge.getRepetition(), getName(), mergeFields(toMerge.asGroupType()));
}
代码示例来源:origin: julienledem/redelm
ColumnIO getParent(int r) {
if (getRepetitionLevel() == r && getType().getRepetition() == Repetition.REPEATED) {
return this;
} else if (getParent()!=null && getParent().getDefinitionLevel()>=r) {
return getParent().getParent(r);
} else {
throw new InvalidRecordException("no parent("+r+") for "+Arrays.toString(this.getFieldPath()));
}
}
代码示例来源:origin: com.facebook.presto.hive/hive-apache
@Override
protected Type union(Type toMerge, boolean strict) {
if (!toMerge.isPrimitive() || (strict && !primitive.equals(toMerge.asPrimitiveType().getPrimitiveTypeName()))) {
throw new IncompatibleSchemaModificationException("can not merge type " + toMerge + " into " + this);
}
Types.PrimitiveBuilder<PrimitiveType> builder = Types.primitive(
primitive, toMerge.getRepetition());
if (PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY == primitive) {
builder.length(length);
}
return builder.named(getName());
}
}
代码示例来源:origin: com.twitter/parquet-thrift
private boolean hasMissingRequiredFieldInGroupType(GroupType requested, GroupType fullSchema) {
for (Type field : fullSchema.getFields()) {
if (requested.containsField(field.getName())) {
Type requestedType = requested.getType(field.getName());
// if a field is in requested schema and the type of it is a group type, then do recursive check
if (!field.isPrimitive()) {
if (hasMissingRequiredFieldInGroupType(requestedType.asGroupType(), field.asGroupType())) {
return true;
} else {
continue;// check next field
}
}
} else {
if (field.getRepetition() == Type.Repetition.REQUIRED) {
return true; // if a field is missing in requested schema and it's required
} else {
continue; // the missing field is not required, then continue checking next field
}
}
}
return false;
}
内容来源于网络,如有侵权,请联系作者删除!