org.apache.parquet.example.data.Group类的使用及代码示例

x33g5p2x  于2022-01-20 转载在 其他  
字(9.6k)|赞(0)|评价(0)|浏览(144)

本文整理了Java中org.apache.parquet.example.data.Group类的一些代码示例，展示了Group类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Group类的具体详情如下：
包路径：org.apache.parquet.example.data.Group
类名称：Group

Group介绍

暂无

代码示例

代码示例来源：origin: apache/incubator-druid

/**
 * convert a repeated field into a list of primitives or groups
 */
private static List<Object> convertRepeatedFieldToList(Group g, int fieldIndex, boolean binaryAsString)
{
 Type t = g.getType().getFields().get(fieldIndex);
 assert t.getRepetition().equals(Type.Repetition.REPEATED);
 int repeated = g.getFieldRepetitionCount(fieldIndex);
 List<Object> vals = new ArrayList<>();
 for (int i = 0; i < repeated; i++) {
  if (t.isPrimitive()) {
   vals.add(convertPrimitiveField(g, fieldIndex, i, binaryAsString));
  } else {
   vals.add(g.getGroup(fieldIndex, i));
  }
 }
 return vals;
}

代码示例来源：origin: apache/hive

protected static void writeRepeateMapData(
 ParquetWriter<Group> writer, int elementNum, boolean isNull) throws IOException {
 SimpleGroupFactory f = new SimpleGroupFactory(schema);
 int mapMaxSize = 4;
 for (int i = 0; i < elementNum; i++) {
  Group group = f.newGroup();
  if (!isNull) {
   for (int j = 0; j < mapMaxSize; j++) {
    group.addGroup("map_int32_for_repeat_test").append("key", j).append("value", j);
   }
  }
  writer.write(group);
 }
 writer.close();
}

代码示例来源：origin: apache/incubator-druid

PrimitiveType pt = (PrimitiveType) g.getType().getFields().get(fieldIndex);
OriginalType ot = pt.getOriginalType();
    long ts = g.getInteger(fieldIndex, index) * MILLIS_IN_DAY;
    return ts;
   case TIME_MICROS:
    return g.getLong(fieldIndex, index);
   case TIME_MILLIS:
    return g.getInteger(fieldIndex, index);
   case TIMESTAMP_MICROS:
    return TimeUnit.MILLISECONDS.convert(g.getLong(fieldIndex, index), TimeUnit.MICROSECONDS);
   case TIMESTAMP_MILLIS:
    return g.getLong(fieldIndex, index);
   case INTERVAL:
    Binary intervalVal = g.getBinary(fieldIndex, index);
    IntBuffer intBuf = intervalVal.toByteBuffer().order(ByteOrder.LITTLE_ENDIAN).asIntBuffer();
    int months = intBuf.get(0);
   case INT_16:
   case INT_32:
    return g.getInteger(fieldIndex, index);
   case INT_64:
    return g.getLong(fieldIndex, index);
    return g.getInteger(fieldIndex, index);
   case UINT_64:
    return g.getLong(fieldIndex, index);
   case DECIMAL:

代码示例来源：origin: apache/ignite

/**
 * Read coefficient matrix from parquet.
 *
 * @param g Coefficient group.
 * @return Vector of coefficients.
 */
private static Vector readLinRegCoefficients(SimpleGroup g) {
  Vector coefficients;
  Group coeffGroup = g.getGroup(1, 0).getGroup(3, 0);
  final int amountOfCoefficients = coeffGroup.getFieldRepetitionCount(0);
  coefficients = new DenseVector(amountOfCoefficients);
  for (int j = 0; j < amountOfCoefficients; j++) {
    double coefficient = coeffGroup.getGroup(0, j).getDouble(0, 0);
    coefficients.set(j, coefficient);
  }
  return coefficients;
}

代码示例来源：origin: apache/incubator-druid

/**
 * Convert a primitive group field to a "ingestion friendly" java object
 *
 * @return "ingestion ready" java object, or null
 */
@Nullable
private static Object convertPrimitiveField(Group g, int fieldIndex, boolean binaryAsString)
{
 PrimitiveType pt = (PrimitiveType) g.getType().getFields().get(fieldIndex);
 if (pt.isRepetition(Type.Repetition.REPEATED) && g.getFieldRepetitionCount(fieldIndex) > 1) {
  List<Object> vals = new ArrayList<>();
  for (int i = 0; i < g.getFieldRepetitionCount(fieldIndex); i++) {
   vals.add(convertPrimitiveField(g, fieldIndex, i, binaryAsString));
  }
  return vals;
 }
 return convertPrimitiveField(g, fieldIndex, 0, binaryAsString);
}

代码示例来源：origin: iflytek/Guitar

map = new HashMap<String, Object>();
Group group = (Group) record.data;
int size = group.getFieldRepetitionCount("map");
for (int i = 0; i < size; i++) {
  Group mapGroup = group.getGroup("map", i);
  if ("Integer".equals(dataType)) {
    String k = mapGroup.getString("key", 0);
    Integer v = mapGroup.getInteger("value", 0);
    map.put(k, v);
  } else if ("Long".equals(dataType)) {
    String k = mapGroup.getString("key", 0);
    Long v = mapGroup.getLong("value", 0);
    map.put(k, v);
  } else if ("Float".equals(dataType)) {
    String k = mapGroup.getString("key", 0);
    Float v = mapGroup.getFloat("value", 0);
    map.put(k, v);
  } else if ("Double".equals(dataType)) {
    String k = mapGroup.getString("key", 0);
    Double v = mapGroup.getDouble("value", 0);
    map.put(k, v);
  } else if ("Boolean".equals(dataType)) {
    String k = mapGroup.getString("key", 0);
    Boolean v = mapGroup.getBoolean("value", 0);
    map.put(k, v);
  } else if ("String".equals(dataType)) {
    String k = mapGroup.getString("key", 0);
    String v = mapGroup.getString("value", 0);
    map.put(k, v);

代码示例来源：origin: org.apache.parquet/parquet-column

public void add(String field, NanoTime value) {
 add(getType().getFieldIndex(field), value);
}

代码示例来源：origin: apache/ignite

/**
 * Prints the given group in the row of Parquet file.
 *
 * @param g The given group.
 */
private static void printGroup(Group g) {
  int fieldCnt = g.getType().getFieldCount();
  for (int field = 0; field < fieldCnt; field++) {
    int valCnt = g.getFieldRepetitionCount(field);
    Type fieldType = g.getType().getType(field);
    String fieldName = fieldType.getName();
    for (int idx = 0; idx < valCnt; idx++) {
      if (fieldType.isPrimitive())
        System.out.println(fieldName + " " + g.getValueToString(field, idx));
      else
        printGroup(g.getGroup(field, idx));
    }
  }
  System.out.println();
}

代码示例来源：origin: org.apache.parquet/parquet-column

private void writeGroup(Group group, GroupType type) {
  int fieldCount = type.getFieldCount();
  for (int field = 0; field < fieldCount; ++field) {
   int valueCount = group.getFieldRepetitionCount(field);
   if (valueCount > 0) {
    Type fieldType = type.getType(field);
    String fieldName = fieldType.getName();
    recordConsumer.startField(fieldName, field);
    for (int index = 0; index < valueCount; ++index) {
     if (fieldType.isPrimitive()) {
      group.writeValue(field, index, recordConsumer);
     } else {
      recordConsumer.startGroup();
      writeGroup(group.getGroup(field, index), fieldType.asGroupType());
      recordConsumer.endGroup();
     }
    }
    recordConsumer.endField(fieldName, field);
   }
  }
 }
}

代码示例来源：origin: org.apache.parquet/parquet-column

public Group append(String fieldName, float value) {
 add(fieldName, value);
 return this;
}

代码示例来源：origin: org.apache.parquet/parquet-column

public Group getGroup(String field, int index) {
 return getGroup(getType().getFieldIndex(field), index);
}

代码示例来源：origin: org.lasersonlab.apache.parquet/parquet-column

public Group addGroup(String field) {
 if (LOG.isDebugEnabled()) {
  LOG.debug("add group {} to {}", field, getType().getName());
 }
 return addGroup(getType().getFieldIndex(field));
}

代码示例来源：origin: apache/ignite

/**
 * Form the node data according data in parquet row.
 *
 * @param g The given group presenting the node data from Spark DT model.
 */
@NotNull private static SparkModelParser.NodeData extractNodeDataFromParquetRow(SimpleGroup g) {
  NodeData nodeData = new NodeData();
  nodeData.id = g.getInteger(0, 0);
  nodeData.prediction = g.getDouble(1, 0);
  nodeData.leftChildId = g.getInteger(5, 0);
  nodeData.rightChildId = g.getInteger(6, 0);
  if (nodeData.leftChildId == -1 && nodeData.rightChildId == -1) {
    nodeData.featureIdx = -1;
    nodeData.threshold = -1;
    nodeData.isLeafNode = true;
  }
  else {
    final SimpleGroup splitGrp = (SimpleGroup)g.getGroup(7, 0);
    nodeData.featureIdx = splitGrp.getInteger(0, 0);
    nodeData.threshold = splitGrp.getGroup(1, 0).getGroup(0, 0).getDouble(0, 0);
  }
  return nodeData;
}

代码示例来源：origin: apache/incubator-druid

@Override
public Set<String> discoverRootFields(Group obj)
{
 return obj.getType()
      .getFields()
      .stream()
      .filter(Type::isPrimitive)
      .map(Type::getName)
      .collect(Collectors.toSet());
}

代码示例来源：origin: iflytek/Guitar

Group time = line.getGroup("time", 0);
    line.getString(0, 0) + "\t" +
    line.getString(1, 0) + "\t"
    line.getString("city", 0) + "\t" +
    line.getString("ip", 0) + "\t"

代码示例来源：origin: iflytek/Guitar

@Override
public Object get(String key) throws Exception {
  if (data instanceof Group) {
    Group group = (Group) data;
    Group tmp = group.getGroup(key, 0);
    return new ParquetRecord(tmp);
  }
  return null;
}

代码示例来源：origin: apache/hive

protected static void writeListData(ParquetWriter<Group> writer, boolean isDictionaryEncoding,
 int elementNum) throws IOException {
 SimpleGroupFactory f = new SimpleGroupFactory(schema);
 int listMaxSize = 4;
 int listElementIndex = 0;
 for (int i = 0; i < elementNum; i++) {
  boolean isNull = isNull(i);
  Group group = f.newGroup();
  int listSize = i % listMaxSize + 1;
  if (!isNull) {
   for (int j = 0; j < listSize; j++) {
    group.append("list_int32_field", getIntValue(isDictionaryEncoding, listElementIndex));
    group.append("list_int64_field", getLongValue(isDictionaryEncoding, listElementIndex));
    group.append("list_double_field", getDoubleValue(isDictionaryEncoding, listElementIndex));
    group.append("list_float_field", getFloatValue(isDictionaryEncoding, listElementIndex));
    group.append("list_boolean_field", getBooleanValue(listElementIndex));
    group.append("list_binary_field", getBinaryValue(isDictionaryEncoding, listElementIndex));
    HiveDecimal hd = getDecimal(isDictionaryEncoding, listElementIndex).setScale(2);
    HiveDecimalWritable hdw = new HiveDecimalWritable(hd);
    group.append("list_decimal_field", Binary.fromConstantByteArray(hdw.getInternalStorage()));
    listElementIndex++;
   }
  }
  for (int j = 0; j < listMaxSize; j++) {
   group.append("list_binary_field_for_repeat_test", getBinaryValue(isDictionaryEncoding, i));
  }
  writer.write(group);
 }
 writer.close();
}

代码示例来源：origin: org.apache.parquet/parquet-column

@Override
public void start() {
 current = parent.getCurrentRecord().addGroup(index);
}

代码示例来源：origin: iflytek/Guitar

@Override
public Double getDouble(String key, Double dDef) throws Exception {
  if (data instanceof Group) {
    Group group = (Group) data;
    Double value = group.getDouble(key, 0);
    if (value != null && value != Double.MIN_VALUE) {
      return value;
    }
  }
  return dDef;
}

代码示例来源：origin: iflytek/Guitar

public Float getFloat(String key, float fDef) throws Exception {
  if (data instanceof Group) {
    Group group = (Group) data;
    Float value = group.getFloat(key, 0);
    if (value != null && value != Float.MIN_VALUE) {
      return value;
    }
  }
  return fDef;
}

内容来源于网络，如有侵权，请联系作者删除！

Group

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com