org.apache.hadoop.hive.ql.io.orc.Reader.getMetadataValue()方法的使用及代码示例

x33g5p2x  于2022-01-29 转载在 其他  
字(6.0k)|赞(0)|评价(0)|浏览(148)

本文整理了Java中org.apache.hadoop.hive.ql.io.orc.Reader.getMetadataValue方法的一些代码示例,展示了Reader.getMetadataValue的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Reader.getMetadataValue方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.io.orc.Reader
类名称:Reader
方法名:getMetadataValue

Reader.getMetadataValue介绍

[英]Get a user metadata value.
[中]获取用户元数据值。

代码示例

代码示例来源:origin: apache/hive

static String getKeyIndexAsString(Reader reader) {
 try {
  ByteBuffer val =
    reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)
      .duplicate();
  return utf8Decoder.decode(val).toString();
 } catch (CharacterCodingException e) {
  throw new IllegalArgumentException("Bad string encoding for " +
    OrcRecordUpdater.ACID_KEY_INDEX_NAME, e);
 }
}

代码示例来源:origin: apache/hive

private static boolean needsCompaction(FileStatus bucket, FileSystem fs) throws IOException {
 //create reader, look at footer
 //no need to check side file since it can only be in a streaming ingest delta
 Reader orcReader = OrcFile.createReader(bucket.getPath(), OrcFile.readerOptions(fs.getConf()).filesystem(fs));
 if (orcReader.hasMetadataValue(ACID_STATS)) {
  try {
   ByteBuffer val = orcReader.getMetadataValue(ACID_STATS).duplicate();
   String acidStats = utf8Decoder.decode(val).toString();
   String[] parts = acidStats.split(",");
   long updates = Long.parseLong(parts[1]);
   long deletes = Long.parseLong(parts[2]);
   return deletes > 0 || updates > 0;
  } catch (CharacterCodingException e) {
   throw new IllegalArgumentException("Bad string encoding for " + ACID_STATS, e);
  }
 } else {
  throw new IllegalStateException("AcidStats missing in " + bucket.getPath());
 }
}

代码示例来源:origin: apache/hive

static RecordIdentifier[] parseKeyIndex(Reader reader) {
 String[] stripes;
 try {
  if (!reader.hasMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)) {
   return null;
  }
  ByteBuffer val =
    reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)
      .duplicate();
  stripes = utf8Decoder.decode(val).toString().split(";");
 } catch (CharacterCodingException e) {
  throw new IllegalArgumentException("Bad string encoding for " +
    OrcRecordUpdater.ACID_KEY_INDEX_NAME, e);
 }
 RecordIdentifier[] result = new RecordIdentifier[stripes.length];
 for(int i=0; i < stripes.length; ++i) {
  if (stripes[i].length() != 0) {
   String[] parts = stripes[i].split(",");
   result[i] = new RecordIdentifier();
   result[i].setValues(Long.parseLong(parts[0]),
     Integer.parseInt(parts[1]), Long.parseLong(parts[2]));
  }
 }
 return result;
}

代码示例来源:origin: apache/drill

static RecordIdentifier[] parseKeyIndex(Reader reader) {
 String[] stripes;
 try {
  ByteBuffer val =
    reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)
      .duplicate();
  stripes = utf8Decoder.decode(val).toString().split(";");
 } catch (CharacterCodingException e) {
  throw new IllegalArgumentException("Bad string encoding for " +
    OrcRecordUpdater.ACID_KEY_INDEX_NAME, e);
 }
 RecordIdentifier[] result = new RecordIdentifier[stripes.length];
 for(int i=0; i < stripes.length; ++i) {
  if (stripes[i].length() != 0) {
   String[] parts = stripes[i].split(",");
   result[i] = new RecordIdentifier();
   result[i].setValues(Long.parseLong(parts[0]),
     Integer.parseInt(parts[1]), Long.parseLong(parts[2]));
  }
 }
 return result;
}

代码示例来源:origin: apache/hive

/**
 * This is smart enough to handle streaming ingest where there could be a
 * {@link OrcAcidUtils#DELTA_SIDE_FILE_SUFFIX} side file.
 * @param dataFile - ORC acid data file
 * @return version property from file if there,
 *          {@link #ORC_ACID_VERSION_DEFAULT} otherwise
 */
@VisibleForTesting
public static int getAcidVersionFromDataFile(Path dataFile, FileSystem fs) throws IOException {
 FileStatus fileStatus = fs.getFileStatus(dataFile);
 Reader orcReader = OrcFile.createReader(dataFile,
   OrcFile.readerOptions(fs.getConf())
     .filesystem(fs)
     //make sure to check for side file in case streaming ingest died
     .maxLength(getLogicalLength(fs, fileStatus)));
 if (orcReader.hasMetadataValue(ACID_VERSION_KEY)) {
  char[] versionChar = UTF8.decode(orcReader.getMetadataValue(ACID_VERSION_KEY)).array();
  String version = new String(versionChar);
  return Integer.valueOf(version);
 }
 return ORC_ACID_VERSION_DEFAULT;
}
/**

代码示例来源:origin: apache/hive

writer.addUserMetadata(metadataKey, reader.getMetadataValue(metadataKey));

代码示例来源:origin: apache/hive

Reader reader = OrcFile.createReader(testFilePath,
  OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(byteBuf(5,7,11,13,17,19), reader.getMetadataValue("clobber"));
assertEquals(byteBuf(1,2,3,4,5,6,7,-1,-2,127,-128),
  reader.getMetadataValue("my.meta"));
assertEquals(bigBuf, reader.getMetadataValue("big"));
try {
 reader.getMetadataValue("unknown");
 assertTrue(false);
} catch (IllegalArgumentException iae) {

代码示例来源:origin: apache/hive

Mockito.when(reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME))
  .thenReturn(ByteBuffer.wrap("10,20,30;40,50,60;40,50,61"
    .getBytes("UTF-8")));

代码示例来源:origin: com.facebook.presto.hive/hive-apache

static RecordIdentifier[] parseKeyIndex(Reader reader) {
 String[] stripes;
 try {
  ByteBuffer val =
    reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)
      .duplicate();
  stripes = utf8Decoder.decode(val).toString().split(";");
 } catch (CharacterCodingException e) {
  throw new IllegalArgumentException("Bad string encoding for " +
    OrcRecordUpdater.ACID_KEY_INDEX_NAME, e);
 }
 RecordIdentifier[] result = new RecordIdentifier[stripes.length];
 for(int i=0; i < stripes.length; ++i) {
  if (stripes[i].length() != 0) {
   String[] parts = stripes[i].split(",");
   result[i] = new RecordIdentifier();
   result[i].setValues(Long.parseLong(parts[0]),
     Integer.parseInt(parts[1]), Long.parseLong(parts[2]));
  }
 }
 return result;
}

代码示例来源:origin: com.facebook.presto/presto-raptor

Closer<Writer, IOException> writer = closer(createWriter(path(output), writerOptions), Writer::close)) {
if (reader.hasMetadataValue(OrcFileMetadata.KEY)) {
  ByteBuffer orcFileMetadata = reader.getMetadataValue(OrcFileMetadata.KEY);
  writer.get().addUserMetadata(OrcFileMetadata.KEY, orcFileMetadata);

代码示例来源:origin: prestosql/presto

Closer<Writer, IOException> writer = closer(createWriter(path(output), writerOptions), Writer::close)) {
if (reader.hasMetadataValue(OrcFileMetadata.KEY)) {
  ByteBuffer orcFileMetadata = reader.getMetadataValue(OrcFileMetadata.KEY);
  writer.get().addUserMetadata(OrcFileMetadata.KEY, orcFileMetadata);

相关文章