本文整理了Java中org.apache.hadoop.hive.ql.io.orc.Reader.getMetadataValue
方法的一些代码示例,展示了Reader.getMetadataValue
的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Reader.getMetadataValue
方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.io.orc.Reader
类名称:Reader
方法名:getMetadataValue
[英]Get a user metadata value.
[中]获取用户元数据值。
代码示例来源:origin: apache/hive
static String getKeyIndexAsString(Reader reader) {
try {
ByteBuffer val =
reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)
.duplicate();
return utf8Decoder.decode(val).toString();
} catch (CharacterCodingException e) {
throw new IllegalArgumentException("Bad string encoding for " +
OrcRecordUpdater.ACID_KEY_INDEX_NAME, e);
}
}
代码示例来源:origin: apache/hive
private static boolean needsCompaction(FileStatus bucket, FileSystem fs) throws IOException {
//create reader, look at footer
//no need to check side file since it can only be in a streaming ingest delta
Reader orcReader = OrcFile.createReader(bucket.getPath(), OrcFile.readerOptions(fs.getConf()).filesystem(fs));
if (orcReader.hasMetadataValue(ACID_STATS)) {
try {
ByteBuffer val = orcReader.getMetadataValue(ACID_STATS).duplicate();
String acidStats = utf8Decoder.decode(val).toString();
String[] parts = acidStats.split(",");
long updates = Long.parseLong(parts[1]);
long deletes = Long.parseLong(parts[2]);
return deletes > 0 || updates > 0;
} catch (CharacterCodingException e) {
throw new IllegalArgumentException("Bad string encoding for " + ACID_STATS, e);
}
} else {
throw new IllegalStateException("AcidStats missing in " + bucket.getPath());
}
}
代码示例来源:origin: apache/hive
static RecordIdentifier[] parseKeyIndex(Reader reader) {
String[] stripes;
try {
if (!reader.hasMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)) {
return null;
}
ByteBuffer val =
reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)
.duplicate();
stripes = utf8Decoder.decode(val).toString().split(";");
} catch (CharacterCodingException e) {
throw new IllegalArgumentException("Bad string encoding for " +
OrcRecordUpdater.ACID_KEY_INDEX_NAME, e);
}
RecordIdentifier[] result = new RecordIdentifier[stripes.length];
for(int i=0; i < stripes.length; ++i) {
if (stripes[i].length() != 0) {
String[] parts = stripes[i].split(",");
result[i] = new RecordIdentifier();
result[i].setValues(Long.parseLong(parts[0]),
Integer.parseInt(parts[1]), Long.parseLong(parts[2]));
}
}
return result;
}
代码示例来源:origin: apache/drill
static RecordIdentifier[] parseKeyIndex(Reader reader) {
String[] stripes;
try {
ByteBuffer val =
reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)
.duplicate();
stripes = utf8Decoder.decode(val).toString().split(";");
} catch (CharacterCodingException e) {
throw new IllegalArgumentException("Bad string encoding for " +
OrcRecordUpdater.ACID_KEY_INDEX_NAME, e);
}
RecordIdentifier[] result = new RecordIdentifier[stripes.length];
for(int i=0; i < stripes.length; ++i) {
if (stripes[i].length() != 0) {
String[] parts = stripes[i].split(",");
result[i] = new RecordIdentifier();
result[i].setValues(Long.parseLong(parts[0]),
Integer.parseInt(parts[1]), Long.parseLong(parts[2]));
}
}
return result;
}
代码示例来源:origin: apache/hive
/**
* This is smart enough to handle streaming ingest where there could be a
* {@link OrcAcidUtils#DELTA_SIDE_FILE_SUFFIX} side file.
* @param dataFile - ORC acid data file
* @return version property from file if there,
* {@link #ORC_ACID_VERSION_DEFAULT} otherwise
*/
@VisibleForTesting
public static int getAcidVersionFromDataFile(Path dataFile, FileSystem fs) throws IOException {
FileStatus fileStatus = fs.getFileStatus(dataFile);
Reader orcReader = OrcFile.createReader(dataFile,
OrcFile.readerOptions(fs.getConf())
.filesystem(fs)
//make sure to check for side file in case streaming ingest died
.maxLength(getLogicalLength(fs, fileStatus)));
if (orcReader.hasMetadataValue(ACID_VERSION_KEY)) {
char[] versionChar = UTF8.decode(orcReader.getMetadataValue(ACID_VERSION_KEY)).array();
String version = new String(versionChar);
return Integer.valueOf(version);
}
return ORC_ACID_VERSION_DEFAULT;
}
/**
代码示例来源:origin: apache/hive
writer.addUserMetadata(metadataKey, reader.getMetadataValue(metadataKey));
代码示例来源:origin: apache/hive
Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(byteBuf(5,7,11,13,17,19), reader.getMetadataValue("clobber"));
assertEquals(byteBuf(1,2,3,4,5,6,7,-1,-2,127,-128),
reader.getMetadataValue("my.meta"));
assertEquals(bigBuf, reader.getMetadataValue("big"));
try {
reader.getMetadataValue("unknown");
assertTrue(false);
} catch (IllegalArgumentException iae) {
代码示例来源:origin: apache/hive
Mockito.when(reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME))
.thenReturn(ByteBuffer.wrap("10,20,30;40,50,60;40,50,61"
.getBytes("UTF-8")));
代码示例来源:origin: com.facebook.presto.hive/hive-apache
static RecordIdentifier[] parseKeyIndex(Reader reader) {
String[] stripes;
try {
ByteBuffer val =
reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)
.duplicate();
stripes = utf8Decoder.decode(val).toString().split(";");
} catch (CharacterCodingException e) {
throw new IllegalArgumentException("Bad string encoding for " +
OrcRecordUpdater.ACID_KEY_INDEX_NAME, e);
}
RecordIdentifier[] result = new RecordIdentifier[stripes.length];
for(int i=0; i < stripes.length; ++i) {
if (stripes[i].length() != 0) {
String[] parts = stripes[i].split(",");
result[i] = new RecordIdentifier();
result[i].setValues(Long.parseLong(parts[0]),
Integer.parseInt(parts[1]), Long.parseLong(parts[2]));
}
}
return result;
}
代码示例来源:origin: com.facebook.presto/presto-raptor
Closer<Writer, IOException> writer = closer(createWriter(path(output), writerOptions), Writer::close)) {
if (reader.hasMetadataValue(OrcFileMetadata.KEY)) {
ByteBuffer orcFileMetadata = reader.getMetadataValue(OrcFileMetadata.KEY);
writer.get().addUserMetadata(OrcFileMetadata.KEY, orcFileMetadata);
代码示例来源:origin: prestosql/presto
Closer<Writer, IOException> writer = closer(createWriter(path(output), writerOptions), Writer::close)) {
if (reader.hasMetadataValue(OrcFileMetadata.KEY)) {
ByteBuffer orcFileMetadata = reader.getMetadataValue(OrcFileMetadata.KEY);
writer.get().addUserMetadata(OrcFileMetadata.KEY, orcFileMetadata);
内容来源于网络,如有侵权,请联系作者删除!