本文整理了Java中org.apache.hadoop.hive.ql.io.orc.Reader.getStripes
方法的一些代码示例,展示了Reader.getStripes
的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Reader.getStripes
方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.io.orc.Reader
类名称:Reader
方法名:getStripes
[英]Get the list of stripes.
[中]拿到条纹的清单。
代码示例来源:origin: apache/hive
static boolean isAcidKeyIndexValid(Reader reader) {
if (reader.getNumberOfRows() == 0) {
return true;
}
// The number of stripes should match the key index count
List<StripeInformation> stripes = reader.getStripes();
RecordIdentifier[] keyIndex = OrcRecordUpdater.parseKeyIndex(reader);
if (keyIndex == null) {
return false;
}
for (int idx = 0; idx < keyIndex.length; ++idx) {
if (keyIndex[idx] == null) {
LOG.info("*** keyIndex[" + idx + "] is null");
return false;
}
}
return stripes.size() == keyIndex.length;
}
代码示例来源:origin: apache/hive
public OrcFileStripeMergeRecordReader(Configuration conf, FileSplit split) throws IOException {
path = split.getPath();
start = split.getStart();
end = start + split.getLength();
FileSystem fs = path.getFileSystem(conf);
this.reader = OrcFile.createReader(path, OrcFile.readerOptions(conf).filesystem(fs));
this.iter = reader.getStripes().iterator();
this.stripeIdx = 0;
this.stripeStatistics = ((ReaderImpl) reader).getOrcProtoStripeStatistics();
}
代码示例来源:origin: apache/drill
public OrcFileStripeMergeRecordReader(Configuration conf, FileSplit split) throws IOException {
path = split.getPath();
start = split.getStart();
end = start + split.getLength();
FileSystem fs = path.getFileSystem(conf);
this.reader = OrcFile.createReader(path, OrcFile.readerOptions(conf).filesystem(fs));
this.iter = reader.getStripes().iterator();
this.stripeIdx = 0;
this.stripeStatistics = ((ReaderImpl) reader).getOrcProtoStripeStatistics();
}
代码示例来源:origin: apache/drill
int stripeCount = 0;
boolean isTail = true;
List<StripeInformation> stripes = reader.getStripes();
for(StripeInformation stripe: stripes) {
if (offset > stripe.getOffset()) {
代码示例来源:origin: apache/hive
List<StripeInformation> stripes = reader.getStripes();
List<StripeStatistics> stripeStats = reader.getOrcProtoStripeStatistics();
代码示例来源:origin: apache/hive
RecordIdentifier maxKey = null;
List<StripeInformation> stripes = reader.getStripes();
for(StripeInformation stripe: stripes) {
if (offset > stripe.getOffset()) {
代码示例来源:origin: apache/hive
@Test
public void emptyFile() throws Exception {
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector = ObjectInspectorFactory.getReflectionObjectInspector
(BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
Writer writer = OrcFile.createWriter(testFilePath,
OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(1000)
.compress(CompressionKind.NONE)
.bufferSize(100));
writer.close();
Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(false, reader.rows().hasNext());
assertEquals(CompressionKind.NONE, reader.getCompression());
assertEquals(0, reader.getNumberOfRows());
assertEquals(0, reader.getCompressionSize());
assertEquals(false, reader.getMetadataKeys().iterator().hasNext());
assertEquals(3, reader.getContentLength());
assertEquals(false, reader.getStripes().iterator().hasNext());
}
代码示例来源:origin: apache/hive
int rowCount = 0;
long currentOffset = -1;
for (StripeInformation stripe : reader.getStripes()) {
stripeCount += 1;
rowCount += stripe.getNumberOfRows();
代码示例来源:origin: apache/drill
long maxOffset = options.getMaxOffset();
boolean isTail = true;
for(StripeInformation stripe: reader.getStripes()) {
if (offset > stripe.getOffset()) {
rowOffset += stripe.getNumberOfRows();
代码示例来源:origin: apache/hive
assertEquals(50000, reader.getNumberOfRows());
assertEquals(0, reader.getRowIndexStride());
StripeInformation stripe = reader.getStripes().iterator().next();
assertEquals(true, stripe.getDataLength() != 0);
assertEquals(0, stripe.getIndexLength());
代码示例来源:origin: apache/hive
reader.getStripes().iterator();
long offsetOfStripe2 = 0;
long offsetOfStripe4 = 0;
代码示例来源:origin: apache/hive
long maxRowId = syntheticProps.getRowIdOffset();
for(StripeInformation stripe: reader.getStripes()) {
if (splitStart > stripe.getOffset()) {
代码示例来源:origin: apache/hive
reader.getStripes().iterator();
long offsetOfStripe2 = 0;
long offsetOfStripe4 = 0;
代码示例来源:origin: apache/hive
OrcFile.readerOptions(conf).filesystem(fs));
int i = 0;
for(StripeInformation stripe: reader.getStripes()) {
i += 1;
assertTrue("stripe " + i + " is too long at " + stripe.getDataLength(),
代码示例来源:origin: apache/hive
.thenReturn(ByteBuffer.wrap("10,20,30;40,50,60;40,50,61"
.getBytes("UTF-8")));
Mockito.when(reader.getStripes())
.thenReturn(createStripes(2, 2, 1));
代码示例来源:origin: apache/hive
OrcFile.readerOptions(conf).filesystem(fs));
int i = 0;
for(StripeInformation stripe: reader.getStripes()) {
i += 1;
assertTrue("stripe " + i + " is too long at " + stripe.getDataLength(),
代码示例来源:origin: apache/hive
List<StripeInformation> stripes = orcReader.getStripes();
StripeInformation lastStripe = stripes.get(stripes.size() - 1);
long lastStripeOffset = lastStripe.getOffset();
代码示例来源:origin: apache/hive
for(StripeInformation stripe: reader.getStripes()) {
if (offset > stripe.getOffset()) {
rowOffset += stripe.getNumberOfRows();
代码示例来源:origin: apache/hive
List<StripeInformation> stripes = reader.getStripes();
代码示例来源:origin: apache/hive
List<StripeInformation> stripes = reader.getStripes();
内容来源于网络,如有侵权,请联系作者删除!