org.apache.hadoop.hive.ql.io.orc.Reader.getObjectInspector()方法的使用及代码示例

x33g5p2x  于2022-01-29 转载在 其他  
字(7.6k)|赞(0)|评价(0)|浏览(96)

本文整理了Java中org.apache.hadoop.hive.ql.io.orc.Reader.getObjectInspector方法的一些代码示例，展示了Reader.getObjectInspector的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Reader.getObjectInspector方法的具体详情如下：
包路径：org.apache.hadoop.hive.ql.io.orc.Reader
类名称：Reader
方法名：getObjectInspector

Reader.getObjectInspector介绍

[英]Get the object inspector for looking at the objects.
[中]让对象检查器查看对象。

代码示例

代码示例来源：origin: apache/hive

private ArrayList<SampleRec> dumpBucket(Path orcFile) throws IOException {
 org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.getLocal(new Configuration());
 Reader reader = OrcFile.createReader(orcFile,
  OrcFile.readerOptions(conf).filesystem(fs));
 RecordReader rows = reader.rows();
 StructObjectInspector inspector = (StructObjectInspector) reader
  .getObjectInspector();
 System.out.format("Found Bucket File : %s \n", orcFile.getName());
 ArrayList<SampleRec> result = new ArrayList<SampleRec>();
 while (rows.hasNext()) {
  Object row = rows.next(null);
  SampleRec rec = (SampleRec) deserializeDeltaFileRow(row, inspector)[5];
  result.add(rec);
 }
 return result;
}

代码示例来源：origin: prestodb/presto

private static void assertFileContentsOrcHive(
    Type type,
    TempFile tempFile,
    Iterable<?> expectedValues)
    throws Exception
{
  JobConf configuration = new JobConf(new Configuration(false));
  configuration.set(READ_COLUMN_IDS_CONF_STR, "0");
  configuration.setBoolean(READ_ALL_COLUMNS, false);
  Reader reader = OrcFile.createReader(
      new Path(tempFile.getFile().getAbsolutePath()),
      new ReaderOptions(configuration));
  org.apache.hadoop.hive.ql.io.orc.RecordReader recordReader = reader.rows();
  StructObjectInspector rowInspector = (StructObjectInspector) reader.getObjectInspector();
  StructField field = rowInspector.getStructFieldRef("test");
  Iterator<?> iterator = expectedValues.iterator();
  Object rowData = null;
  while (recordReader.hasNext()) {
    rowData = recordReader.next(rowData);
    Object expectedValue = iterator.next();
    Object actualValue = rowInspector.getStructFieldData(rowData, field);
    actualValue = decodeRecordReaderValue(type, actualValue);
    assertColumnValueEquals(type, actualValue, expectedValue);
  }
  assertFalse(iterator.hasNext());
}

代码示例来源：origin: apache/hive

private ArrayList<SampleRec> dumpBucket(Path orcFile) throws IOException {
 org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.getLocal(new Configuration());
 Reader reader = OrcFile.createReader(orcFile,
     OrcFile.readerOptions(conf).filesystem(fs));
 RecordReader rows = reader.rows();
 StructObjectInspector inspector = (StructObjectInspector) reader
     .getObjectInspector();
 System.out.format("Found Bucket File : %s \n", orcFile.getName());
 ArrayList<SampleRec> result = new ArrayList<SampleRec>();
 while (rows.hasNext()) {
  Object row = rows.next(null);
  SampleRec rec = (SampleRec) deserializeDeltaFileRow(row, inspector)[5];
  result.add(rec);
 }
 return result;
}

代码示例来源：origin: apache/hive

ObjectInspector orcOi = reader.getObjectInspector();
ObjectInspector stoi = TypeInfoUtils
  .getStandardJavaObjectInspectorFromTypeInfo(OrcTestReducer3.typeInfo);

代码示例来源：origin: apache/hive

.version(reader.getFileVersion())
.rowIndexStride(reader.getRowIndexStride())
.inspector(reader.getObjectInspector());
rr.seekToRow(lastRow);
OrcStruct row = (OrcStruct) rr.next(null);
StructObjectInspector soi = (StructObjectInspector) reader.getObjectInspector();

代码示例来源：origin: apache/hive

assertEquals(reader.getCompression(), CompressionKind.ZLIB);
StructObjectInspector soi =
  (StructObjectInspector)reader.getObjectInspector();
StructTypeInfo ti =
  (StructTypeInfo)TypeInfoUtils.getTypeInfoFromObjectInspector(soi);

代码示例来源：origin: apache/hive

.version(fileVersion)
.rowIndexStride(rowIndexStride)
.inspector(reader.getObjectInspector());

代码示例来源：origin: apache/hive

(StructObjectInspector) reader.getObjectInspector();
assertEquals(ObjectInspector.Category.STRUCT,
  readerInspector.getCategory());

代码示例来源：origin: apache/hive

.getObjectInspector();
assertEquals(ObjectInspector.Category.STRUCT, readerInspector.getCategory());
assertEquals("struct<boolean1:boolean,byte1:tinyint,short1:smallint,"

代码示例来源：origin: apache/drill

.version(version)
.rowIndexStride(rowIndexStride)
.inspector(reader.getObjectInspector());

代码示例来源：origin: apache/hive

(StructObjectInspector) reader.getObjectInspector();
assertEquals(ObjectInspector.Category.STRUCT,
  readerInspector.getCategory());

代码示例来源：origin: apache/hive

(StructObjectInspector) reader.getObjectInspector();
assertEquals(ObjectInspector.Category.STRUCT,
  readerInspector.getCategory());

代码示例来源：origin: apache/hive

OrcFile.readerOptions(conf).filesystem(fs));
StructObjectInspector readerInspector =
  (StructObjectInspector) reader.getObjectInspector();
List<? extends StructField> fields = readerInspector.getAllStructFieldRefs();
HiveDecimalObjectInspector doi = (HiveDecimalObjectInspector) readerInspector.

代码示例来源：origin: apache/hive

row = (OrcStruct) rows.next(null);
assertEquals(1, rows.getRowNumber());
inspector = reader.getObjectInspector();
assertEquals("struct<time:timestamp,union:uniontype<int,string>,decimal:decimal(38,18)>",
  inspector.getTypeName());

代码示例来源：origin: apache/hive

OrcFile.readerOptions(conf).filesystem(fs));
StructObjectInspector readerInspector =
  (StructObjectInspector) reader.getObjectInspector();
List<? extends StructField> fields = readerInspector.getAllStructFieldRefs();
HiveDecimalObjectInspector doi = (HiveDecimalObjectInspector) readerInspector.

代码示例来源：origin: org.apache.pig/pig

private TypeInfo getTypeInfoFromLocation(String location, Job job) throws IOException {
  FileSystem fs = FileSystem.get(new Path(location).toUri(), job.getConfiguration());
  Path path = getFirstFile(location, fs, new NonEmptyOrcFileFilter(fs));
  if (path == null) {
    log.info("Cannot find any ORC files from " + location +
        ". Probably multiple load store in script.");
    return null;
  }
  Reader reader = OrcFile.createReader(fs, path);
  ObjectInspector oip = (ObjectInspector)reader.getObjectInspector();
  return TypeInfoUtils.getTypeInfoFromObjectInspector(oip);
}

代码示例来源：origin: ECNU-1X/DataX-Masking

private int getAllColumnsCount(String filePath) {
  int columnsCount;
  final String colFinal = "_col";
  Path path = new Path(filePath);
  try {
    Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(hadoopConf));
    String type_struct = reader.getObjectInspector().getTypeName();
    columnsCount = (type_struct.length() - type_struct.replace(colFinal, "").length())
        / colFinal.length();
    return columnsCount;
  } catch (IOException e) {
    String message = "读取orcfile column列数失败，请联系系统管理员";
    throw DataXException.asDataXException(HdfsReaderErrorCode.READ_FILE_ERROR, message);
  }
}

代码示例来源：origin: org.apache.spark/spark-hive_2.10

public SparkOrcNewRecordReader(Reader file, Configuration conf,
  long offset, long length) throws IOException {
 List<OrcProto.Type> types = file.getTypes();
 numColumns = (types.size() == 0) ? 0 : types.get(0).getSubtypesCount();
 value = new OrcStruct(numColumns);
 this.reader = OrcInputFormat.createReaderFromFile(file, conf, offset,
   length);
 this.objectInspector = file.getObjectInspector();
}

代码示例来源：origin: org.apache.spark/spark-hive_2.11

public SparkOrcNewRecordReader(Reader file, Configuration conf,
  long offset, long length) throws IOException {
 List<OrcProto.Type> types = file.getTypes();
 numColumns = (types.size() == 0) ? 0 : types.get(0).getSubtypesCount();
 value = new OrcStruct(numColumns);
 this.reader = OrcInputFormat.createReaderFromFile(file, conf, offset,
   length);
 this.objectInspector = file.getObjectInspector();
}

代码示例来源：origin: org.apache.pig/pig

@Override
  public boolean accept(Path path) {
    Reader reader;
    try {
      reader = OrcFile.createReader(fs, path);
      ObjectInspector oip = (ObjectInspector)reader.getObjectInspector();
      ResourceFieldSchema rs = HiveUtils.getResourceFieldSchema(TypeInfoUtils.getTypeInfoFromObjectInspector(oip));
      if (rs.getSchema().getFields().length!=0) {
        return true;
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    return false;
  }
}

内容来源于网络，如有侵权，请联系作者删除！

reader

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com