org.apache.hadoop.hive.ql.exec.Utilities.createEmptyFile()方法的使用及代码示例

x33g5p2x  于2022-02-01 转载在 其他  
字(5.7k)|赞(0)|评价(0)|浏览(105)

本文整理了Java中org.apache.hadoop.hive.ql.exec.Utilities.createEmptyFile()方法的一些代码示例,展示了Utilities.createEmptyFile()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Utilities.createEmptyFile()方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.exec.Utilities
类名称:Utilities
方法名:createEmptyFile

Utilities.createEmptyFile介绍

暂无

代码示例

代码示例来源:origin: apache/hive

@SuppressWarnings("rawtypes")
private static Path createDummyFileForEmptyTable(JobConf job, MapWork work,
  Path hiveScratchDir, String alias)
    throws Exception {
 TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc();
 if (tableDesc.isNonNative()) {
  // if it does not need native storage, we can't create an empty file for it.
  return null;
 }
 Properties props = tableDesc.getProperties();
 HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, tableDesc);
 Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job, props, false);
 LOG.info("Changed input file for alias {} to newPath", alias, newPath);
 // update the work
 LinkedHashMap<Path, ArrayList<String>> pathToAliases = work.getPathToAliases();
 ArrayList<String> newList = new ArrayList<String>(1);
 newList.add(alias);
 pathToAliases.put(newPath, newList);
 work.setPathToAliases(pathToAliases);
 PartitionDesc pDesc = work.getAliasToPartnInfo().get(alias).clone();
 work.addPathToPartitionInfo(newPath, pDesc);
 return newPath;
}

代码示例来源:origin: apache/drill

@SuppressWarnings("rawtypes")
private static Path createDummyFileForEmptyTable(JobConf job, MapWork work,
  Path hiveScratchDir, String alias)
    throws Exception {
 TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc();
 if (tableDesc.isNonNative()) {
  // if it does not need native storage, we can't create an empty file for it.
  return null;
 }
 Properties props = tableDesc.getProperties();
 HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, tableDesc);
 Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job, props, false);
 if (LOG.isInfoEnabled()) {
  LOG.info("Changed input file for alias " + alias + " to " + newPath);
 }
 // update the work
 LinkedHashMap<Path, ArrayList<String>> pathToAliases = work.getPathToAliases();
 ArrayList<String> newList = new ArrayList<String>();
 newList.add(alias);
 pathToAliases.put(newPath, newList);
 work.setPathToAliases(pathToAliases);
 PartitionDesc pDesc = work.getAliasToPartnInfo().get(alias).clone();
 work.addPathToPartitionInfo(newPath, pDesc);
 return newPath;
}

代码示例来源:origin: apache/hive

@SuppressWarnings("rawtypes")
private static Path createDummyFileForEmptyPartition(Path path, JobConf job, PartitionDesc partDesc,
                           Path hiveScratchDir) throws Exception {
 String strPath = path.toString();
 // The input file does not exist, replace it by a empty file
 if (partDesc.getTableDesc().isNonNative()) {
  // if this isn't a hive table we can't create an empty file for it.
  return path;
 }
 Properties props = SerDeUtils.createOverlayedProperties(
   partDesc.getTableDesc().getProperties(), partDesc.getProperties());
 HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, partDesc);
 boolean oneRow = partDesc.getInputFileFormatClass() == OneNullRowInputFormat.class;
 Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job, props, oneRow);
 LOG.info("Changed input file {} to empty file {} ({})", strPath, newPath, oneRow);
 return newPath;
}

代码示例来源:origin: apache/drill

@SuppressWarnings("rawtypes")
private static Path createDummyFileForEmptyPartition(Path path, JobConf job, MapWork work,
  Path hiveScratchDir)
    throws Exception {
 String strPath = path.toString();
 // The input file does not exist, replace it by a empty file
 PartitionDesc partDesc = work.getPathToPartitionInfo().get(path);
 if (partDesc.getTableDesc().isNonNative()) {
  // if this isn't a hive table we can't create an empty file for it.
  return path;
 }
 Properties props = SerDeUtils.createOverlayedProperties(
   partDesc.getTableDesc().getProperties(), partDesc.getProperties());
 HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, partDesc);
 boolean oneRow = partDesc.getInputFileFormatClass() == OneNullRowInputFormat.class;
 Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job, props, oneRow);
 if (LOG.isInfoEnabled()) {
  LOG.info("Changed input file " + strPath + " to empty file " + newPath + " (" + oneRow + ")");
 }
 // update the work
 work.addPathToAlias(newPath, work.getPathToAliases().get(path));
 work.removePathToAlias(path);
 work.removePathToPartitionInfo(path);
 work.addPathToPartitionInfo(newPath, partDesc);
 return newPath;
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

@SuppressWarnings("rawtypes")
private static Path createDummyFileForEmptyTable(JobConf job, MapWork work,
  Path hiveScratchDir, String alias, int sequenceNumber)
    throws Exception {
 TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc();
 if (tableDesc.isNonNative()) {
  // if this isn't a hive table we can't create an empty file for it.
  return null;
 }
 Properties props = tableDesc.getProperties();
 HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, tableDesc);
 Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job,
   sequenceNumber, props, false);
 if (LOG.isInfoEnabled()) {
  LOG.info("Changed input file for alias " + alias + " to " + newPath);
 }
 // update the work
 LinkedHashMap<String, ArrayList<String>> pathToAliases = work.getPathToAliases();
 ArrayList<String> newList = new ArrayList<String>();
 newList.add(alias);
 pathToAliases.put(newPath.toUri().toString(), newList);
 work.setPathToAliases(pathToAliases);
 LinkedHashMap<String, PartitionDesc> pathToPartitionInfo = work.getPathToPartitionInfo();
 PartitionDesc pDesc = work.getAliasToPartnInfo().get(alias).clone();
 pathToPartitionInfo.put(newPath.toUri().toString(), pDesc);
 work.setPathToPartitionInfo(pathToPartitionInfo);
 return newPath;
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job,
  sequenceNumber, props, oneRow);

相关文章

微信公众号

最新文章

更多

Utilities类方法