org.apache.hadoop.hive.ql.exec.Utilities.renameOrMoveFiles()方法的使用及代码示例

x33g5p2x  于2022-02-01 转载在 其他  
字(6.3k)|赞(0)|评价(0)|浏览(106)

本文整理了Java中org.apache.hadoop.hive.ql.exec.Utilities.renameOrMoveFiles()方法的一些代码示例,展示了Utilities.renameOrMoveFiles()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Utilities.renameOrMoveFiles()方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.exec.Utilities
类名称:Utilities
方法名:renameOrMoveFiles

Utilities.renameOrMoveFiles介绍

[英]Rename src to dst, or in the case dst already exists, move files in src to dst. If there is an existing file with the same name, the new file's name will be appended with "_1", "_2", etc.
[中]将src重命名为dst,或者在dst已经存在的情况下,将src中的文件移动到dst。如果存在同名的现有文件,则新文件的名称将附加“_1”、“_2”等。

代码示例

代码示例来源:origin: apache/hive

private void moveUpFiles(Path specPath, Configuration hconf, Logger log)
  throws IOException, HiveException {
 FileSystem fs = specPath.getFileSystem(hconf);
 if (fs.exists(specPath)) {
  FileStatus[] taskOutputDirs = fs.listStatus(specPath);
  if (taskOutputDirs != null) {
   for (FileStatus dir : taskOutputDirs) {
    Utilities.renameOrMoveFiles(fs, dir.getPath(), specPath);
    fs.delete(dir.getPath(), true);
   }
  }
 }
}

代码示例来源:origin: apache/drill

private void moveUpFiles(Path specPath, Configuration hconf, Logger log)
  throws IOException, HiveException {
 FileSystem fs = specPath.getFileSystem(hconf);
 if (fs.exists(specPath)) {
  FileStatus[] taskOutputDirs = fs.listStatus(specPath);
  if (taskOutputDirs != null) {
   for (FileStatus dir : taskOutputDirs) {
    Utilities.renameOrMoveFiles(fs, dir.getPath(), specPath);
    fs.delete(dir.getPath(), true);
   }
  }
 }
}

代码示例来源:origin: apache/hive

private static void moveFile(FileSystem fs, FileStatus file, Path dst) throws IOException,
  HiveException {
 Path srcFilePath = file.getPath();
 String fileName = srcFilePath.getName();
 Path dstFilePath = new Path(dst, fileName);
 if (file.isDir()) {
  renameOrMoveFiles(fs, srcFilePath, dstFilePath);
 } else {
  if (fs.exists(dstFilePath)) {
   int suffix = 0;
   do {
    suffix++;
    dstFilePath = new Path(dst, fileName + "_" + suffix);
   } while (fs.exists(dstFilePath));
  }
  if (!fs.rename(srcFilePath, dstFilePath)) {
   throw new HiveException("Unable to move: " + srcFilePath + " to: " + dst);
  }
 }
}

代码示例来源:origin: apache/hive

Utilities.renameOrMoveFiles(fs, intermediatePath, specPath);

代码示例来源:origin: apache/drill

Path destDir = finalPath.getParent();
try {
 Utilities.renameOrMoveFiles(fs, incompatFile, destDir);
 LOG.info("Moved incompatible file " + incompatFile + " to " +
   destDir);

代码示例来源:origin: apache/drill

Utilities.renameOrMoveFiles(fs, intermediatePath, specPath);

代码示例来源:origin: apache/drill

Path dstFilePath = new Path(dst, fileName);
if (file.isDir()) {
 renameOrMoveFiles(fs, srcFilePath, dstFilePath);

代码示例来源:origin: apache/hive

Utilities.renameOrMoveFiles(fs, incompatFile, destPath);
 LOG.info("Moved incompatible file " + incompatFile + " to " + destPath);
} catch (HiveException e) {

代码示例来源:origin: apache/hive

Utilities.renameOrMoveFiles(fs, tmpPath, specPath);

代码示例来源:origin: apache/drill

Utilities.renameOrMoveFiles(fs, tmpPath, specPath);
perfLogger.PerfLogEnd("FileSinkOperator", "RenameOrMoveFiles");

代码示例来源:origin: com.facebook.presto.hive/hive-apache

private void moveUpFiles(Path specPath, Configuration hconf, Log log)
  throws IOException, HiveException {
 FileSystem fs = specPath.getFileSystem(hconf);
 if (fs.exists(specPath)) {
  FileStatus[] taskOutputDirs = fs.listStatus(specPath);
  if (taskOutputDirs != null) {
   for (FileStatus dir : taskOutputDirs) {
    Utilities.renameOrMoveFiles(fs, dir.getPath(), specPath);
    fs.delete(dir.getPath(), true);
   }
  }
 }
}

代码示例来源:origin: org.apache.hadoop.hive/hive-exec

private void moveUpFiles(String specPath, Configuration hconf, Log log)
  throws IOException, HiveException {
 FileSystem fs = (new Path(specPath)).getFileSystem(hconf);
 Path finalPath = new Path(specPath);
 if (fs.exists(finalPath)) {
  FileStatus[] taskOutputDirs = fs.listStatus(finalPath);
  if (taskOutputDirs != null) {
   for (FileStatus dir : taskOutputDirs) {
    Utilities.renameOrMoveFiles(fs, dir.getPath(), finalPath);
    fs.delete(dir.getPath(), true);
   }
  }
 }
}

代码示例来源:origin: org.apache.hadoop.hive/hive-exec

Utilities.renameOrMoveFiles(fs, intermediatePath, finalPath);

代码示例来源:origin: org.apache.hadoop.hive/hive-exec

public void mvFileToFinalPath(String specPath, Configuration hconf,
  boolean success, Log log, DynamicPartitionCtx dpCtx) throws IOException, HiveException {
 FileSystem fs = (new Path(specPath)).getFileSystem(hconf);
 Path tmpPath = Utilities.toTempPath(specPath);
 Path intermediatePath = new Path(tmpPath.getParent(), tmpPath.getName()
   + ".intermediate");
 Path finalPath = new Path(specPath);
 if (success) {
  if (fs.exists(tmpPath)) {
   // Step1: rename tmp output folder to intermediate path. After this
   // point, updates from speculative tasks still writing to tmpPath
   // will not appear in finalPath.
   log.info("Moving tmp dir: " + tmpPath + " to: " + intermediatePath);
   Utilities.rename(fs, tmpPath, intermediatePath);
   // Step2: remove any tmp file or double-committed output files
   ArrayList<String> emptyBuckets =
    Utilities.removeTempOrDuplicateFiles(fs, intermediatePath, dpCtx);
   // create empty buckets if necessary
   if (emptyBuckets.size() > 0) {
    createEmptyBuckets(hconf, emptyBuckets);
   }
   // Step3: move to the file destination
   log.info("Moving tmp dir: " + intermediatePath + " to: " + finalPath);
   Utilities.renameOrMoveFiles(fs, intermediatePath, finalPath);
  }
 } else {
  fs.delete(tmpPath, true);
 }
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

Path dstFilePath = new Path(dst, fileName);
if (file.isDir()) {
 renameOrMoveFiles(fs, srcFilePath, dstFilePath);

代码示例来源:origin: com.facebook.presto.hive/hive-apache

Path destDir = finalPath.getParent();
try {
 Utilities.renameOrMoveFiles(fs, incompatFile, destDir);
 LOG.info("Moved incompatible file " + incompatFile + " to " +
   destDir);

代码示例来源:origin: com.facebook.presto.hive/hive-apache

Utilities.renameOrMoveFiles(fs, intermediatePath, specPath);

代码示例来源:origin: com.facebook.presto.hive/hive-apache

public static void mvFileToFinalPath(Path specPath, Configuration hconf,
  boolean success, Log log, DynamicPartitionCtx dpCtx, FileSinkDesc conf,
  Reporter reporter) throws IOException,
  HiveException {
 FileSystem fs = specPath.getFileSystem(hconf);
 Path tmpPath = Utilities.toTempPath(specPath);
 Path taskTmpPath = Utilities.toTaskTempPath(specPath);
 if (success) {
  if (fs.exists(tmpPath)) {
   // remove any tmp file or double-committed output files
   ArrayList<String> emptyBuckets =
     Utilities.removeTempOrDuplicateFiles(fs, tmpPath, dpCtx);
   // create empty buckets if necessary
   if (emptyBuckets.size() > 0) {
    createEmptyBuckets(hconf, emptyBuckets, conf, reporter);
   }
   // move to the file destination
   log.info("Moving tmp dir: " + tmpPath + " to: " + specPath);
   Utilities.renameOrMoveFiles(fs, tmpPath, specPath);
  }
 } else {
  fs.delete(tmpPath, true);
 }
 fs.delete(taskTmpPath, true);
}

相关文章

微信公众号

最新文章

更多

Utilities类方法