org.apache.hadoop.hive.ql.metadata.Hive.copyFiles()方法的使用及代码示例

x33g5p2x  于2022-01-20 转载在 其他  
字(5.4k)|赞(0)|评价(0)|浏览(148)

本文整理了Java中org.apache.hadoop.hive.ql.metadata.Hive.copyFiles()方法的一些代码示例,展示了Hive.copyFiles()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Hive.copyFiles()方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.metadata.Hive
类名称:Hive
方法名:copyFiles

Hive.copyFiles介绍

[英]Copy files. This handles building the mapping for buckets and such between the source and destination
[中]复制文件。这将处理为源和目标之间的bucket等构建映射

代码示例

代码示例来源:origin: apache/hive

copyFiles(conf, fs, srcs, srcFs, destf, isSrcLocal, isOverwrite,
    newFiles, isFullAcidTable && !isBucketed, isManaged);

代码示例来源:origin: apache/drill

moveAcidFiles(srcFs, srcs, destf, newFiles);
} else {
 copyFiles(conf, fs, srcs, srcFs, destf, isSrcLocal, newFiles);

代码示例来源:origin: apache/hive

@Test
public void testCopyNewFilesOnDifferentFileSystem() throws IOException {
 Path sourcePath = new Path(sourceFolder.getRoot().getAbsolutePath());
 sourceFolder.newFile("000000_0");
 sourceFolder.newFile("000001_0");
 sourceFolder.newFile("000000_0.gz");
 sourceFolder.newFile("000001_0.gz");
 Path targetPath = new Path(targetFolder.getRoot().getAbsolutePath());
 // Simulate different filesystems by returning a different URI
 FileSystem spyTargetFs = Mockito.spy(targetPath.getFileSystem(hiveConf));
 Mockito.when(spyTargetFs.getUri()).thenReturn(URI.create("hdfs://" + targetPath.toUri().getPath()));
 try {
  Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false, false, false);
 } catch (HiveException e) {
  e.printStackTrace();
  assertTrue("Hive.copyFiles() threw an unexpected exception.", false);
 }
 assertTrue(spyTargetFs.exists(new Path(targetPath, "000000_0")));
 assertTrue(spyTargetFs.exists(new Path(targetPath, "000001_0")));
 assertTrue(spyTargetFs.exists(new Path(targetPath, "000000_0.gz")));
 assertTrue(spyTargetFs.exists(new Path(targetPath, "000001_0.gz")));
}

代码示例来源:origin: apache/hive

@Test
public void testRenameNewFilesOnSameFileSystem() throws IOException {
 Path sourcePath = new Path(sourceFolder.getRoot().getAbsolutePath());
 sourceFolder.newFile("000000_0");
 sourceFolder.newFile("000001_0");
 sourceFolder.newFile("000000_0.gz");
 sourceFolder.newFile("000001_0.gz");
 Path targetPath = new Path(targetFolder.getRoot().getAbsolutePath());
 FileSystem targetFs = targetPath.getFileSystem(hiveConf);
 try {
  Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false,null, false, false, false);
 } catch (HiveException e) {
  e.printStackTrace();
  assertTrue("Hive.copyFiles() threw an unexpected exception.", false);
 }
 assertTrue(targetFs.exists(new Path(targetPath, "000000_0")));
 assertTrue(targetFs.exists(new Path(targetPath, "000001_0")));
 assertTrue(targetFs.exists(new Path(targetPath, "000000_0.gz")));
 assertTrue(targetFs.exists(new Path(targetPath, "000001_0.gz")));
}

代码示例来源:origin: apache/hive

Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false, false, false);
} catch (HiveException e) {
 e.printStackTrace();
 Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false, false, false);
} catch (HiveException e) {
 e.printStackTrace();

代码示例来源:origin: apache/hive

Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null, false, false, false);
} catch (HiveException e) {
 e.printStackTrace();
 Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null, false, false, false);
} catch (HiveException e) {
 e.printStackTrace();

代码示例来源:origin: apache/drill

Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, newFiles);

代码示例来源:origin: apache/drill

try {
 fs = tbl.getDataLocation().getFileSystem(sessionConf);
 copyFiles(sessionConf, loadPath, tbl.getPath(), fs, isSrcLocal, isAcid, newFiles);
} catch (IOException e) {
 throw new HiveException("addFiles: filesystem error in check phase", e);

代码示例来源:origin: apache/hive

} else {
 FileSystem fs = destPath.getFileSystem(conf);
 copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation,
   (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles,
   tbl.getNumBuckets() > 0, isFullAcidTable, isManaged);

代码示例来源:origin: apache/hive

try {
 FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
 copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation,
   loadFileType == LoadFileType.OVERWRITE_EXISTING, newFiles,
   tbl.getNumBuckets() > 0, isFullAcidTable, isManaged);

代码示例来源:origin: org.apache.hadoop.hive/hive-exec

/**
 * Inserts files specified into the partition. Works by moving files
 *
 * @param srcf
 *          Files to be moved. Leaf directories or globbed file paths
 */
protected void copyFiles(Path srcf) throws HiveException {
 FileSystem fs;
 try {
  fs = FileSystem.get(getDataLocation(), Hive.get().getConf());
  Hive.copyFiles(srcf, new Path(getDataLocation().getPath()), fs);
 } catch (IOException e) {
  throw new HiveException("addFiles: filesystem error in check phase", e);
 }
}

代码示例来源:origin: org.apache.hadoop.hive/hive-exec

} else {
 FileSystem fs = FileSystem.get(tbl.getDataLocation(), getConf());
 Hive.copyFiles(loadPath, newPartPath, fs);

代码示例来源:origin: com.facebook.presto.hive/hive-apache

newFiles = new ArrayList<Path>();
FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, newFiles);

代码示例来源:origin: com.facebook.presto.hive/hive-apache

try {
 fs = tbl.getDataLocation().getFileSystem(sessionConf);
 copyFiles(sessionConf, loadPath, tbl.getPath(), fs, isSrcLocal, isAcid, newFiles);
} catch (IOException e) {
 throw new HiveException("addFiles: filesystem error in check phase", e);

相关文章

微信公众号

最新文章

更多

Hive类方法