org.apache.hadoop.hive.ql.metadata.Hive.getQualifiedPathWithoutSchemeAndAuthority()方法的使用及代码示例

x33g5p2x  于2022-01-20 转载在 其他  
字(6.1k)|赞(0)|评价(0)|浏览(120)

本文整理了Java中org.apache.hadoop.hive.ql.metadata.Hive.getQualifiedPathWithoutSchemeAndAuthority()方法的一些代码示例,展示了Hive.getQualifiedPathWithoutSchemeAndAuthority()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Hive.getQualifiedPathWithoutSchemeAndAuthority()方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.metadata.Hive
类名称:Hive
方法名:getQualifiedPathWithoutSchemeAndAuthority

Hive.getQualifiedPathWithoutSchemeAndAuthority介绍

暂无

代码示例

代码示例来源:origin: apache/hive

private static boolean isSubDir(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs, boolean isSrcLocal) {
 if (srcf == null) {
  LOG.debug("The source path is null for isSubDir method.");
  return false;
 }
 String fullF1 = getQualifiedPathWithoutSchemeAndAuthority(srcf, srcFs).toString() + Path.SEPARATOR;
 String fullF2 = getQualifiedPathWithoutSchemeAndAuthority(destf, destFs).toString() + Path.SEPARATOR;
 boolean isInTest = HiveConf.getBoolVar(srcFs.getConf(), ConfVars.HIVE_IN_TEST);
 // In the automation, the data warehouse is the local file system based.
 LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
 if (isInTest) {
  return fullF1.startsWith(fullF2);
 }
 // schema is diff, return false
 String schemaSrcf = srcf.toUri().getScheme();
 String schemaDestf = destf.toUri().getScheme();
 // if the schemaDestf is null, it means the destination is not in the local file system
 if (schemaDestf == null && isSrcLocal) {
  LOG.debug("The source file is in the local while the dest not.");
  return false;
 }
 // If both schema information are provided, they should be the same.
 if (schemaSrcf != null && schemaDestf != null && !schemaSrcf.equals(schemaDestf)) {
  LOG.debug("The source path's schema is " + schemaSrcf +
   " and the destination path's schema is " + schemaDestf + ".");
  return false;
 }
 LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
 return fullF1.startsWith(fullF2);
}

代码示例来源:origin: apache/hive

public static void clearDestForSubDirSrc(final HiveConf conf, Path dest,
  Path src, boolean isSrcLocal) throws IOException {
 FileSystem destFS = dest.getFileSystem(conf);
 FileSystem srcFS = src.getFileSystem(conf);
 if (isSubDir(src, dest, srcFS, destFS, isSrcLocal)) {
  final Path fullSrcPath = getQualifiedPathWithoutSchemeAndAuthority(src, srcFS);
  final Path fullDestPath = getQualifiedPathWithoutSchemeAndAuthority(dest, destFS);
  if (fullSrcPath.equals(fullDestPath)) {
   return;
  }
  Path parent = fullSrcPath;
  while (!parent.getParent().equals(fullDestPath)) {
   parent = parent.getParent();
  }
  FileStatus[] existingFiles = destFS.listStatus(
    dest, FileUtils.HIDDEN_FILES_PATH_FILTER);
  for (FileStatus fileStatus : existingFiles) {
   if (!fileStatus.getPath().getName().equals(parent.getName())) {
    destFS.delete(fileStatus.getPath(), true);
   }
  }
 }
}

代码示例来源:origin: apache/drill

private static boolean isSubDir(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs, boolean isSrcLocal) {
 if (srcf == null) {
  LOG.debug("The source path is null for isSubDir method.");
  return false;
 }
 String fullF1 = getQualifiedPathWithoutSchemeAndAuthority(srcf, srcFs).toString() + Path.SEPARATOR;
 String fullF2 = getQualifiedPathWithoutSchemeAndAuthority(destf, destFs).toString() + Path.SEPARATOR;
 boolean isInTest = HiveConf.getBoolVar(srcFs.getConf(), ConfVars.HIVE_IN_TEST);
 // In the automation, the data warehouse is the local file system based.
 LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
 if (isInTest) {
  return fullF1.startsWith(fullF2);
 }
 // schema is diff, return false
 String schemaSrcf = srcf.toUri().getScheme();
 String schemaDestf = destf.toUri().getScheme();
 // if the schemaDestf is null, it means the destination is not in the local file system
 if (schemaDestf == null && isSrcLocal) {
  LOG.debug("The source file is in the local while the dest not.");
  return false;
 }
 // If both schema information are provided, they should be the same.
 if (schemaSrcf != null && schemaDestf != null && !schemaSrcf.equals(schemaDestf)) {
  LOG.debug("The source path's schema is " + schemaSrcf +
   " and the destination path's schema is " + schemaDestf + ".");
  return false;
 }
 LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
 return fullF1.startsWith(fullF2);
}

代码示例来源:origin: apache/drill

public static void clearDestForSubDirSrc(final HiveConf conf, Path dest,
  Path src, boolean isSrcLocal) throws IOException {
 FileSystem destFS = dest.getFileSystem(conf);
 FileSystem srcFS = src.getFileSystem(conf);
 if (isSubDir(src, dest, srcFS, destFS, isSrcLocal)) {
  final Path fullSrcPath = getQualifiedPathWithoutSchemeAndAuthority(src, srcFS);
  final Path fullDestPath = getQualifiedPathWithoutSchemeAndAuthority(dest, destFS);
  if (fullSrcPath.equals(fullDestPath)) {
   return;
  }
  Path parent = fullSrcPath;
  while (!parent.getParent().equals(fullDestPath)) {
   parent = parent.getParent();
  }
  FileStatus[] existingFiles = destFS.listStatus(
    dest, FileUtils.HIDDEN_FILES_PATH_FILTER);
  for (FileStatus fileStatus : existingFiles) {
   if (!fileStatus.getPath().getName().equals(parent.getName())) {
    destFS.delete(fileStatus.getPath(), true);
   }
  }
 }
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

private static boolean isSubDir(Path srcf, Path destf, FileSystem fs, boolean isSrcLocal){
 if (srcf == null) {
  LOG.debug("The source path is null for isSubDir method.");
  return false;
 }
 String fullF1 = getQualifiedPathWithoutSchemeAndAuthority(srcf, fs);
 String fullF2 = getQualifiedPathWithoutSchemeAndAuthority(destf, fs);
 boolean isInTest = Boolean.valueOf(HiveConf.getBoolVar(fs.getConf(), ConfVars.HIVE_IN_TEST));
 // In the automation, the data warehouse is the local file system based.
 LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
 if (isInTest) {
  return fullF1.startsWith(fullF2);
 }
 // schema is diff, return false
 String schemaSrcf = srcf.toUri().getScheme();
 String schemaDestf = destf.toUri().getScheme();
 // if the schemaDestf is null, it means the destination is not in the local file system
 if (schemaDestf == null && isSrcLocal) {
  LOG.debug("The source file is in the local while the dest not.");
  return false;
 }
 // If both schema information are provided, they should be the same.
 if (schemaSrcf != null && schemaDestf != null && !schemaSrcf.equals(schemaDestf)) {
  LOG.debug("The source path's schema is " + schemaSrcf +
   " and the destination path's schema is " + schemaDestf + ".");
  return false;
 }
 LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
 return fullF1.startsWith(fullF2);
}

相关文章

微信公众号

最新文章

更多

Hive类方法