org.apache.hadoop.hive.ql.metadata.Partition.getSkewedColNames()方法的使用及代码示例

x33g5p2x  于2022-01-26 转载在 其他  
字(7.7k)|赞(0)|评价(0)|浏览(89)

本文整理了Java中org.apache.hadoop.hive.ql.metadata.Partition.getSkewedColNames()方法的一些代码示例，展示了Partition.getSkewedColNames()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Partition.getSkewedColNames()方法的具体详情如下：
包路径：org.apache.hadoop.hive.ql.metadata.Partition
类名称：Partition
方法名：getSkewedColNames

Partition.getSkewedColNames介绍

暂无

代码示例

代码示例来源：origin: apache/hive

/**
  * Check if we prune it for list bucketing
  * 1. column name is part of skewed column
  * 2. partition has skewed value to location map
  * @param part
  * @param columnName
  * @return
  */
 private boolean isPruneForListBucketing(Partition part, String columnName) {
  return ListBucketingPrunerUtils.isListBucketingPart(part)
    && (part.getSkewedColNames().contains(columnName));
 }
}

代码示例来源：origin: apache/drill

/**
  * Check if we prune it for list bucketing
  * 1. column name is part of skewed column
  * 2. partition has skewed value to location map
  * @param part
  * @param columnName
  * @return
  */
 private boolean isPruneForListBucketing(Partition part, String columnName) {
  return ListBucketingPrunerUtils.isListBucketingPart(part)
    && (part.getSkewedColNames().contains(columnName));
 }
}

代码示例来源：origin: apache/hive

/**
 * check if the partition is list bucketing
 *
 * @param part
 * @return
 */
public static boolean isListBucketingPart(Partition part) {
 return (part.getSkewedColNames() != null) && (part.getSkewedColNames().size() > 0)
   && (part.getSkewedColValues() != null) && (part.getSkewedColValues().size() > 0)
   && (part.getSkewedColValueLocationMaps() != null)
   && (part.getSkewedColValueLocationMaps().size() > 0);
}

代码示例来源：origin: apache/drill

/**
 * check if the partition is list bucketing
 *
 * @param part
 * @return
 */
public static boolean isListBucketingPart(Partition part) {
 return (part.getSkewedColNames() != null) && (part.getSkewedColNames().size() > 0)
   && (part.getSkewedColValues() != null) && (part.getSkewedColValues().size() > 0)
   && (part.getSkewedColValueLocationMaps() != null)
   && (part.getSkewedColValueLocationMaps().size() > 0);
}

代码示例来源：origin: apache/hive

/**
 * Decide whether should select the default directory.
 *
 * @param part
 * @param selectedPaths
 * @param nonSkewedValueMatchResult
 */
private static void decideDefaultDirSelection(Partition part, List<Path> selectedPaths,
  List<Boolean> nonSkewedValueMatchResult) {
 boolean skipDefDir = true;
 for (Boolean v : nonSkewedValueMatchResult) {
  if ((v == null) || v) {
   skipDefDir = false; // we skip default directory only if all value is false
   break;
  }
 }
 if (!skipDefDir) {
  StringBuilder builder = new StringBuilder();
  builder.append(part.getLocation());
  builder.append(Path.SEPARATOR);
  builder
    .append((FileUtils.makeDefaultListBucketingDirName(
      part.getSkewedColNames(),
      ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)));
  selectedPaths.add(new Path(builder.toString()));
 }
}

代码示例来源：origin: apache/drill

/**
 * Decide whether should select the default directory.
 *
 * @param part
 * @param selectedPaths
 * @param nonSkewedValueMatchResult
 */
private static void decideDefaultDirSelection(Partition part, List<Path> selectedPaths,
  List<Boolean> nonSkewedValueMatchResult) {
 boolean skipDefDir = true;
 for (Boolean v : nonSkewedValueMatchResult) {
  if ((v == null) || v) {
   skipDefDir = false; // we skip default directory only if all value is false
   break;
  }
 }
 if (!skipDefDir) {
  StringBuilder builder = new StringBuilder();
  builder.append(part.getLocation());
  builder.append(Path.SEPARATOR);
  builder
    .append((FileUtils.makeDefaultListBucketingDirName(
      part.getSkewedColNames(),
      ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)));
  selectedPaths.add(new Path(builder.toString()));
 }
}

代码示例来源：origin: apache/hive

List<List<String>> uniqSkewedValues) throws SemanticException {
List<String> skewedCols = part.getSkewedColNames(); // Retrieve skewed column.

代码示例来源：origin: apache/drill

List<List<String>> uniqSkewedValues) throws SemanticException {
List<String> skewedCols = part.getSkewedColNames(); // Retrieve skewed column.

代码示例来源：origin: apache/drill

lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(),
  part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories(), conf);

代码示例来源：origin: apache/hive

lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(),
  part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories());

代码示例来源：origin: apache/hive

inputFormatClass = part.getInputFormatClass();
 isArchived = ArchiveUtils.isArchived(part);
 lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(),
   part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories());
 isListBucketed = part.isStoredAsSubDirectories();
 listBucketColNames = part.getSkewedColNames();
} else {

代码示例来源：origin: apache/drill

inputFormatClass = part.getInputFormatClass();
 isArchived = ArchiveUtils.isArchived(part);
 lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(),
   part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories(), conf);
 isListBucketed = part.isStoredAsSubDirectories();
 listBucketColNames = part.getSkewedColNames();
} else {

代码示例来源：origin: apache/drill

destTableId++;
lbCtx = constructListBucketingCtx(dest_part.getSkewedColNames(),
  dest_part.getSkewedColValues(), dest_part.getSkewedColValueLocationMaps(),
  dest_part.isStoredAsSubDirectories(), conf);

代码示例来源：origin: apache/hive

destTableId++;
lbCtx = constructListBucketingCtx(destinationPartition.getSkewedColNames(),
  destinationPartition.getSkewedColValues(), destinationPartition.getSkewedColValueLocationMaps(),
  destinationPartition.isStoredAsSubDirectories());

代码示例来源：origin: com.facebook.presto.hive/hive-apache

/**
  * Check if we prune it for list bucketing
  * 1. column name is part of skewed column
  * 2. partition has skewed value to location map
  * @param part
  * @param columnName
  * @return
  */
 private boolean isPruneForListBucketing(Partition part, String columnName) {
  return ListBucketingPrunerUtils.isListBucketingPart(part)
    && (part.getSkewedColNames().contains(columnName));
 }
}

代码示例来源：origin: com.facebook.presto.hive/hive-apache

/**
 * check if the partition is list bucketing
 *
 * @param part
 * @return
 */
public static boolean isListBucketingPart(Partition part) {
 return (part.getSkewedColNames() != null) && (part.getSkewedColNames().size() > 0)
   && (part.getSkewedColValues() != null) && (part.getSkewedColValues().size() > 0)
   && (part.getSkewedColValueLocationMaps() != null)
   && (part.getSkewedColValueLocationMaps().size() > 0);
}

代码示例来源：origin: com.facebook.presto.hive/hive-apache

/**
 * Decide whether should select the default directory.
 *
 * @param part
 * @param selectedPaths
 * @param nonSkewedValueMatchResult
 */
private static void decideDefaultDirSelection(Partition part, List<Path> selectedPaths,
  List<Boolean> nonSkewedValueMatchResult) {
 boolean skipDefDir = true;
 for (Boolean v : nonSkewedValueMatchResult) {
  if ((v == null) || v) {
   skipDefDir = false; // we skip default directory only if all value is false
   break;
  }
 }
 if (!skipDefDir) {
  StringBuilder builder = new StringBuilder();
  builder.append(part.getLocation());
  builder.append(Path.SEPARATOR);
  builder
    .append((FileUtils.makeDefaultListBucketingDirName(
      part.getSkewedColNames(),
      ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)));
  selectedPaths.add(new Path(builder.toString()));
 }
}

代码示例来源：origin: com.facebook.presto.hive/hive-apache

List<List<String>> uniqSkewedValues) throws SemanticException {
List<String> skewedCols = part.getSkewedColNames(); // Retrieve skewed column.

代码示例来源：origin: com.facebook.presto.hive/hive-apache

lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(),
  part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories(), conf);

代码示例来源：origin: com.facebook.presto.hive/hive-apache

inputFormatClass = part.getInputFormatClass();
 isArchived = ArchiveUtils.isArchived(part);
 lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(),
   part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories(), conf);
 isListBucketed = part.isStoredAsSubDirectories();
 listBucketColNames = part.getSkewedColNames();
} else {

内容来源于网络，如有侵权，请联系作者删除！

Partition

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多

    Partition类方法

    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com