org.apache.hadoop.hive.ql.exec.Utilities.getMergeWork()方法的使用及代码示例

x33g5p2x  于2022-02-01 转载在 其他  
字(6.4k)|赞(0)|评价(0)|浏览(130)

本文整理了Java中org.apache.hadoop.hive.ql.exec.Utilities.getMergeWork()方法的一些代码示例，展示了Utilities.getMergeWork()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Utilities.getMergeWork()方法的具体详情如下：
包路径：org.apache.hadoop.hive.ql.exec.Utilities
类名称：Utilities
方法名：getMergeWork

Utilities.getMergeWork介绍

暂无

代码示例

代码示例来源：origin: apache/hive

@Override
 public Object call() {
  return Utilities.getMergeWork(jconf, prefix);
 }
}));

代码示例来源：origin: apache/hive

@Override
 public Object call() {
  return Utilities.getMergeWork(jconf, prefix);
 }
}));

代码示例来源：origin: apache/drill

@Override
 public Object call() {
  return Utilities.getMergeWork(jconf, prefix);
 }
}));

代码示例来源：origin: apache/drill

@Override
 public Object call() {
  return Utilities.getMergeWork(jconf, prefix);
 }
}));

代码示例来源：origin: apache/hive

public static BaseWork getMergeWork(Configuration jconf) {
 String currentMergePrefix = jconf.get(DagUtils.TEZ_MERGE_CURRENT_MERGE_FILE_PREFIX);
 if (StringUtils.isEmpty(currentMergePrefix)) {
  return null;
 }
 return getMergeWork(jconf, jconf.get(DagUtils.TEZ_MERGE_CURRENT_MERGE_FILE_PREFIX));
}

代码示例来源：origin: apache/drill

public static BaseWork getMergeWork(Configuration jconf) {
 if ((jconf.get(DagUtils.TEZ_MERGE_CURRENT_MERGE_FILE_PREFIX) == null)
   || (jconf.get(DagUtils.TEZ_MERGE_CURRENT_MERGE_FILE_PREFIX).isEmpty())) {
  return null;
 }
 return getMergeWork(jconf, jconf.get(DagUtils.TEZ_MERGE_CURRENT_MERGE_FILE_PREFIX));
}

代码示例来源：origin: apache/hive

private static MapWork populateMapWork(JobConf jobConf, String inputName) {
 MapWork work = null;
 if (inputName != null) {
  work = (MapWork) Utilities.getMergeWork(jobConf, inputName);
  // work can still be null if there is no merge work for this input
 }
 if (work == null) {
  work = Utilities.getMapWork(jobConf);
 }
 return work;
}

代码示例来源：origin: apache/drill

private static MapWork populateMapWork(JobConf jobConf, String inputName) {
 MapWork work = null;
 if (inputName != null) {
  work = (MapWork) Utilities.getMergeWork(jobConf, inputName);
  // work can still be null if there is no merge work for this input
 }
 if (work == null) {
  work = Utilities.getMapWork(jobConf);
 }
 return work;
}

代码示例来源：origin: apache/hive

private static MapWork findMapWork(JobConf job) throws HiveException {
 String inputName = job.get(Utilities.INPUT_NAME, null);
 if (LOG.isDebugEnabled()) {
  LOG.debug("Initializing for input " + inputName);
 }
 String prefixes = job.get(DagUtils.TEZ_MERGE_WORK_FILE_PREFIXES);
 if (prefixes != null && !StringUtils.isBlank(prefixes)) {
  // Currently SMB is broken, so we cannot check if it's  compatible with IO elevator.
  // So, we don't use the below code that would get the correct MapWork. See HIVE-16985.
  return null;
 }
 BaseWork work = null;
 // HIVE-16985: try to find the fake merge work for SMB join, that is really another MapWork.
 if (inputName != null) {
  if (prefixes == null ||
    !Lists.newArrayList(prefixes.split(",")).contains(inputName)) {
   inputName = null;
  }
 }
 if (inputName != null) {
  work = Utilities.getMergeWork(job, inputName);
 }
 if (work == null || !(work instanceof MapWork)) {
  work = Utilities.getMapWork(job);
 }
 return (MapWork) work;
}

代码示例来源：origin: apache/hive

protected void init(JobConf job) {
 if (mrwork == null || pathToPartitionInfo == null) {
  if (HiveConf.getVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
   mrwork = (MapWork) Utilities.getMergeWork(job);
   if (mrwork == null) {
    mrwork = Utilities.getMapWork(job);
   }
  } else {
   mrwork = Utilities.getMapWork(job);
  }
  pathToPartitionInfo = mrwork.getPathToPartitionInfo();
 }
}

代码示例来源：origin: apache/drill

protected void init(JobConf job) {
 if (mrwork == null || pathToPartitionInfo == null) {
  if (HiveConf.getVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
   mrwork = (MapWork) Utilities.getMergeWork(job);
   if (mrwork == null) {
    mrwork = Utilities.getMapWork(job);
   }
  } else {
   mrwork = Utilities.getMapWork(job);
  }
  // Prune partitions
  if (HiveConf.getVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")
    && HiveConf.getBoolVar(job, HiveConf.ConfVars.SPARK_DYNAMIC_PARTITION_PRUNING)) {
   SparkDynamicPartitionPruner pruner = new SparkDynamicPartitionPruner();
   try {
    pruner.prune(mrwork, job);
   } catch (Exception e) {
    throw new RuntimeException(e);
   }
  }
  pathToPartitionInfo = mrwork.getPathToPartitionInfo();
 }
}

代码示例来源：origin: apache/hive

MapWork mrwork;
if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
 mrwork = (MapWork) Utilities.getMergeWork(jobConf);
 if (mrwork == null) {
  mrwork = Utilities.getMapWork(jobConf);

代码示例来源：origin: apache/drill

MapWork mrwork;
if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
 mrwork = (MapWork) Utilities.getMergeWork(jobConf);
 if (mrwork == null) {
  mrwork = Utilities.getMapWork(jobConf);

代码示例来源：origin: com.facebook.presto.hive/hive-apache

@Override
 public Object call() {
  return Utilities.getMergeWork(jconf, prefix);
 }
}));

代码示例来源：origin: com.facebook.presto.hive/hive-apache

public static BaseWork getMergeWork(JobConf jconf) {
 if ((jconf.get(DagUtils.TEZ_MERGE_CURRENT_MERGE_FILE_PREFIX) == null)
   || (jconf.get(DagUtils.TEZ_MERGE_CURRENT_MERGE_FILE_PREFIX).isEmpty())) {
  return null;
 }
 return getMergeWork(jconf, jconf.get(DagUtils.TEZ_MERGE_CURRENT_MERGE_FILE_PREFIX));
}

代码示例来源：origin: com.facebook.presto.hive/hive-apache

private static MapWork populateMapWork(JobConf jobConf, String inputName) {
 MapWork work = null;
 if (inputName != null) {
  work = (MapWork) Utilities.getMergeWork(jobConf, inputName);
  // work can still be null if there is no merge work for this input
 }
 if (work == null) {
  work = Utilities.getMapWork(jobConf);
 }
 return work;
}

代码示例来源：origin: org.apache.hive/hive-llap-server

private static MapWork findMapWork(JobConf job) throws HiveException {
 String inputName = job.get(Utilities.INPUT_NAME, null);
 if (LOG.isDebugEnabled()) {
  LOG.debug("Initializing for input " + inputName);
 }
 String prefixes = job.get(DagUtils.TEZ_MERGE_WORK_FILE_PREFIXES);
 if (prefixes != null && !StringUtils.isBlank(prefixes)) {
  // Currently SMB is broken, so we cannot check if it's  compatible with IO elevator.
  // So, we don't use the below code that would get the correct MapWork. See HIVE-16985.
  return null;
 }
 BaseWork work = null;
 // HIVE-16985: try to find the fake merge work for SMB join, that is really another MapWork.
 if (inputName != null) {
  if (prefixes == null ||
    !Lists.newArrayList(prefixes.split(",")).contains(inputName)) {
   inputName = null;
  }
 }
 if (inputName != null) {
  work = Utilities.getMergeWork(job, inputName);
 }
 if (work == null || !(work instanceof MapWork)) {
  work = Utilities.getMapWork(job);
 }
 return (MapWork) work;
}

代码示例来源：origin: com.facebook.presto.hive/hive-apache

protected void init(JobConf job) {
 if (mrwork == null || pathToPartitionInfo == null) {
  if (HiveConf.getVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
   mrwork = (MapWork) Utilities.getMergeWork(job);
   if (mrwork == null) {
    mrwork = Utilities.getMapWork(job);
   }
  } else {
   mrwork = Utilities.getMapWork(job);
  }
  pathToPartitionInfo = mrwork.getPathToPartitionInfo();
 }
}

内容来源于网络，如有侵权，请联系作者删除！

utilities

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多

    Utilities类方法

    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com