org.apache.hadoop.hive.ql.exec.Utilities.getTezTasks()方法的使用及代码示例

x33g5p2x  于2022-02-01 转载在 其他  
字(8.4k)|赞(0)|评价(0)|浏览(61)

本文整理了Java中org.apache.hadoop.hive.ql.exec.Utilities.getTezTasks()方法的一些代码示例，展示了Utilities.getTezTasks()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Utilities.getTezTasks()方法的具体详情如下：
包路径：org.apache.hadoop.hive.ql.exec.Utilities
类名称：Utilities
方法名：getTezTasks

Utilities.getTezTasks介绍

暂无

代码示例

代码示例来源：origin: apache/drill

public static List<TezTask> getTezTasks(List<Task<? extends Serializable>> tasks) {
 List<TezTask> tezTasks = new ArrayList<TezTask>();
 if (tasks != null) {
  getTezTasks(tasks, tezTasks);
 }
 return tezTasks;
}

代码示例来源：origin: apache/hive

public static int getNumClusterJobs(List<Task<? extends Serializable>> tasks) {
 return getMRTasks(tasks).size() + getTezTasks(tasks).size() + getSparkTasks(tasks).size();
}

代码示例来源：origin: apache/drill

private static void getTezTasks(List<Task<? extends Serializable>> tasks, List<TezTask> tezTasks) {
 for (Task<? extends Serializable> task : tasks) {
  if (task instanceof TezTask && !tezTasks.contains(task)) {
   tezTasks.add((TezTask) task);
  }
  if (task.getDependentTasks() != null) {
   getTezTasks(task.getDependentTasks(), tezTasks);
  }
 }
}

代码示例来源：origin: apache/hive

protected ExecutionMode getExecutionMode(QueryPlan plan) {
 int numMRJobs = Utilities.getMRTasks(plan.getRootTasks()).size();
 int numSparkJobs = Utilities.getSparkTasks(plan.getRootTasks()).size();
 int numTezJobs = Utilities.getTezTasks(plan.getRootTasks()).size();
 ExecutionMode mode = ExecutionMode.MR;
 if (0 == (numMRJobs + numSparkJobs + numTezJobs)) {
  mode = ExecutionMode.NONE;
 } else if (numSparkJobs > 0) {
  return ExecutionMode.SPARK;
 } else if (numTezJobs > 0) {
  mode = ExecutionMode.TEZ;
  // Need to go in and check if any of the tasks is running in LLAP mode.
  for (TezTask tezTask : Utilities.getTezTasks(plan.getRootTasks())) {
   if (tezTask.getWork().getLlapMode()) {
    mode = ExecutionMode.LLAP;
    break;
   }
  }
 }
 return mode;
}

代码示例来源：origin: apache/hive

private ApplicationId determineLlapId(final HiveConf conf, QueryPlan plan) throws IOException {
  // Note: for now, LLAP is only supported in Tez tasks. Will never come to MR; others may
  //       be added here, although this is only necessary to have extra debug information.
  for (TezTask tezTask : Utilities.getTezTasks(plan.getRootTasks())) {
   if (!tezTask.getWork().getLlapMode()) continue;
   // In HS2, the client should have been cached already for the common case.
   // Otherwise, this may actually introduce delay to compilation for the first query.
   String hosts = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS);
   if (hosts != null && !hosts.isEmpty()) {
    ApplicationId llapId = LlapRegistryService.getClient(conf).getApplicationId();
    LOG.info("The query will use LLAP instance " + llapId + " (" + hosts + ")");
    return llapId;
   } else {
    LOG.info("Cannot determine LLAP instance on client - service hosts are not set");
    return null;
   }
  }
  return null;
 }
}

代码示例来源：origin: apache/drill

protected ExecutionMode getExecutionMode(QueryPlan plan) {
 int numMRJobs = Utilities.getMRTasks(plan.getRootTasks()).size();
 int numSparkJobs = Utilities.getSparkTasks(plan.getRootTasks()).size();
 int numTezJobs = Utilities.getTezTasks(plan.getRootTasks()).size();
 ExecutionMode mode = ExecutionMode.MR;
 if (0 == (numMRJobs + numSparkJobs + numTezJobs)) {
  mode = ExecutionMode.NONE;
 } else if (numSparkJobs > 0) {
  return ExecutionMode.SPARK;
 } else if (numTezJobs > 0) {
  mode = ExecutionMode.TEZ;
  // Need to go in and check if any of the tasks is running in LLAP mode.
  for (TezTask tezTask : Utilities.getTezTasks(plan.getRootTasks())) {
   if (tezTask.getWork().getLlapMode()) {
    mode = ExecutionMode.LLAP;
    break;
   }
  }
 }
 return mode;
}

代码示例来源：origin: apache/drill

private ApplicationId determineLlapId(final HiveConf conf, QueryPlan plan) throws IOException {
  // Note: for now, LLAP is only supported in Tez tasks. Will never come to MR; others may
  //       be added here, although this is only necessary to have extra debug information.
  for (TezTask tezTask : Utilities.getTezTasks(plan.getRootTasks())) {
   if (!tezTask.getWork().getLlapMode()) continue;
   // In HS2, the client should have been cached already for the common case.
   // Otherwise, this may actually introduce delay to compilation for the first query.
   String hosts = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS);
   if (hosts != null && !hosts.isEmpty()) {
    ApplicationId llapId = LlapRegistryService.getClient(conf).getApplicationId();
    LOG.info("The query will use LLAP instance " + llapId + " (" + hosts + ")");
    return llapId;
   } else {
    LOG.info("Cannot determine LLAP instance on client - service hosts are not set");
    return null;
   }
  }
  return null;
 }
}

代码示例来源：origin: apache/hive

List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
for (TezTask tezTask : rootTasks) {
 LOG.info("Printing ORC row group counter for tez task: " + tezTask.getName());

代码示例来源：origin: apache/drill

List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
for (TezTask tezTask : rootTasks) {
 LOG.info("Printing ORC row group counter for tez task: " + tezTask.getName());

代码示例来源：origin: apache/hive

@Override
public void run(HookContext hookContext) throws Exception {
 assert (hookContext.getHookType() == HookContext.HookType.POST_EXEC_HOOK ||
  hookContext.getHookType() == HookContext.HookType.ON_FAILURE_HOOK);
 HiveConf conf = hookContext.getConf();
 if (!"tez".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE))) {
  return;
 }
 LOG.info("Executing post execution hook to print workload manager events summary..");
 SessionState.LogHelper console = SessionState.getConsole();
 QueryPlan plan = hookContext.getQueryPlan();
 if (plan == null) {
  return;
 }
 List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
 for (TezTask tezTask : rootTasks) {
  WmContext wmContext = tezTask.getDriverContext().getCtx().getWmContext();
  if (wmContext != null) {
   wmContext.printJson(console);
   wmContext.shortPrint(console);
  }
 }
}

代码示例来源：origin: apache/hive

List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
for (TezTask tezTask : rootTasks) {
 LOG.info("Printing summary for tez task: " + tezTask.getName());

代码示例来源：origin: apache/drill

List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
for (TezTask tezTask : rootTasks) {
 LOG.info("Printing summary for tez task: " + tezTask.getName());

代码示例来源：origin: apache/hive

List<TezTask> tezTasks = Utilities.getTezTasks(plan.getRootTasks());
ExecutionMode executionMode = getExecutionMode(plan, mrTasks, tezTasks);

代码示例来源：origin: apache/hive

List<TezTask> tezTasks = Utilities.getTezTasks(tasks);
if (!tezTasks.isEmpty()) {
 for (TezTask tezTask : tezTasks) {

代码示例来源：origin: apache/hive

List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
for (TezTask tezTask : rootTasks) {
 List<BaseWork> baseWorks = tezTask.getWork().getAllWork();

代码示例来源：origin: apache/hive

int numTezJobs = Utilities.getTezTasks(plan.getRootTasks()).size();
if (numMrJobs + numTezJobs <= 0) {
 return; // ignore client only queries

代码示例来源：origin: apache/drill

int numTezJobs = Utilities.getTezTasks(plan.getRootTasks()).size();
if (numMrJobs + numTezJobs <= 0) {
 return; // ignore client only queries

代码示例来源：origin: apache/drill

int jobs = mrJobs + Utilities.getTezTasks(plan.getRootTasks()).size()
  + Utilities.getSparkTasks(plan.getRootTasks()).size();
if (jobs > 0) {

代码示例来源：origin: apache/hive

int jobs = mrJobs + Utilities.getTezTasks(plan.getRootTasks()).size()
  + Utilities.getSparkTasks(plan.getRootTasks()).size();
if (jobs > 0) {

代码示例来源：origin: apache/hive

/**
 * This test tests that Utilities.get*Tasks do not repeat themselves in the process
 * of extracting tasks from a given set of root tasks when given DAGs that can have
 * multiple paths, such as the case with Diamond-shaped DAGs common to replication.
 */
@Test
public void testGetTasksHaveNoRepeats() {
 CountingWrappingTask mrTask = new CountingWrappingTask(new ExecDriver());
 CountingWrappingTask tezTask = new CountingWrappingTask(new TezTask());
 CountingWrappingTask sparkTask = new CountingWrappingTask(new SparkTask());
 // First check - we should not have repeats in results
 assertEquals("No repeated MRTasks from Utilities.getMRTasks", 1,
   Utilities.getMRTasks(getTestDiamondTaskGraph(mrTask)).size());
 assertEquals("No repeated TezTasks from Utilities.getTezTasks", 1,
   Utilities.getTezTasks(getTestDiamondTaskGraph(tezTask)).size());
 assertEquals("No repeated TezTasks from Utilities.getSparkTasks", 1,
   Utilities.getSparkTasks(getTestDiamondTaskGraph(sparkTask)).size());
 // Second check - the tasks we looked for must not have been accessed more than
 // once as a result of the traversal (note that we actually wind up accessing
 // 2 times , because each visit counts twice, once to check for existence, and
 // once to visit.
 assertEquals("MRTasks should have been visited only once", 2, mrTask.getDepCallCount());
 assertEquals("TezTasks should have been visited only once", 2, tezTask.getDepCallCount());
 assertEquals("SparkTasks should have been visited only once", 2, sparkTask.getDepCallCount());
}

内容来源于网络，如有侵权，请联系作者删除！

utilities

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多

    Utilities类方法

    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com