org.apache.hadoop.hive.ql.exec.Utilities.getPlanPath()方法的使用及代码示例

x33g5p2x  于2022-02-01 转载在 其他  
字(7.4k)|赞(0)|评价(0)|浏览(106)

本文整理了Java中org.apache.hadoop.hive.ql.exec.Utilities.getPlanPath()方法的一些代码示例,展示了Utilities.getPlanPath()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Utilities.getPlanPath()方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.exec.Utilities
类名称:Utilities
方法名:getPlanPath

Utilities.getPlanPath介绍

暂无

代码示例

代码示例来源:origin: apache/hive

private static Path getPlanPath(Configuration conf, String name) {
 Path planPath = getPlanPath(conf);
 if (planPath == null) {
  return null;
 }
 return new Path(planPath, name);
}

代码示例来源:origin: apache/drill

private static Path getPlanPath(Configuration conf, String name) {
 Path planPath = getPlanPath(conf);
 if (planPath == null) {
  return null;
 }
 return new Path(planPath, name);
}

代码示例来源:origin: apache/hive

public static void clearWorkMapForConf(Configuration conf) {
 // Remove cached query plans for the current query only
 Path mapPath = getPlanPath(conf, MAP_PLAN_NAME);
 Path reducePath = getPlanPath(conf, REDUCE_PLAN_NAME);
 if (mapPath != null) {
  gWorkMap.get(conf).remove(mapPath);
 }
 if (reducePath != null) {
  gWorkMap.get(conf).remove(reducePath);
 }
 // TODO: should this also clean merge work?
}

代码示例来源:origin: apache/drill

private static void setPlanPath(Configuration conf, Path hiveScratchDir) throws IOException {
 if (getPlanPath(conf) == null) {
  // this is the unique conf ID, which is kept in JobConf as part of the plan file name
  String jobID = UUID.randomUUID().toString();
  Path planPath = new Path(hiveScratchDir, jobID);
  FileSystem fs = planPath.getFileSystem(conf);
  fs.mkdirs(planPath);
  HiveConf.setVar(conf, HiveConf.ConfVars.PLAN, planPath.toUri().toString());
 }
}

代码示例来源:origin: apache/hive

private static void setPlanPath(Configuration conf, Path hiveScratchDir) throws IOException {
 if (getPlanPath(conf) == null) {
  // this is the unique conf ID, which is kept in JobConf as part of the plan file name
  String jobID = UUID.randomUUID().toString();
  Path planPath = new Path(hiveScratchDir, jobID);
  if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) {
   FileSystem fs = planPath.getFileSystem(conf);
   // since we are doing RPC creating a directory is un-necessary
   fs.mkdirs(planPath);
  }
  HiveConf.setVar(conf, HiveConf.ConfVars.PLAN, planPath.toUri().toString());
 }
}

代码示例来源:origin: apache/drill

public static void clearWorkMapForConf(Configuration conf) {
 // Remove cached query plans for the current query only
 Path mapPath = getPlanPath(conf, MAP_PLAN_NAME);
 Path reducePath = getPlanPath(conf, REDUCE_PLAN_NAME);
 if (mapPath != null) {
  gWorkMap.get(conf).remove(mapPath);
 }
 if (reducePath != null) {
  gWorkMap.get(conf).remove(reducePath);
 }
 // TODO: should this also clean merge work?
}

代码示例来源:origin: apache/hive

public static void clearWork(Configuration conf) {
 Path mapPath = getPlanPath(conf, MAP_PLAN_NAME);
 Path reducePath = getPlanPath(conf, REDUCE_PLAN_NAME);
 // if the plan path hasn't been initialized just return, nothing to clean.
 if (mapPath == null && reducePath == null) {
  return;
 }
 try {
  FileSystem fs = mapPath.getFileSystem(conf);
  if (fs.exists(mapPath)) {
   fs.delete(mapPath, true);
  }
  if (fs.exists(reducePath)) {
   fs.delete(reducePath, true);
  }
 } catch (Exception e) {
  LOG.warn("Failed to clean-up tmp directories.", e);
 } finally {
  // where a single process works with multiple plans - we must clear
  // the cache before working with the next plan.
  clearWorkMapForConf(conf);
 }
}

代码示例来源:origin: apache/drill

public static void clearWork(Configuration conf) {
 Path mapPath = getPlanPath(conf, MAP_PLAN_NAME);
 Path reducePath = getPlanPath(conf, REDUCE_PLAN_NAME);
 // if the plan path hasn't been initialized just return, nothing to clean.
 if (mapPath == null && reducePath == null) {
  return;
 }
 try {
  FileSystem fs = mapPath.getFileSystem(conf);
  if (fs.exists(mapPath)) {
   fs.delete(mapPath, true);
  }
  if (fs.exists(reducePath)) {
   fs.delete(reducePath, true);
  }
 } catch (Exception e) {
  LOG.warn("Failed to clean-up tmp directories.", e);
 } finally {
  // where a single process works with multiple plans - we must clear
  // the cache before working with the next plan.
  clearWorkMapForConf(conf);
 }
}

代码示例来源:origin: apache/hive

/**
 * Pushes work into the global work map
 */
public static void setBaseWork(Configuration conf, String name, BaseWork work) {
 Path path = getPlanPath(conf, name);
 setHasWork(conf, name);
 gWorkMap.get(conf).put(path, work);
}

代码示例来源:origin: apache/drill

/**
 * Pushes work into the global work map
 */
public static void setBaseWork(Configuration conf, String name, BaseWork work) {
 Path path = getPlanPath(conf, name);
 setHasWork(conf, name);
 gWorkMap.get(conf).put(path, work);
}

代码示例来源:origin: apache/hive

/**
 * Returns true if a plan is both configured for vectorized execution
 * and the node is vectorized.
 *
 * The plan may be configured for vectorization
 * but vectorization disallowed eg. for FetchOperator execution.
 */
public static boolean getIsVectorized(Configuration conf) {
 if (conf.get(VECTOR_MODE) != null) {
  // this code path is necessary, because with HS2 and client
  // side split generation we end up not finding the map work.
  // This is because of thread local madness (tez split
  // generation is multi-threaded - HS2 plan cache uses thread
  // locals).
  return
    conf.getBoolean(VECTOR_MODE, false);
 } else {
  if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED) &&
   Utilities.getPlanPath(conf) != null) {
   MapWork mapWork = Utilities.getMapWork(conf);
   return mapWork.getVectorMode();
  } else {
   return false;
  }
 }
}

代码示例来源:origin: apache/drill

/**
 * Returns true if a plan is both configured for vectorized execution
 * and the node is vectorized and the Input File Format is marked VectorizedInputFileFormat.
 *
 * The plan may be configured for vectorization
 * but vectorization disallowed eg. for FetchOperator execution.
 */
public static boolean getUseVectorizedInputFileFormat(Configuration conf) {
 if (conf.get(VECTOR_MODE) != null) {
  // this code path is necessary, because with HS2 and client
  // side split generation we end up not finding the map work.
  // This is because of thread local madness (tez split
  // generation is multi-threaded - HS2 plan cache uses thread
  // locals).
  return
    conf.getBoolean(VECTOR_MODE, false) &&
    conf.getBoolean(USE_VECTORIZED_INPUT_FILE_FORMAT, false);
 } else {
  if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED) &&
   Utilities.getPlanPath(conf) != null) {
   MapWork mapWork = Utilities.getMapWork(conf);
   return (mapWork.getVectorMode() && mapWork.getUseVectorizedInputFileFormat());
  } else {
   return false;
  }
 }
}

代码示例来源:origin: apache/drill

path = getPlanPath(conf, name);
LOG.info("PLAN PATH = " + path);
if (path == null) { // Map/reduce plan may not be generated

代码示例来源:origin: apache/hive

Path path = getPlanPath(conf, name);
LOG.debug("PLAN PATH = {}", path);
if (path == null) { // Map/reduce plan may not be generated

代码示例来源:origin: apache/drill

/**
 * @param conf
 * @return the configured VectorizedRowBatchCtx for a MapWork task.
 */
public static VectorizedRowBatchCtx getVectorizedRowBatchCtx(Configuration conf) {
 VectorizedRowBatchCtx result = null;
 if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED) &&
   Utilities.getPlanPath(conf) != null) {
  MapWork mapWork = Utilities.getMapWork(conf);
  if (mapWork != null && mapWork.getVectorMode()) {
   result = mapWork.getVectorizedRowBatchCtx();
  }
 }
 return result;
}

代码示例来源:origin: apache/hive

/**
 * @param conf
 * @return the configured VectorizedRowBatchCtx for a MapWork task.
 */
public static VectorizedRowBatchCtx getVectorizedRowBatchCtx(Configuration conf) {
 VectorizedRowBatchCtx result = null;
 if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED) &&
   Utilities.getPlanPath(conf) != null) {
  MapWork mapWork = Utilities.getMapWork(conf);
  if (mapWork != null && mapWork.getVectorMode()) {
   result = mapWork.getVectorizedRowBatchCtx();
  }
 }
 return result;
}

代码示例来源:origin: apache/hive

setPlanPath(conf, hiveScratchDir);
Path planPath = getPlanPath(conf, name);
setHasWork(conf, name);

代码示例来源:origin: apache/drill

setPlanPath(conf, hiveScratchDir);
Path planPath = getPlanPath(conf, name);
setHasWork(conf, name);

代码示例来源:origin: apache/hive

&& (Utilities.getPlanPath(conf) != null);

代码示例来源:origin: apache/drill

&& (Utilities.getPlanPath(conf) != null);

相关文章

微信公众号

最新文章

更多

Utilities类方法