本文整理了Java中org.apache.hadoop.hive.ql.exec.Utilities类的一些代码示例，展示了Utilities类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Utilities类的具体详情如下：
包路径：org.apache.hadoop.hive.ql.exec.Utilities
类名称：Utilities

Utilities介绍

[英]Utilities.
[中]公用事业

代码示例

代码示例来源：origin: apache/hive

@Override
public void configure(JobConf job) {
 jc = job;
 work = (ColumnTruncateWork) Utilities.getMapWork(job);
 Path specPath = work.getOutputDir();
 Path tmpPath = Utilities.toTempPath(specPath);
 Path taskTmpPath = Utilities.toTaskTempPath(specPath);
 updatePaths(tmpPath, taskTmpPath);
 try {
  fs = specPath.getFileSystem(job);
  autoDelete = fs.deleteOnExit(outPath);
 } catch (IOException e) {
  this.exception = true;
  throw new RuntimeException(e);
 }
}

代码示例来源：origin: apache/hive

private Class<? extends RecordReader> getDefaultRecordReader()
  throws SemanticException {
 String name;
 name = conf.getVar(HiveConf.ConfVars.HIVESCRIPTRECORDREADER);
 try {
  return (Class<? extends RecordReader>) Class.forName(name, true,
    Utilities.getSessionSpecifiedClassLoader());
 } catch (ClassNotFoundException e) {
  throw new SemanticException(e);
 }
}

代码示例来源：origin: apache/drill

private static String[] getTempFilesFromConf(Configuration conf) {
 String addedFiles = Utilities.getResourceFiles(conf, SessionState.ResourceType.FILE);
 if (StringUtils.isNotBlank(addedFiles)) {
  HiveConf.setVar(conf, ConfVars.HIVEADDEDFILES, addedFiles);
 }
 String addedJars = Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR);
 if (StringUtils.isNotBlank(addedJars)) {
  HiveConf.setVar(conf, ConfVars.HIVEADDEDJARS, addedJars);
 }
 String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS);
 // need to localize the additional jars and files
 // we need the directory on hdfs to which we shall put all these files
 String allFiles = auxJars + "," + addedJars + "," + addedFiles;
 return allFiles.split(",");
}

代码示例来源：origin: apache/hive

private static MapWork populateMapWork(JobConf jobConf, String inputName) {
 MapWork work = null;
 if (inputName != null) {
  work = (MapWork) Utilities.getMergeWork(jobConf, inputName);
  // work can still be null if there is no merge work for this input
 }
 if (work == null) {
  work = Utilities.getMapWork(jobConf);
 }
 return work;
}

代码示例来源：origin: apache/drill

private void updatePaths(Path tp, Path ttp) {
 String taskId = Utilities.getTaskId(jc);
 tmpPath = tp;
 taskTmpPath = ttp;
 finalPath = new Path(tp, taskId);
 outPath = new Path(ttp, Utilities.toTempPath(taskId));
}

代码示例来源：origin: apache/hive

protected void init(JobConf job) {
 if (mrwork == null || pathToPartitionInfo == null) {
  if (HiveConf.getVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
   mrwork = (MapWork) Utilities.getMergeWork(job);
   if (mrwork == null) {
    mrwork = Utilities.getMapWork(job);
   }
  } else {
   mrwork = Utilities.getMapWork(job);
  }
  pathToPartitionInfo = mrwork.getPathToPartitionInfo();
 }
}

代码示例来源：origin: apache/drill

if ((!conf.getBoolVar(HiveConf.ConfVars.HIVE_INSERT_INTO_EXTERNAL_TABLES)) &&
   (dest_tab.getTableType().equals(TableType.EXTERNAL_TABLE))) {
  throw new SemanticException(
   dest_path = new Path(dest_tab.getPath(), dpCtx.getSPPath());
 table_desc = Utilities.getTableDesc(dest_tab);
    outputs.add(output);
   } catch (HiveException e) {
    throw new SemanticException(e.getMessage(), e);
 dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri()
   .getAuthority(), partPath.toUri().getPath());
 table_desc = Utilities.getTableDesc(dest_tab);
 dest_path);
boolean isHiveServerQuery = SessionState.get().isHiveServerQuery();
fileSinkDesc.setHiveServerQuery(isHiveServerQuery);
 String tName = Utilities.getDbTableName(tableDesc.getTableName())[1];
 try {
  Warehouse wh = new Warehouse(conf);

代码示例来源：origin: apache/hive

String hiveJar = conf.getJar();
String hadoopExec = conf.getVar(HiveConf.ConfVars.HADOOPBIN);
conf.setVar(ConfVars.HIVEADDEDJARS, Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR));
Path planPath = new Path(ctx.getLocalTmpPath(), "plan.xml");
MapredLocalWork plan = getWork();
LOG.info("Generating plan file " + planPath.toString());
 out = FileSystem.getLocal(conf).create(planPath);
 SerializationUtilities.serializePlan(plan, out);
 out.close();
String cmdLine = hadoopExec + " jar " + jarCmd + " -localtask -plan " + planPath.toString()
  + " " + isSilent + " " + hiveConfArgs;
String files = Utilities.getResourceFiles(conf, SessionState.ResourceType.FILE);
 workDir = ctx.getLocalTmpPath().toUri().getPath();
  Path p = new Path(f);
  String target = p.toUri().getPath();
  String link = workDir + Path.SEPARATOR + p.getName();
  if (FileUtil.symLink(target, link) != 0) {
 if (SessionState.get() != null) {
  SessionState.get().addLocalMapRedErrors(getId(), errPrintStream.getOutput());

代码示例来源：origin: apache/hive

Path path = getPlanPath(conf, name);
LOG.debug("PLAN PATH = {}", path);
if (path == null) { // Map/reduce plan may not be generated
Kryo kryo = SerializationUtilities.borrowKryo();
try {
 String engine = HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE);
 if (engine.equals("spark")) {
  if (StringUtils.isNotEmpty(addedJars)) {
   ClassLoader loader = Thread.currentThread().getContextClassLoader();
   ClassLoader newLoader = addToClassPath(loader, addedJars.split(";"));
   Thread.currentThread().setContextClassLoader(newLoader);
   kryo.setClassLoader(newLoader);
 final long serializedSize;
 final String planMode;
 if (HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) {
  String planStringPath = path.toUri().getPath();
  LOG.debug("Loading plan from string: {}", planStringPath);
  String planString = conf.getRaw(planStringPath);
   gWork.getName(), humanReadableByteCount(serializedSize));
 gWorkMap.get(conf).put(path, gWork);
 return gWork;

代码示例来源：origin: apache/drill

public JobConf pushProjectionsAndFilters(JobConf jobConf, Path path)
   throws IOException {
  updateMrWork(jobConf);  // TODO: refactor this in HIVE-6366
  final JobConf cloneJobConf = new JobConf(jobConf);
  final PartitionDesc part = pathToPartitionInfo.get(path);

  if ((part != null) && (part.getTableDesc() != null)) {
   Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), cloneJobConf);
  }
  pushProjectionsAndFilters(cloneJobConf, path.toString(), path.toUri().getPath());
  return cloneJobConf;
 }
}

代码示例来源：origin: com.twitter/parquet-hive-0.10-binding

/**
 * Initialize the mrwork variable in order to get all the partition and start to update the jobconf
 *
 * @param job
 */
private void init(final JobConf job) {
 final String plan = HiveConf.getVar(job, HiveConf.ConfVars.PLAN);
 if (mrwork == null && plan != null && plan.length() > 0) {
  mrwork = Utilities.getMapRedWork(job);
  pathToPartitionInfo.clear();
  for (final Map.Entry<String, PartitionDesc> entry : mrwork.getPathToPartitionInfo().entrySet()) {
   pathToPartitionInfo.put(new Path(entry.getKey()).toUri().getPath().toString(), entry.getValue());
  }
 }
}

代码示例来源：origin: apache/hive

if (abort) {
 if (!autoDelete || isMmTable) {
  fs.delete(outPath, true);
  Utilities.writeMmCommitManifest(Lists.newArrayList(outPath),
    tmpPath.getParent(), fs, taskId, conf.getWriteId(), conf.getStmtId(), null, false);
  LOG.info("Merged into " + finalPath + "(" + fss.getLen() + " bytes).");
 if (isMmTable) {
  throw new HiveException("Incompatible files should not happen in MM tables.");
 Path destDir = finalPath.getParent();
 Path destPath = destDir;
   if (!Utilities.isHiveManagedFile(incompatFile)) {
    final String taskId = Utilities.getTaskId(jc);
    Path destFilePath = new Path(destDir, new Path(taskId));
    for (int counter = 1; fs.exists(destFilePath); counter++) {
    Utilities.renameOrMoveFiles(fs, incompatFile, destPath);
    LOG.info("Moved incompatible file " + incompatFile + " to " + destPath);
   } catch (HiveException e) {
throw new HiveException("Failed to close AbstractFileMergeOperator", e);

代码示例来源：origin: apache/hive

conf = new JobConf(MapredLocalTask.class);
} else {
 conf = new JobConf(ExecDriver.class);
 conf.addResource(new Path(jobConfFileName));
boolean isSilent = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESESSIONSILENT);
String queryId = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID, "").trim();
if(queryId.isEmpty()) {
 queryId = "unknown-" + System.currentTimeMillis();
 HiveConf.setVar(conf, HiveConf.ConfVars.HIVEQUERYID, queryId);
Path p = new Path(planFileName);
FileSystem fs = FileSystem.getLocal(conf);
InputStream pathData = fs.open(p);
 ClassLoader loader = conf.getClassLoader();
 if (StringUtils.isNotBlank(libjars)) {
  loader = Utilities.addToClassPath(loader, StringUtils.split(libjars, ","));
 conf.setClassLoader(loader);
 throw new HiveException(e.getMessage(), e);

代码示例来源：origin: apache/hive

job.setMapOutputKeyClass(NullWritable.class);
job.setMapOutputValueClass(NullWritable.class);
if(work.getNumMapTasks() != null) {
 HiveConf.setLongVar(job, HiveConf.ConfVars.MAPREDMINSPLITSIZE, work
   .getMinSplitSize().longValue());
 HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT, work
   .getInputformat());
Path tempOutPath = Utilities.toTempPath(outputPath);
try {
 FileSystem fs = tempOutPath.getFileSystem(job);
 if (!fs.exists(tempOutPath)) {
  fs.mkdirs(tempOutPath);
if (noName && this.getQueryPlan() != null) {
 int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
 jobName = Utilities.abbreviate(this.getQueryPlan().getQueryStr(),
   maxlen - 6);
 Utilities.setMapRedWork(job, mrWork, ctx.getMRTmpPath());
 String addedJars = Utilities.getResourceFiles(job, SessionState.ResourceType.JAR);
 if (!addedJars.isEmpty()) {
  job.set("tmpjars", addedJars);

代码示例来源：origin: apache/hive

public void testAvoidSplitCombination() throws Exception {
  Configuration conf = new Configuration();
  JobConf job = new JobConf(conf);
  TableDesc tblDesc = Utilities.defaultTd;
  tblDesc.setInputFileFormatClass(TestSkipCombineInputFormat.class);
  PartitionDesc partDesc = new PartitionDesc(tblDesc, null);
  LinkedHashMap<Path, PartitionDesc> pt = new LinkedHashMap<>();
  pt.put(new Path("/tmp/testfolder1"), partDesc);
  pt.put(new Path("/tmp/testfolder2"), partDesc);
  MapredWork mrwork = new MapredWork();
  mrwork.getMapWork().setPathToPartitionInfo(pt);
  Path mapWorkPath = new Path("/tmp/" + System.getProperty("user.name"), "hive");
  Utilities.setMapRedWork(conf, mrwork,
    mapWorkPath);
  try {
    Path[] paths = new Path[2];
    paths[0] = new Path("/tmp/testfolder1");
    paths[1] = new Path("/tmp/testfolder2");
    CombineHiveInputFormat combineInputFormat =
      ReflectionUtils.newInstance(CombineHiveInputFormat.class, conf);
    combineInputFormat.pathToPartitionInfo =
      Utilities.getMapWork(conf).getPathToPartitionInfo();
    Set results = combineInputFormat.getNonCombinablePathIndices(job, paths, 2);
    assertEquals("Should have both path indices in the results set", 2, results.size());
  } finally {
    // Cleanup the mapwork path
    FileSystem.get(conf).delete(mapWorkPath, true);
  }
}

代码示例来源：origin: apache/hive

Path tempOutPath = Utilities.toTempPath(outputPath);
FileSystem fs = tempOutPath.getFileSystem(job);
if (!fs.exists(tempOutPath)) {
 fs.mkdirs(tempOutPath);
 int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
 jobName = Utilities.abbreviate(this.getQueryPlan().getQueryStr(),
   maxlen - 6);
Utilities.setMapWork(job, work, ctx.getMRTmpPath(), true);
String pwd = HiveConf.getVar(job, HiveConf.ConfVars.METASTOREPWD);
if (pwd != null) {
 HiveConf.setVar(job, HiveConf.ConfVars.METASTOREPWD, "HIVE");
String addedJars = Utilities.getResourceFiles(job,
  SessionState.ResourceType.JAR);
if (!addedJars.isEmpty()) {
String mesg = " with exception '" + Utilities.getNameMessage(e) + "'";
if (rj != null) {
 mesg = "Ended Job = " + rj.getJobID() + mesg;
  returnVal = 3;
  String mesg = "Job Commit failed with exception '" +
    Utilities.getNameMessage(e) + "'";
  console.printError(mesg, "\n" +
    org.apache.hadoop.util.StringUtils.stringifyException(e));

代码示例来源：origin: apache/hive

private void testTaskIds(String [] taskIds, String expectedAttemptId, String expectedTaskId) {
 Configuration conf = new JobConf(TestOperators.class);
 for (String one: taskIds) {
  conf.set("mapred.task.id", one);
  String attemptId = Utilities.getTaskId(conf);
  assertEquals(expectedAttemptId, attemptId);
  assertEquals(Utilities.getTaskIdFromFilename(attemptId), expectedTaskId);
  assertEquals(Utilities.getTaskIdFromFilename(attemptId + ".gz"), expectedTaskId);
  assertEquals(Utilities.getTaskIdFromFilename
         (Utilities.toTempPath(new Path(attemptId + ".gz")).toString()), expectedTaskId);
 }
}

代码示例来源：origin: apache/hive

private void updatePaths(Path tmpPath, Path taskTmpPath) {
 String taskId = Utilities.getTaskId(jc);
 this.tmpPath = tmpPath;
 this.taskTmpPath = taskTmpPath;
 String inputFile = jc.get(MRJobConfig.MAP_INPUT_FILE);
 int lastSeparator = inputFile.lastIndexOf(Path.SEPARATOR) + 1;
 finalPath = new Path(tmpPath, inputFile.substring(lastSeparator));
 outPath = new Path(taskTmpPath, Utilities.toTempPath(taskId));
}

代码示例来源：origin: apache/hive

aliasList.add("a");
LinkedHashMap<Path, ArrayList<String>> pa = new LinkedHashMap<>();
pa.put(new Path("/tmp/testfolder"), aliasList);
pt.put(new Path("/tmp/testfolder"), partDesc);
mrwork.getMapWork().setAliasToWork(ao);
JobConf job = new JobConf(TestPlan.class);
job.set("fs.default.name", "file:///");
Utilities.setMapRedWork(job, mrwork, new Path(System.getProperty("java.io.tmpdir") + File.separator +
 System.getProperty("user.name") + File.separator + "hive"));
MapredWork mrwork2 = Utilities.getMapRedWork(job);
Utilities.clearWork(job);

代码示例来源：origin: apache/hive

private Path setupTempDirWithSingleOutputFile(Configuration hconf) throws IOException {
 Path tempDirPath = new Path("file://" + temporaryFolder.newFolder().getAbsolutePath());
 Path taskOutputPath = new Path(tempDirPath, Utilities.getTaskId(hconf));
 FileSystem.getLocal(hconf).create(taskOutputPath).close();
 return tempDirPath;
}