我提交有问题 MapReduce java中的远程作业。我有一个cdh集群，我复制了 mapred-site.xml , yarn-site.xml , core-site.xml 以及 hdfs-site.xml 我的创意项目。但我发现它总是 .staging 目录，所以我认为作业已提交到 localhost 即使没有 hadoop 环境 localhost .

package com.mule.cell.basic;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class WordCount {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        final String INPUT_PATH = "/tmp/test.txt";
        final String OUTPUT_PATH = "/tmp/output";

        System.setProperty("HADOOP_USER_NAME", "hdfs");

        ClassLoader classloader = Thread.currentThread().getContextClassLoader();

        Configuration conf = new Configuration();
        conf.addResource(classloader.getResource("yarn-site.xml"));
        conf.addResource(classloader.getResource("mapred-site.xml"));
        conf.addResource(classloader.getResource("hdfs-site.xml"));
        conf.addResource(classloader.getResource("core-site.xml"));

        Job job = Job.getInstance(conf, "yuri");
        job.setMapperClass(com.mule.cell.basic.WordCountMap.class);
        job.setCombinerClass(com.mule.cell.basic.WordCountReduce.class);
        job.setReducerClass(com.mule.cell.basic.WordCountReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.setJar("C:\\Users\\aoding\\workspace\\alextinng\\cell\\mapreduce\\target\\mapreduce-1.0-SNAPSHOT.    jar");
        job.setNumReduceTasks(1);

        FileInputFormat.addInputPath(job, new Path(INPUT_PATH));
        FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

但如果我配置 Configuration 对象，作业已提交到远程群集并成功完成。

package com.mule.cell.basic;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class WordCount {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        final String INPUT_PATH = "/tmp/test.txt";
        final String OUTPUT_PATH = "/tmp/output";

        System.setProperty("HADOOP_USER_NAME", "hdfs");

        ClassLoader classloader = Thread.currentThread().getContextClassLoader();

        Configuration conf = new Configuration();
        conf.set("yarn.resourcemanager.address", "xxxxx:8032");
        conf.set("mapreduce.framework.name", "yarn");
        conf.set("fs.default.name", "hdfs://xxxxxx:8020");
        conf.set("mapreduce.app-submission.cross-platform", "true");

        Job job = Job.getInstance(conf, "yuri");
        job.setMapperClass(com.mule.cell.basic.WordCountMap.class);
        job.setCombinerClass(com.mule.cell.basic.WordCountReduce.class);
        job.setReducerClass(com.mule.cell.basic.WordCountReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.setJar("C:\\Users\\aoding\\workspace\\alextinng\\cell\\mapreduce\\target\\mapreduce-1.0-SNAPSHOT.    jar");
        job.setNumReduceTasks(1);

        FileInputFormat.addInputPath(job, new Path(INPUT_PATH));
        FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}