classnotfoundexception

x33g5p2x  于 2021-06-02  发布在  Hadoop
关注(0)|答案(2)|浏览(292)

我们目前正在尝试使用MapReduce作业和titan依赖项将一些文件从hdfs批量加载到titan中。然而,一旦map作业开始,我们就遇到了一个问题,它找不到tinkerpop类。这是错误:

java.lang.ClassNotFoundException: org.apache.tinkerpop.gremlin.structure.Vertex

我在某个地方读到,Titan1.0.0只与Tinkerpop3.0.1-Cubbing兼容,所以这就是我们的版本的依赖性。查看pom.xml和代码可能会有所帮助
pom.xml文件:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>replacementID</groupId>
  <artifactId>replacementID</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <dependencies>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.7.3</version>
    </dependency>
    <dependency>
        <groupId>com.thinkaurelius.titan</groupId>
        <artifactId>titan-hbase</artifactId>
        <version>1.0.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.tinkerpop</groupId>
        <artifactId>hadoop-gremlin</artifactId>
        <version>3.0.1-incubating</version>
    </dependency>
    <dependency>
        <groupId>org.apache.tinkerpop</groupId>
        <artifactId>gremlin-core</artifactId>
        <version>3.0.1-incubating</version>
    </dependency>
    <dependency>
        <groupId>org.apache.tinkerpop</groupId>
        <artifactId>gremlin-driver</artifactId>
        <version>3.0.1-incubating</version>
    </dependency>
  </dependencies>
</project>

Map器:

package edu.rosehulman.brubakbd;
import java.io.IOException;

import org.apache.commons.configuration.BaseConfiguration;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanVertex;
import org.apache.tinkerpop.gremlin.structure.Vertex;

public class TitanMRMapper extends Mapper<LongWritable, Text, Text, IntWritable>{

    @Override
    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{
        String line = value.toString();
        String[] vals = line.split("\t");

        BaseConfiguration conf = new BaseConfiguration();
                conf.setProperty("gremlin.graph", "com.thinkaurelius.titan.core.TitanFactory");
                conf.setProperty("storage.backend", "hbase");
                conf.setProperty("storage.hostname", "hadoop-16.csse.rose-hulman.edu");
                conf.setProperty("storage.batch-loading", true);
                conf.setProperty("storage.hbase.ext.zookeeper.znode.parent","/hbase-unsecure");
                conf.setProperty("storage.hbase.ext.hbase.zookeeper.property.clientPort", 2181);
                conf.setProperty("cache.db-cache",true);
                conf.setProperty("cache.db-cache-clean-wait", 20);
                conf.setProperty("cache.db-cache-time", 180000);
                conf.setProperty("cache.db-cache-size", 0.5);

        TitanGraph graph = TitanFactory.open(conf);
        TitanVertex v1 = graph.addVertex();
        v1.property("pageID", vals[0]);
        TitanVertex v2 = graph.addVertex();
        v2.property("pageID", vals[1]);

        v1.addEdge("links_To", v2);

        graph.tx().commit();
    }
}

司机:

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;

public class TitanMR {

    public static void main(String[] args) throws Exception{
        if (args.length != 1){
            System.err.println("Usage: TitanMR <input path>");
            System.exit(-1);
        }

        Job job = new Job();
        job.setJarByClass(TitanMR.class);
        job.setJobName("TitanMR");

        FileInputFormat.addInputPath(job, new Path(args[0]));
        job.setOutputFormatClass(NullOutputFormat.class);

        job.setMapperClass(TitanMRMapper.class);
        job.setNumReduceTasks(0);

        System.out.println("about to submit job");
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}
d4so4syb

d4so4syb1#

在pom.xml中升级gremlin jars

<dependency>
    <groupId>org.apache.tinkerpop</groupId>
    <artifactId>hadoop-gremlin</artifactId>
    <version>3.2.3</version>
</dependency>

<dependency>
    <groupId>org.apache.tinkerpop</groupId>
    <artifactId>gremlin-core</artifactId>
    <version>3.2.3</version>
</dependency>

<dependency>
    <groupId>org.apache.tinkerpop</groupId>
    <artifactId>gremlin-driver</artifactId>
    <version>3.2.3</version>
</dependency>
nom7f22z

nom7f22z2#

我建议您考虑创建一个包含所有项目依赖项的uberjar。因为您的构建使用的是apachemaven,所以您可以使用apachemaven汇编插件或apachemaven shade插件。

相关问题