我有一个非常简单的spark应用程序：

object SimpleApp {

  def main(args: Array[String]) {

    val logger = LoggerFactory.getLogger(getClass.getName)
    val spark = SparkSession.builder.appName("Simple Application").getOrCreate()
    val sc = spark.sparkContext

    val executorNumber = sc.getConf.get("spark.executor.instances").toInt

    val coreNumber = sc.getConf.get("spark.executor.cores").toInt

    logger.info("spark.executor.instances " + executorNumber)
    logger.info("spark.executor.cores " + coreNumber)
    spark.stop()

  }
}

我使用以下sbt文件构建此应用程序：

name := "Simple Project"
version := "1.0"
scalaVersion := "2.11.12"
val sparkVersion = "2.4.0"
libraryDependencies ++= Seq(
  "org.apache.spark" %% "spark-core" % sparkVersion,
  "org.apache.spark" %% "spark-sql" % sparkVersion
)

我可以用spark submit运行这个应用程序

spark-submit --num-executors 5 --executor-cores 7  --executor-memory 4G --class "SimpleApp" --master yarn --deploy-mode cluster simple-project_2.11-1.0.jar

但在色调上，使用相同的参数，它会失败并产生错误

10:43:49.253 [Driver] ERROR org.apache.spark.deploy.yarn.ApplicationMaster - User class threw exception: java.lang.NoClassDefFoundError: com/codahale/metrics/json/MetricsModule
java.lang.NoClassDefFoundError: com/codahale/metrics/json/MetricsModule
        at java.lang.Class.forName0(Native Method) ~[?:1.8.0_181]
        at java.lang.Class.forName(Class.java:348) ~[?:1.8.0_181]
        at org.apache.spark.util.Utils$.classForName(Utils.scala:242) ~[spark-core_2.11-2.4.0-cdh6.3.2.jar:2.4.0-cdh6.3.2]
       ...
        at scala.Option.getOrElse(Option.scala:121) ~[scala-library-2.11.12.jar:?]
        at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:935) ~[spark-sql_2.11-2.4.0-cdh6.3.2.jar:2.4.0-cdh6.3.2]
        at SimpleApp$.main(SimpleApp.scala:14) ~[simple-project_2.11-1.0.jar:?]
        at SimpleApp.main(SimpleApp.scala) ~[simple-project_2.11-1.0.jar:?]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_181]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_181]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_181]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_181]
        at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:673) [spark-yarn_2.11-2.4.0-cdh6.3.2.jar:2.4.0-cdh6.3.2]
Caused by: java.lang.ClassNotFoundException: com.codahale.metrics.json.MetricsModule
        at java.net.URLClassLoader.findClass(URLClassLoader.java:381) ~[?:1.8.0_181]
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_181]
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349) ~[?:1.8.0_181]
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_181]
        ... 25 more