org.apache.spark.SparkContext.<init>()方法的使用及代码示例

x33g5p2x  于2022-01-30 转载在 其他  
字(10.1k)|赞(0)|评价(0)|浏览(151)

本文整理了Java中org.apache.spark.SparkContext.<init>()方法的一些代码示例，展示了SparkContext.<init>()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。SparkContext.<init>()方法的具体详情如下：
包路径：org.apache.spark.SparkContext
类名称：SparkContext
方法名：<init>

SparkContext.<init>介绍

暂无

代码示例

代码示例来源：origin: org.apache.spark/spark-core_2.11

@Test
 public void scalaSparkContext() {
  List<String> jars = List$.MODULE$.empty();
  Map<String, String> environment = Map$.MODULE$.empty();

  new SparkContext(new SparkConf().setMaster("local").setAppName("name")).stop();
  new SparkContext("local", "name", new SparkConf()).stop();
  new SparkContext("local", "name").stop();
  new SparkContext("local", "name", "sparkHome").stop();
  new SparkContext("local", "name", "sparkHome", jars).stop();
  new SparkContext("local", "name", "sparkHome", jars, environment).stop();
 }
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
 public void scalaSparkContext() {
  List<String> jars = List$.MODULE$.empty();
  Map<String, String> environment = Map$.MODULE$.empty();

  new SparkContext(new SparkConf().setMaster("local").setAppName("name")).stop();
  new SparkContext("local", "name", new SparkConf()).stop();
  new SparkContext("local", "name").stop();
  new SparkContext("local", "name", "sparkHome").stop();
  new SparkContext("local", "name", "sparkHome", jars).stop();
  new SparkContext("local", "name", "sparkHome", jars, environment).stop();
 }
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

public static void main(String[] args) throws Exception {
 assertNotEquals(0, args.length);
 assertEquals(args[0], "hello");
 new SparkContext().stop();
 synchronized (LOCK) {
  LOCK.notifyAll();
 }
}

代码示例来源：origin: org.apache.spark/spark-core

public static void main(String[] args) throws Exception {
 assertNotEquals(0, args.length);
 assertEquals(args[0], "hello");
 new SparkContext().stop();
 synchronized (LOCK) {
  LOCK.notifyAll();
 }
}

代码示例来源：origin: Impetus/Kundera

@Override
public void initialize(Map<String, Object> puProperties)
{
  reader = new SparkEntityReader(kunderaMetadata);
  setExternalProperties(puProperties);
  initializePropertyReader();
  PersistenceUnitMetadata pum = kunderaMetadata.getApplicationMetadata().getPersistenceUnitMetadata(
      getPersistenceUnit());
  sparkconf = new SparkConf(true);
  configureClientProperties(pum);
  sparkContext = new SparkContext(sparkconf);
  sqlContext = new HiveContext(sparkContext);
}

代码示例来源：origin: stackoverflow.com

val sc = new SparkContext();
val fs = FileSystem.get(sc.hadoopConfig)
val snapshotPath = fs.createSnapshot("path to createsnapshot of","snapshot name")

.....
.....
if (condition satisfied) {
 fs.deleteSnapshot(snapshotPath,"snapshot name")
}

代码示例来源：origin: stackoverflow.com

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import scala.io.Source
import java.io.File

 val sc = new SparkContext("your_spark_master", "your_application_name")
 // Read from a local dir
 val sourceRecords = new File("your_dir_path").listFiles().flatMap(file => Source.fromFile(file).getLines.toList).filter(_.length > 0)
 val sourceRecordsBc = sc.broadcast(sourceRecords)
 val inputRDD = sc.textFile("hdfs://your_input_path")
 val outputRDD = inputRDD.filter { _.length > 0 }
  .map { line => (line, line.split(",")) }
  .filter { _._2.size == 21 }
  .flatMap {
   case (line, target) => {
    val pos_last_name = 0
    for (
     inputLine <- sourceRecordsBc.value if inputLine != line;
     val source = inputLine.split(",") if source.size == 21 && source(pos_last_name) == target(pos_last_name)
    ) yield (inputLine, line)
   }
  }
 outputRDD.saveAsTextFile("hdfs://your_output_path")

代码示例来源：origin: stackoverflow.com

import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext
import org.apache.phoenix.spark._

val sc = new SparkContext("local", "phoenix-test")
val sqlContext = new SQLContext(sc)

val df = sqlContext.load(
 "org.apache.phoenix.spark", 
 Map("table" -> "TABLE1", "zkUrl" -> "phoenix-server:2181")
)

df
 .filter(df("COL1") === "test_row_1" && df("ID") === 1L)
 .select(df("ID"))
 .show

代码示例来源：origin: stackoverflow.com

scala.collection.immutable.Range rangeMscala = 
 scala.collection.immutable.Range$.MODULE$.apply(1, 10);

SparkContext sc = new SparkContext();

RDD dataSetMscala = 
 sc.parallelize(rangeMscala, 3, scala.reflect.ClassTag$.MODULE$.Object());

代码示例来源：origin: bhdrkn/Java-Examples

@Bean(destroyMethod = "stop")
public SparkContext sparkContext() {
  final SparkConf conf = new SparkConf()
      .setAppName("MLSparkService")
      .setMaster("local");
  return new SparkContext(conf);
}

代码示例来源：origin: com.stratio.meta/meta-core

/**
 * Initialize the DeepSparkContext adding the required jars if the deployment is not local.
 * @param config The {@link com.stratio.meta.core.engine.EngineConfig}
 * @return A new context.
 */
private DeepSparkContext initializeDeep(EngineConfig config){
 //DeepSparkContext result = new DeepSparkContext(config.getSparkMaster(), config.getJobName());
 SparkConf sparkConf = new SparkConf().set("spark.driver.port",
                      "0")//String.valueOf(StreamingUtils.findFreePort()))
                    .set("spark.ui.port",
                      "0");//String.valueOf(StreamingUtils.findFreePort()));
 DeepSparkContext result = new DeepSparkContext(new SparkContext(config.getSparkMaster(), config.getJobName(), sparkConf));
 if(!config.getSparkMaster().toLowerCase().startsWith("local")){
  for(String jar : config.getJars()){
   result.addJar(jar);
  }
 }
 return  result;
}

代码示例来源：origin: amidst/toolbox

public static void main(String[] args) throws Exception {
  String arffFile = "datasets/simulated/syntheticData.arff";
  String sparkFile = "datasets/simulated/syntheticData.json";
  SparkConf conf = new SparkConf().setAppName("SparkLink!").setMaster("local");
  SparkContext sc = new SparkContext(conf);
  SQLContext sqlContext = new SQLContext(sc);
  JavaSparkContext jsc = new JavaSparkContext(sc);
  ARFFtoSparkFormat(arffFile, sparkFile, "json", sqlContext, jsc);
}

代码示例来源：origin: com.holdenkarau/spark-testing-base_2.11

@Before
public void runBefore() {
 initialized = (_sc != null);
 if (!initialized) {
  _sc = new SparkContext(conf());
  _jsc = new JavaSparkContext(_sc);
  beforeAllTestCasesHook();
 }
}

代码示例来源：origin: org.talend.components/components-adapter-beam

public Pipeline createPipeline() {
    SparkContextOptions sparkOpts = options.as(SparkContextOptions.class);

    SparkConf conf = new SparkConf();
    conf.setAppName(appName);
    conf.setMaster("local[2]");
    conf.set("spark.driver.allowMultipleContexts", "true");
    JavaSparkContext jsc = new JavaSparkContext(new SparkContext(conf));
    sparkOpts.setProvidedSparkContext(jsc);
    sparkOpts.setUsesProvidedSparkContext(true);
    sparkOpts.setRunner(SparkRunner.class);

    return Pipeline.create(sparkOpts);
  }
}

代码示例来源：origin: com.holdenkarau/spark-testing-base_2.10

@Before
public void runBefore() {
 initialized = (_sc != null);
 if (!initialized) {
  _sc = new SparkContext(conf());
  _jsc = new JavaSparkContext(_sc);
  beforeAllTestCasesHook();
 }
}

代码示例来源：origin: amidst/toolbox

public static void main(String[] args) throws Exception {
    SparkConf conf = new SparkConf().setAppName("SLink!").setMaster("local");
    SparkContext sc = new SparkContext(conf);
    SQLContext sqlContext = new SQLContext(sc);

    //Path to dataset
    String path ="datasets/simulated/WI_samples.json";

    //Create an AMIDST object for managing the data
    DataSpark dataSpark = DataSparkLoader.open(sqlContext, path);

    //Print all the instances in the dataset
    dataSpark.collectDataStream()
        .forEach(
            dataInstance -> System.out.println(dataInstance)
        );

  }
}

代码示例来源：origin: Talend/components

public Pipeline createPipeline() {
    SparkContextOptions sparkOpts = options.as(SparkContextOptions.class);

    SparkConf conf = new SparkConf();
    conf.setAppName(appName);
    conf.setMaster("local[2]");
    conf.set("spark.driver.allowMultipleContexts", "true");
    JavaSparkContext jsc = new JavaSparkContext(new SparkContext(conf));
    sparkOpts.setProvidedSparkContext(jsc);
    sparkOpts.setUsesProvidedSparkContext(true);
    sparkOpts.setRunner(SparkRunner.class);

    return Pipeline.create(sparkOpts);
  }
}

代码示例来源：origin: io.zipkin.dependencies/zipkin-dependencies-cassandra

public void run() {
 long microsLower = day * 1000;
 long microsUpper = (day * 1000) + TimeUnit.DAYS.toMicros(1) - 1;
 log.info("Running Dependencies job for {}: {} ≤ Span.timestamp {}", dateStamp, microsLower,
   microsUpper);
 SparkContext sc = new SparkContext(conf);
 List<DependencyLink> links = javaFunctions(sc)
  .cassandraTable(keyspace, "traces")
  .spanBy(ROW_TRACE_ID, Long.class)
  .flatMapValues(new CassandraRowsToDependencyLinks(logInitializer, microsLower, microsUpper))
  .values()
  .mapToPair(LINK_TO_PAIR)
  .reduceByKey(MERGE_LINK)
  .values()
  .collect();
 sc.stop();
 saveToCassandra(links);
}

代码示例来源：origin: amidst/toolbox

public static void main(String[] args) throws Exception {
  SparkConf conf = new SparkConf().setAppName("SparkLink!").setMaster("local");;
  SparkContext sc = new SparkContext(conf);
  SQLContext sqlContext = new SQLContext(sc);

代码示例来源：origin: jgperrin/net.jgp.labs.spark

private void start() {
  SparkConf conf = new SparkConf().setAppName("Checkpoint").setMaster(
    "local[*]");
  SparkContext sparkContext = new SparkContext(conf);
  // We need to specify where Spark will save the checkpoint file. It can be
  // an HDFS location.
  sparkContext.setCheckpointDir("/tmp");
  SparkSession spark = SparkSession.builder().appName("Checkpoint").master(
    "local[*]").getOrCreate();

  String filename = "data/tuple-data-file.csv";
  Dataset<Row> df1 = spark.read().format("csv").option("inferSchema", "true")
    .option("header", "false")
    .load(filename);
  System.out.println("DF #1 - step #1: simple dump of the dataframe");
  df1.show();

  System.out.println("DF #2 - step #2: same as DF #1 - step #1");
  Dataset<Row> df2 = df1.checkpoint(false);
  df2.show();

  df1 = df1.withColumn("x", df1.col("_c0"));
  System.out.println(
    "DF #1 - step #2: new column x, which is the same as _c0");
  df1.show();

  System.out.println("DF #2 - step #2: no operation was done on df2");
  df2.show();
 }
}

内容来源于网络，如有侵权，请联系作者删除！

SparkContext

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com