org.apache.spark.api.java.JavaSparkContext.accumulator()方法的使用及代码示例

x33g5p2x  于2022-01-21 转载在 其他  
字(10.6k)|赞(0)|评价(0)|浏览(134)

本文整理了Java中org.apache.spark.api.java.JavaSparkContext.accumulator()方法的一些代码示例，展示了JavaSparkContext.accumulator()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。JavaSparkContext.accumulator()方法的具体详情如下：
包路径：org.apache.spark.api.java.JavaSparkContext
类名称：JavaSparkContext
方法名：accumulator

JavaSparkContext.accumulator介绍

暂无

代码示例

代码示例来源：origin: apache/hive

public SparkCounter(
 String name,
 String displayName,
 String groupName,
 long initValue,
 JavaSparkContext sparkContext) {
 this.name = name;
 this.displayName = displayName;
 LongAccumulatorParam longParam = new LongAccumulatorParam();
 String accumulatorName = groupName + "_" + name;
 this.accumulator = sparkContext.accumulator(initValue, accumulatorName, longParam);
}

代码示例来源：origin: databricks/learning-spark

JavaRDD<String> rdd = sc.textFile(inputFile);
final Accumulator<Integer> count = sc.accumulator(0);
rdd.foreach(new VoidFunction<String>(){ public void call(String line) {
   if (line.contains("KK6JKQ")) {
System.out.println("Lines with 'KK6JKQ': " + count.value());
final Accumulator<Integer> blankLines = sc.accumulator(0);
JavaRDD<String> callSigns = rdd.flatMap(
 new FlatMapFunction<String, String>() { public Iterable<String> call(String line) {
System.out.println("Blank lines: "+ blankLines.value());
final Accumulator<Integer> validSignCount = sc.accumulator(0);
final Accumulator<Integer> invalidSignCount = sc.accumulator(0);
JavaRDD<String> validCallSigns = callSigns.filter(
 new Function<String, Boolean>(){ public Boolean call(String callSign) {

代码示例来源：origin: org.apache.spark/spark-core_2.11

Accumulator<Float> floatAccum = sc.accumulator(10.0f, floatAccumulatorParam);
rdd.foreach(x -> floatAccum.add((float) x));
assertEquals((Float) 25.0f, floatAccum.value());

代码示例来源：origin: org.apache.spark/spark-core_2.10

Accumulator<Float> floatAccum = sc.accumulator(10.0f, floatAccumulatorParam);
rdd.foreach(x -> floatAccum.add((float) x));
assertEquals((Float) 25.0f, floatAccum.value());

代码示例来源：origin: org.apache.spark/spark-core

Accumulator<Float> floatAccum = sc.accumulator(10.0f, floatAccumulatorParam);
rdd.foreach(x -> floatAccum.add((float) x));
assertEquals((Float) 25.0f, floatAccum.value());

代码示例来源：origin: apache/tinkerpop

public SparkMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers, final JavaSparkContext sparkContext) {
  if (null != vertexProgram) {
    for (final MemoryComputeKey key : vertexProgram.getMemoryComputeKeys()) {
      this.memoryComputeKeys.put(key.getKey(), key);
    }
  }
  for (final MapReduce mapReduce : mapReducers) {
    this.memoryComputeKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false));
  }
  for (final MemoryComputeKey memoryComputeKey : this.memoryComputeKeys.values()) {
    this.sparkMemory.put(
        memoryComputeKey.getKey(),
        sparkContext.accumulator(ObjectWritable.empty(), memoryComputeKey.getKey(), new MemoryAccumulator<>(memoryComputeKey)));
  }
  this.broadcast = sparkContext.broadcast(Collections.emptyMap());
}

代码示例来源：origin: co.cask.cdap/hydrator-spark-core2

public SparkStageStatisticsCollector(JavaSparkContext jsc) {
 this.inputRecordCounter = jsc.accumulator(0.0);
 this.outputRecordCounter = jsc.accumulator(0.0);
 this.errorRecordCounter = jsc.accumulator(0.0);
}

代码示例来源：origin: com.github.hyukjinkwon/spark-client

public SparkCounter(
 String name,
 String displayName,
 String groupName,
 long initValue,
 JavaSparkContext sparkContext) {
 this.name = name;
 this.displayName = displayName;
 LongAccumulatorParam longParam = new LongAccumulatorParam();
 String accumulatorName = groupName + "_" + name;
 this.accumulator = sparkContext.accumulator(initValue, accumulatorName, longParam);
}

代码示例来源：origin: org.spark-project.hive/spark-client

public SparkCounter(
 String name,
 String displayName,
 String groupName,
 long initValue,
 JavaSparkContext sparkContext) {
 this.name = name;
 this.displayName = displayName;
 LongAccumulatorParam longParam = new LongAccumulatorParam();
 String accumulatorName = groupName + "_" + name;
 this.accumulator = sparkContext.accumulator(initValue, accumulatorName, longParam);
}

代码示例来源：origin: org.apache.pig/pig

public SparkCounter(
    String name,
    String displayName,
    String groupName,
    T initValue,
    JavaSparkContext sparkContext) {
  this.name = name;
  this.displayName = displayName;
  String accumulatorName = groupName + "_" + name;
  if (sparkContext == null){
    //Spark executors can register new Accumulators but they won't make it back to the driver hence the limitation
    throw new  RuntimeException("Not allowed to create SparkCounter on backend executor.");
  }
  this.accumulator = sparkContext.accumulator(initValue, accumulatorName,  createAccumulatorParam());
}

代码示例来源：origin: phuonglh/vn.vitk

public Tokenizer(String master, String lexiconFileName, String regexpFileName, String whitespaceModelFileName, boolean lr) {
  this(master, lexiconFileName, regexpFileName);
  classifier = new WhitespaceClassifier(lexicon, patterns);
  model = classifier.load(whitespaceModelFileName);
  contexts = jsc.accumulator(new LinkedList<WhitespaceContext>(), new WhitespaceContextAccumulatorParam());
}

代码示例来源：origin: uber/hudi

public static int handleErrors(JavaSparkContext jsc, String instantTime, JavaRDD<WriteStatus> writeResponse) {
 Accumulator<Integer> errors = jsc.accumulator(0);
 writeResponse.foreach(writeStatus -> {
  if (writeStatus.hasErrors()) {
   errors.add(1);
   logger.error(String.format("Error processing records :writeStatus:%s",
     writeStatus.getStat().toString()));
  }
 });
 if (errors.value() == 0) {
  logger.info(
    String.format("Dataset imported into hoodie dataset with %s instant time.", instantTime));
  return 0;
 }
 logger.error(String.format("Import failed with %d errors.", errors.value()));
 return -1;
}

代码示例来源：origin: com.uber.hoodie/hoodie-utilities

public static int handleErrors(JavaSparkContext jsc, String instantTime, JavaRDD<WriteStatus> writeResponse) {
 Accumulator<Integer> errors = jsc.accumulator(0);
 writeResponse.foreach(writeStatus -> {
  if (writeStatus.hasErrors()) {
   errors.add(1);
   logger.error(String.format("Error processing records :writeStatus:%s",
     writeStatus.getStat().toString()));
  }
 });
 if (errors.value() == 0) {
  logger.info(
    String.format("Dataset imported into hoodie dataset with %s instant time.", instantTime));
  return 0;
 }
 logger.error(String.format("Import failed with %d errors.", errors.value()));
 return -1;
}

代码示例来源：origin: org.apache.crunch/crunch-spark

public SparkRuntime(SparkPipeline pipeline,
          JavaSparkContext sparkContext,
          Configuration conf,
          Map<PCollectionImpl<?>, Set<Target>> outputTargets,
          Map<PCollectionImpl<?>, MaterializableIterable> toMaterialize,
          Map<PCollection<?>, StorageLevel> toCache,
          Map<PipelineCallable<?>, Set<Target>> allPipelineCallables) {
 this.pipeline = pipeline;
 this.sparkContext = sparkContext;
 this.conf = conf;
 this.counters = sparkContext.accumulator(Maps.<String, Map<String, Long>>newHashMap(), pipeline.getName(),
   new CounterAccumulatorParam());
 this.ctxt = new SparkRuntimeContext(sparkContext.appName(), counters,
   sparkContext.broadcast(WritableUtils.toByteArray(conf)));
 this.outputTargets = Maps.newTreeMap(DEPTH_COMPARATOR);
 this.outputTargets.putAll(outputTargets);
 this.toMaterialize = toMaterialize;
 this.toCache = toCache;
 this.allPipelineCallables = allPipelineCallables;
 this.activePipelineCallables = allPipelineCallables.keySet();
 this.status.set(Status.READY);
 this.monitorThread = new Thread(new Runnable() {
  @Override
  public void run() {
   monitorLoop();
  }
 });
}

代码示例来源：origin: apache/crunch

public SparkRuntime(SparkPipeline pipeline,
          JavaSparkContext sparkContext,
          Configuration conf,
          Map<PCollectionImpl<?>, Set<Target>> outputTargets,
          Map<PCollectionImpl<?>, MaterializableIterable> toMaterialize,
          Map<PCollection<?>, StorageLevel> toCache,
          Map<PipelineCallable<?>, Set<Target>> allPipelineCallables) {
 this.pipeline = pipeline;
 this.sparkContext = sparkContext;
 this.conf = conf;
 this.counters = sparkContext.accumulator(Maps.<String, Map<String, Long>>newHashMap(), pipeline.getName(),
   new CounterAccumulatorParam());
 this.ctxt = new SparkRuntimeContext(sparkContext.appName(), counters,
   sparkContext.broadcast(WritableUtils.toByteArray(conf)));
 this.outputTargets = Maps.newTreeMap(DEPTH_COMPARATOR);
 this.outputTargets.putAll(outputTargets);
 this.toMaterialize = toMaterialize;
 this.toCache = toCache;
 this.allPipelineCallables = allPipelineCallables;
 this.activePipelineCallables = allPipelineCallables.keySet();
 this.status.set(Status.READY);
 this.monitorThread = new Thread(new Runnable() {
  @Override
  public void run() {
   monitorLoop();
  }
 });
}

代码示例来源：origin: org.qcri.rheem/rheem-spark

public void accept(JavaRDD<?> rdd, SparkExecutor sparkExecutor) throws RheemException {
  if (this.isMarkedForInstrumentation() && !this.isRddCached()) {
    final Accumulator<Integer> accumulator = sparkExecutor.sc.accumulator(0);
    this.rdd = rdd.filter(dataQuantum -> {
      accumulator.add(1);
      return true;
    });
    this.accumulator = accumulator;
  } else {
    this.rdd = rdd;
  }
}

代码示例来源：origin: org.apache.tinkerpop/spark-gremlin

public SparkMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers, final JavaSparkContext sparkContext) {
  if (null != vertexProgram) {
    for (final MemoryComputeKey key : vertexProgram.getMemoryComputeKeys()) {
      this.memoryComputeKeys.put(key.getKey(), key);
    }
  }
  for (final MapReduce mapReduce : mapReducers) {
    this.memoryComputeKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false));
  }
  for (final MemoryComputeKey memoryComputeKey : this.memoryComputeKeys.values()) {
    this.sparkMemory.put(
        memoryComputeKey.getKey(),
        sparkContext.accumulator(ObjectWritable.empty(), memoryComputeKey.getKey(), new MemoryAccumulator<>(memoryComputeKey)));
  }
  this.broadcast = sparkContext.broadcast(Collections.emptyMap());
}

代码示例来源：origin: ai.grakn/grakn-kb

public GraknSparkMemory(final VertexProgram<?> vertexProgram,
            final Set<MapReduce> mapReducers,
            final JavaSparkContext sparkContext) {
  if (null != vertexProgram) {
    for (final MemoryComputeKey key : vertexProgram.getMemoryComputeKeys()) {
      this.memoryComputeKeys.put(key.getKey(), key);
    }
  }
  for (final MapReduce mapReduce : mapReducers) {
    this.memoryComputeKeys.put(
        mapReduce.getMemoryKey(),
        MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false));
  }
  for (final MemoryComputeKey memoryComputeKey : this.memoryComputeKeys.values()) {
    this.sparkMemory.put(
        memoryComputeKey.getKey(),
        sparkContext.accumulator(ObjectWritable.empty(), memoryComputeKey.getKey(),
            new MemoryAccumulator<>(memoryComputeKey)));
  }
  this.broadcast = sparkContext.broadcast(Collections.emptyMap());
}

代码示例来源：origin: deepspark/deepspark

final Accumulator<Integer> totalCount = sc.accumulator(0);

代码示例来源：origin: oeljeklaus-you/UserActionAnalyzePlatform

Accumulator<String> sessionAggrStatAccumulator=context.accumulator("",new SessionAggrStatAccumulator());

内容来源于网络，如有侵权，请联系作者删除！

JavaSparkContext

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多

    JavaSparkContext类方法

    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com