org.apache.spark.SparkContext.longAccumulator()方法的使用及代码示例

x33g5p2x  于2022-01-30 转载在 其他  
字(8.1k)|赞(0)|评价(0)|浏览(133)

本文整理了Java中org.apache.spark.SparkContext.longAccumulator()方法的一些代码示例，展示了SparkContext.longAccumulator()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。SparkContext.longAccumulator()方法的具体详情如下：
包路径：org.apache.spark.SparkContext
类名称：SparkContext
方法名：longAccumulator

SparkContext.longAccumulator介绍

暂无

代码示例

代码示例来源：origin: org.apache.spark/spark-core_2.11

@Test
public void foreachPartition() {
 LongAccumulator accum = sc.sc().longAccumulator();
 JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World"));
 rdd.foreachPartition(iter -> {
  while (iter.hasNext()) {
   iter.next();
   accum.add(1);
  }
 });
 assertEquals(2, accum.value().intValue());
}

代码示例来源：origin: org.apache.spark/spark-core_2.10

@Test
public void foreachPartition() {
 LongAccumulator accum = sc.sc().longAccumulator();
 JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World"));
 rdd.foreachPartition(iter -> {
  while (iter.hasNext()) {
   iter.next();
   accum.add(1);
  }
 });
 assertEquals(2, accum.value().intValue());
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
public void foreachPartition() {
 LongAccumulator accum = sc.sc().longAccumulator();
 JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World"));
 rdd.foreachPartition(iter -> {
  while (iter.hasNext()) {
   iter.next();
   accum.add(1);
  }
 });
 assertEquals(2, accum.value().intValue());
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

@Test
public void foreach() {
 LongAccumulator accum = sc.sc().longAccumulator();
 JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World"));
 rdd.foreach(s -> accum.add(1));
 assertEquals(2, accum.value().intValue());
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
public void foreach() {
 LongAccumulator accum = sc.sc().longAccumulator();
 JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World"));
 rdd.foreach(s -> accum.add(1));
 assertEquals(2, accum.value().intValue());
}

代码示例来源：origin: org.apache.spark/spark-core_2.10

@Test
public void foreach() {
 LongAccumulator accum = sc.sc().longAccumulator();
 JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World"));
 rdd.foreach(s -> accum.add(1));
 assertEquals(2, accum.value().intValue());
}

代码示例来源：origin: apache/kylin

final LongAccumulator bytesWritten = sc.sc().longAccumulator();

代码示例来源：origin: org.apache.spark/spark-sql

@Test
public void testForeach() {
 LongAccumulator accum = jsc.sc().longAccumulator();
 List<String> data = Arrays.asList("a", "b", "c");
 Dataset<String> ds = spark.createDataset(data, Encoders.STRING());
 ds.foreach((ForeachFunction<String>) s -> accum.add(1));
 Assert.assertEquals(3, accum.value().intValue());
}

代码示例来源：origin: org.apache.spark/spark-sql_2.11

@Test
public void testForeach() {
 LongAccumulator accum = jsc.sc().longAccumulator();
 List<String> data = Arrays.asList("a", "b", "c");
 Dataset<String> ds = spark.createDataset(data, Encoders.STRING());
 ds.foreach((ForeachFunction<String>) s -> accum.add(1));
 Assert.assertEquals(3, accum.value().intValue());
}

代码示例来源：origin: org.apache.spark/spark-sql_2.10

@Test
public void testForeach() {
 LongAccumulator accum = jsc.sc().longAccumulator();
 List<String> data = Arrays.asList("a", "b", "c");
 Dataset<String> ds = spark.createDataset(data, Encoders.STRING());
 ds.foreach((ForeachFunction<String>) s -> accum.add(1));
 Assert.assertEquals(3, accum.value().intValue());
}

代码示例来源：origin: uber/marmaray

public void registerAccumulators(@NonNull final SparkContext sparkContext) {
  this.registeredKeys.entrySet().forEach(
    entry -> {
      entry.setValue(Optional.of(sparkContext.longAccumulator()));
    }
  );
}

代码示例来源：origin: spirom/learning-spark-with-java

public static LongAccumulator getInstance(JavaSparkContext jsc) {
  if (instance == null) {
   synchronized (RecordCounter.class) {
    if (instance == null) {
     System.out.println("*** Initializing RecordCounter");
     instance = jsc.sc().longAccumulator("RecordCounter");
    }
   }
  }
  return instance;
 }
}

代码示例来源：origin: uber/marmaray

private void logWriteMetrics(final Optional<JavaRDD<WriteStatus>> writesStatuses) {
  if (writesStatuses.isPresent() && this.dataFeedMetrics.isPresent()) {
    final LongAccumulator totalCount = writesStatuses.get().rdd().sparkContext().longAccumulator();
    final LongAccumulator errorCount = writesStatuses.get().rdd().sparkContext().longAccumulator();
    writesStatuses.get().foreach(writeStatus -> {
        errorCount.add(writeStatus.getFailedRecords().size());
        totalCount.add(writeStatus.getTotalRecords());
      });
    this.dataFeedMetrics.get().createLongMetric(DataFeedMetricNames.ERROR_ROWCOUNT, errorCount.value(),
        this.dataFeedMetricsTags);
    this.dataFeedMetrics.get().createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT,
        totalCount.value() - errorCount.value(), this.dataFeedMetricsTags);
  }
}

代码示例来源：origin: cloudera-labs/envelope

public Accumulators(Set<AccumulatorRequest> requests) {
 for (AccumulatorRequest request : requests) {
  String name = request.getName();
  Class<?> clazz = request.getClazz();
  
  if (clazz == Long.class) {
   LongAccumulator acc = Contexts.getSparkSession().sparkContext().longAccumulator(name);
   longAccumulators.put(name, acc);
  }
  
  if (clazz == Double.class) {
   DoubleAccumulator acc = Contexts.getSparkSession().sparkContext().doubleAccumulator(name);
   doubleAccumulators.put(name, acc);
  }
  
  LOG.info("Processed accumulator request: " + name);
 }
}

代码示例来源：origin: uber/marmaray

/**
 * {@link #updateSinkStat(Optional)} will compute {@link SinkStat} and persist changes into {@link IMetadataManager}.
 * As a part of {@link SinkStat} computation; it will compute avg record size for current run.
 * @param writesStatuses
 */
private void updateSinkStat(final Optional<JavaRDD<WriteStatus>> writesStatuses) {
  if (writesStatuses.isPresent()) {
    final LongAccumulator avgRecordSizeCounter = writesStatuses.get().rdd().sparkContext().longAccumulator();
    writesStatuses.get().foreach(
      writeStatus -> {
        final long writeBytes = writeStatus.getStat().getTotalWriteBytes();
        final long numInserts = writeStatus.getStat().getNumWrites()
            - writeStatus.getStat().getNumUpdateWrites();
        if (writeBytes > 0 && numInserts > 0) {
          avgRecordSizeCounter.add(writeBytes / numInserts);
        }
      }
    );
    final long avgRecordSize = (int) avgRecordSizeCounter.avg();
    if (avgRecordSize > 0) {
      log.info("Updating Sink Stat manager : avgRecordSize : {}", avgRecordSize);
      this.sinkStatMgr.getCurrentStat().put(SinkStat.AVG_RECORD_SIZE, Long.toString(avgRecordSize));
    }
  }
  this.sinkStatMgr.persist();
}

代码示例来源：origin: org.apache.kylin/kylin-engine-spark

final LongAccumulator bytesWritten = sc.sc().longAccumulator();

代码示例来源：origin: org.apache.spark/spark-streaming_2.11

@SuppressWarnings("unchecked")
@Test
public void testForeachRDD() {
 final LongAccumulator accumRdd = ssc.sparkContext().sc().longAccumulator();
 final LongAccumulator accumEle = ssc.sparkContext().sc().longAccumulator();
 List<List<Integer>> inputData = Arrays.asList(
   Arrays.asList(1,1,1),
   Arrays.asList(1,1,1));
 JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaTestUtils.attachTestOutputStream(stream.count()); // dummy output
 stream.foreachRDD(rdd -> {
  accumRdd.add(1);
  rdd.foreach(i -> accumEle.add(1));
 });
 // This is a test to make sure foreachRDD(VoidFunction2) can be called from Java
 stream.foreachRDD((rdd, time) -> {});
 JavaTestUtils.runStreams(ssc, 2, 2);
 Assert.assertEquals(2, accumRdd.value().intValue());
 Assert.assertEquals(6, accumEle.value().intValue());
}

代码示例来源：origin: org.apache.spark/spark-streaming_2.10

@SuppressWarnings("unchecked")
@Test
public void testForeachRDD() {
 final LongAccumulator accumRdd = ssc.sparkContext().sc().longAccumulator();
 final LongAccumulator accumEle = ssc.sparkContext().sc().longAccumulator();
 List<List<Integer>> inputData = Arrays.asList(
   Arrays.asList(1,1,1),
   Arrays.asList(1,1,1));
 JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaTestUtils.attachTestOutputStream(stream.count()); // dummy output
 stream.foreachRDD(rdd -> {
  accumRdd.add(1);
  rdd.foreach(i -> accumEle.add(1));
 });
 // This is a test to make sure foreachRDD(VoidFunction2) can be called from Java
 stream.foreachRDD((rdd, time) -> {});
 JavaTestUtils.runStreams(ssc, 2, 2);
 Assert.assertEquals(2, accumRdd.value().intValue());
 Assert.assertEquals(6, accumEle.value().intValue());
}

内容来源于网络，如有侵权，请联系作者删除！

SparkContext

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com