org.apache.spark.streaming.Durations.seconds()方法的使用及代码示例

x33g5p2x  于2022-01-18 转载在 其他  
字(8.2k)|赞(0)|评价(0)|浏览(310)

本文整理了Java中org.apache.spark.streaming.Durations.seconds()方法的一些代码示例,展示了Durations.seconds()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Durations.seconds()方法的具体详情如下:
包路径:org.apache.spark.streaming.Durations
类名称:Durations
方法名:seconds

Durations.seconds介绍

暂无

代码示例

代码示例来源:origin: baghelamit/iot-traffic-monitor

/**
 * Method to get window traffic counts of different type of vehicles for each route.
 * Window duration = 30 seconds and Slide interval = 10 seconds
 * 
 * @param filteredIotDataStream IoT data stream
 */
public void processWindowTrafficData(JavaDStream<IoTData> filteredIotDataStream) {
  // reduce by key and window (30 sec window and 10 sec slide).
  JavaPairDStream<AggregateKey, Long> countDStreamPair = filteredIotDataStream
      .mapToPair(iot -> new Tuple2<>(new AggregateKey(iot.getRouteId(), iot.getVehicleType()), 1L))
      .reduceByKeyAndWindow((a, b) -> a + b, Durations.seconds(30), Durations.seconds(10));
  // Transform to dstream of TrafficData
  JavaDStream<WindowTrafficData> trafficDStream = countDStreamPair.map(windowTrafficDataFunc);
  // Map Cassandra table column
  Map<String, String> columnNameMappings = new HashMap<String, String>();
  columnNameMappings.put("routeId", "routeid");
  columnNameMappings.put("vehicleType", "vehicletype");
  columnNameMappings.put("totalCount", "totalcount");
  columnNameMappings.put("timeStamp", "timestamp");
  columnNameMappings.put("recordDate", "recorddate");
  // call CassandraStreamingJavaUtil function to save in DB
  javaFunctions(trafficDStream).writerBuilder("traffickeyspace", "window_traffic",
      CassandraJavaUtil.mapToRow(WindowTrafficData.class, columnNameMappings)).saveToCassandra();
}

代码示例来源:origin: ebi-wp/kafka-streams-api-websockets

JavaStreamingContext streamingContext = new JavaStreamingContext(sparkConf, Durations.seconds(5));

代码示例来源:origin: dibbhatt/kafka-spark-consumer

JavaStreamingContext jsc = new JavaStreamingContext(_sparkConf, Durations.seconds(30));

代码示例来源:origin: nivdul/spark-in-practice

/**
 *  Load the data using TwitterUtils: we obtain a DStream of tweets
 *
 *  More about TwitterUtils:
 *  https://spark.apache.org/docs/1.4.0/api/java/index.html?org/apache/spark/streaming/twitter/TwitterUtils.html
 */
public JavaDStream<Status> loadData() {
 // create the spark configuration and spark context
 SparkConf conf = new SparkConf()
   .setAppName("Spark Streaming")
   .set("spark.driver.allowMultipleContexts", "true")
   .setMaster("local[*]");
 // create a java streaming context and define the window (2 seconds batch)
 jssc = new JavaStreamingContext(conf, Durations.seconds(2));
 System.out.println("Initializing Twitter stream...");
 // create a DStream (sequence of RDD). The object tweetsStream is a DStream of tweet statuses:
 // - the Status class contains all information of a tweet
 // See http://twitter4j.org/javadoc/twitter4j/Status.html
 // and fill the keys and tokens in the Streamutils class!
 JavaDStream<Status> tweetsStream = TwitterUtils.createStream(jssc, StreamUtils.getAuth());
 return tweetsStream;
}

代码示例来源:origin: jgperrin/net.jgp.labs.spark

private void start() {
  // Create a local StreamingContext with two working thread and batch
  // interval of
  // 1 second
  SparkConf conf = new SparkConf().setMaster("local[2]").setAppName(
    "NetworkWordCount");
  JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations
    .seconds(5));

  JavaDStream<String> msgDataStream = jssc.textFileStream(StreamingUtils
    .getInputDirectory());
  msgDataStream.print();

  jssc.start();
  try {
   jssc.awaitTermination();
  } catch (InterruptedException e) {
   // TODO Auto-generated catch block
   e.printStackTrace();
  }
 }
}

代码示例来源:origin: co.cask.cdap/hydrator-spark-core2

@Override
public SparkCollection<T> window(StageSpec stageSpec, Windower windower) {
 String stageName = stageSpec.getName();
 return wrap(stream.transform(new CountingTransformFunction<T>(stageName, sec.getMetrics(), "records.in", null))
        .window(Durations.seconds(windower.getWidth()), Durations.seconds(windower.getSlideInterval()))
        .transform(new CountingTransformFunction<T>(stageName, sec.getMetrics(), "records.out",
                              sec.getDataTracer(stageName))));
}

代码示例来源:origin: baghelamit/iot-traffic-monitor

/**
 * Method to get total traffic counts of different type of vehicles for each route.
 * 
 * @param filteredIotDataStream IoT data stream
 */
public void processTotalTrafficData(JavaDStream<IoTData> filteredIotDataStream) {
  // We need to get count of vehicle group by routeId and vehicleType
  JavaPairDStream<AggregateKey, Long> countDStreamPair = filteredIotDataStream
      .mapToPair(iot -> new Tuple2<>(new AggregateKey(iot.getRouteId(), iot.getVehicleType()), 1L))
      .reduceByKey((a, b) -> a + b);
  
  // Need to keep state for total count
  JavaMapWithStateDStream<AggregateKey, Long, Long, Tuple2<AggregateKey, Long>> countDStreamWithStatePair = countDStreamPair
      .mapWithState(StateSpec.function(totalSumFunc).timeout(Durations.seconds(3600)));//maintain state for one hour
  // Transform to dstream of TrafficData
  JavaDStream<Tuple2<AggregateKey, Long>> countDStream = countDStreamWithStatePair.map(tuple2 -> tuple2);
  JavaDStream<TotalTrafficData> trafficDStream = countDStream.map(totalTrafficDataFunc);
  // Map Cassandra table column
  Map<String, String> columnNameMappings = new HashMap<String, String>();
  columnNameMappings.put("routeId", "routeid");
  columnNameMappings.put("vehicleType", "vehicletype");
  columnNameMappings.put("totalCount", "totalcount");
  columnNameMappings.put("timeStamp", "timestamp");
  columnNameMappings.put("recordDate", "recorddate");
  // call CassandraStreamingJavaUtil function to save in DB
  javaFunctions(trafficDStream).writerBuilder("traffickeyspace", "total_traffic",
      CassandraJavaUtil.mapToRow(TotalTrafficData.class, columnNameMappings)).saveToCassandra();
}

代码示例来源:origin: org.apache.spark/spark-streaming_2.10

@Test
public void testSeconds() {
 Assert.assertEquals(new Duration(30 * 1000), Durations.seconds(30));
}

代码示例来源:origin: org.apache.spark/spark-streaming_2.11

@Test
public void testSeconds() {
 Assert.assertEquals(new Duration(30 * 1000), Durations.seconds(30));
}

代码示例来源:origin: jgperrin/net.jgp.labs.spark

private void start() {
  // Create a local StreamingContext with two working thread and batch
  // interval of
  // 1 second
  SparkConf conf = new SparkConf().setMaster("local[2]").setAppName(
    "Streaming Ingestion File System Text File to Dataframe");
  JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations
    .seconds(5));

  JavaDStream<String> msgDataStream = jssc.textFileStream(StreamingUtils
    .getInputDirectory());

  msgDataStream.print();
  // Create JavaRDD<Row>
  msgDataStream.foreachRDD(new RowProcessor());

  jssc.start();
  try {
   jssc.awaitTermination();
  } catch (InterruptedException e) {
   // TODO Auto-generated catch block
   e.printStackTrace();
  }
 }
}

代码示例来源:origin: baghelamit/iot-traffic-monitor

.set("spark.cassandra.connection.keep_alive_ms", prop.getProperty("com.iot.app.cassandra.keep_alive"));		 
JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(5));	
         .mapWithState(StateSpec.function(processedVehicleFunc).timeout(Durations.seconds(3600)));//maintain state for one hour

代码示例来源:origin: jgperrin/net.jgp.labs.spark

"Streaming Ingestion File System Text File to Dataframe");
JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations
  .seconds(5));

代码示例来源:origin: melphi/spark-examples

JavaStreamingContext streamingContext = new JavaStreamingContext(conf, Durations.seconds(10));

代码示例来源:origin: org.apache.spark/spark-streaming_2.10

.numPartitions(10)
.partitioner(new HashPartitioner(10))
.timeout(Durations.seconds(10)));
.numPartitions(10)
.partitioner(new HashPartitioner(10))
.timeout(Durations.seconds(10)));

代码示例来源:origin: org.apache.spark/spark-streaming_2.10

.numPartitions(10)
.partitioner(new HashPartitioner(10))
.timeout(Durations.seconds(10)));
.numPartitions(10)
.partitioner(new HashPartitioner(10))
.timeout(Durations.seconds(10)));

代码示例来源:origin: org.apache.spark/spark-streaming_2.11

.numPartitions(10)
.partitioner(new HashPartitioner(10))
.timeout(Durations.seconds(10)));
.numPartitions(10)
.partitioner(new HashPartitioner(10))
.timeout(Durations.seconds(10)));

代码示例来源:origin: org.apache.spark/spark-streaming_2.11

.numPartitions(10)
.partitioner(new HashPartitioner(10))
.timeout(Durations.seconds(10)));
.numPartitions(10)
.partitioner(new HashPartitioner(10))
.timeout(Durations.seconds(10)));

相关文章

微信公众号

最新文章

更多