org.apache.spark.api.java.JavaPairRDD.lookup()方法的使用及代码示例

x33g5p2x  于2022-01-21 转载在 其他  
字(12.7k)|赞(0)|评价(0)|浏览(80)

本文整理了Java中org.apache.spark.api.java.JavaPairRDD.lookup()方法的一些代码示例，展示了JavaPairRDD.lookup()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。JavaPairRDD.lookup()方法的具体详情如下：
包路径：org.apache.spark.api.java.JavaPairRDD
类名称：JavaPairRDD
方法名：lookup

JavaPairRDD.lookup介绍

暂无

代码示例

代码示例来源：origin: org.apache.spark/spark-core_2.11

@SuppressWarnings("unchecked")
@Test
public void lookup() {
 JavaPairRDD<String, String> categories = sc.parallelizePairs(Arrays.asList(
  new Tuple2<>("Apples", "Fruit"),
  new Tuple2<>("Oranges", "Fruit"),
  new Tuple2<>("Oranges", "Citrus")
 ));
 assertEquals(2, categories.lookup("Oranges").size());
 assertEquals(2, Iterables.size(categories.groupByKey().lookup("Oranges").get(0)));
}

代码示例来源：origin: org.apache.spark/spark-core

@SuppressWarnings("unchecked")
@Test
public void lookup() {
 JavaPairRDD<String, String> categories = sc.parallelizePairs(Arrays.asList(
  new Tuple2<>("Apples", "Fruit"),
  new Tuple2<>("Oranges", "Fruit"),
  new Tuple2<>("Oranges", "Citrus")
 ));
 assertEquals(2, categories.lookup("Oranges").size());
 assertEquals(2, Iterables.size(categories.groupByKey().lookup("Oranges").get(0)));
}

代码示例来源：origin: org.apache.spark/spark-core_2.10

@SuppressWarnings("unchecked")
@Test
public void foldByKey() {
 List<Tuple2<Integer, Integer>> pairs = Arrays.asList(
  new Tuple2<>(2, 1),
  new Tuple2<>(2, 1),
  new Tuple2<>(1, 1),
  new Tuple2<>(3, 2),
  new Tuple2<>(3, 1)
 );
 JavaPairRDD<Integer, Integer> rdd = sc.parallelizePairs(pairs);
 JavaPairRDD<Integer, Integer> sums = rdd.foldByKey(0, (a, b) -> a + b);
 assertEquals(1, sums.lookup(1).get(0).intValue());
 assertEquals(2, sums.lookup(2).get(0).intValue());
 assertEquals(3, sums.lookup(3).get(0).intValue());
}

代码示例来源：origin: org.apache.spark/spark-core_2.10

@SuppressWarnings("unchecked")
@Test
public void lookup() {
 JavaPairRDD<String, String> categories = sc.parallelizePairs(Arrays.asList(
  new Tuple2<>("Apples", "Fruit"),
  new Tuple2<>("Oranges", "Fruit"),
  new Tuple2<>("Oranges", "Citrus")
 ));
 assertEquals(2, categories.lookup("Oranges").size());
 assertEquals(2, Iterables.size(categories.groupByKey().lookup("Oranges").get(0)));
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

@SuppressWarnings("unchecked")
@Test
public void foldByKey() {
 List<Tuple2<Integer, Integer>> pairs = Arrays.asList(
  new Tuple2<>(2, 1),
  new Tuple2<>(2, 1),
  new Tuple2<>(1, 1),
  new Tuple2<>(3, 2),
  new Tuple2<>(3, 1)
 );
 JavaPairRDD<Integer, Integer> rdd = sc.parallelizePairs(pairs);
 JavaPairRDD<Integer, Integer> sums = rdd.foldByKey(0, (a, b) -> a + b);
 assertEquals(1, sums.lookup(1).get(0).intValue());
 assertEquals(2, sums.lookup(2).get(0).intValue());
 assertEquals(3, sums.lookup(3).get(0).intValue());
}

代码示例来源：origin: org.apache.spark/spark-core

@SuppressWarnings("unchecked")
@Test
public void foldByKey() {
 List<Tuple2<Integer, Integer>> pairs = Arrays.asList(
  new Tuple2<>(2, 1),
  new Tuple2<>(2, 1),
  new Tuple2<>(1, 1),
  new Tuple2<>(3, 2),
  new Tuple2<>(3, 1)
 );
 JavaPairRDD<Integer, Integer> rdd = sc.parallelizePairs(pairs);
 JavaPairRDD<Integer, Integer> sums = rdd.foldByKey(0, (a, b) -> a + b);
 assertEquals(1, sums.lookup(1).get(0).intValue());
 assertEquals(2, sums.lookup(2).get(0).intValue());
 assertEquals(3, sums.lookup(3).get(0).intValue());
}

代码示例来源：origin: org.apache.spark/spark-core_2.10

@Test
public void foldByKey() {
 List<Tuple2<Integer, Integer>> pairs = Arrays.asList(
  new Tuple2<>(2, 1),
  new Tuple2<>(2, 1),
  new Tuple2<>(1, 1),
  new Tuple2<>(3, 2),
  new Tuple2<>(3, 1)
 );
 JavaPairRDD<Integer, Integer> rdd = sc.parallelizePairs(pairs);
 JavaPairRDD<Integer, Integer> sums = rdd.foldByKey(0, (a, b) -> a + b);
 Assert.assertEquals(1, sums.lookup(1).get(0).intValue());
 Assert.assertEquals(2, sums.lookup(2).get(0).intValue());
 Assert.assertEquals(3, sums.lookup(3).get(0).intValue());
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
public void foldByKey() {
 List<Tuple2<Integer, Integer>> pairs = Arrays.asList(
  new Tuple2<>(2, 1),
  new Tuple2<>(2, 1),
  new Tuple2<>(1, 1),
  new Tuple2<>(3, 2),
  new Tuple2<>(3, 1)
 );
 JavaPairRDD<Integer, Integer> rdd = sc.parallelizePairs(pairs);
 JavaPairRDD<Integer, Integer> sums = rdd.foldByKey(0, (a, b) -> a + b);
 Assert.assertEquals(1, sums.lookup(1).get(0).intValue());
 Assert.assertEquals(2, sums.lookup(2).get(0).intValue());
 Assert.assertEquals(3, sums.lookup(3).get(0).intValue());
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

@Test
public void foldByKey() {
 List<Tuple2<Integer, Integer>> pairs = Arrays.asList(
  new Tuple2<>(2, 1),
  new Tuple2<>(2, 1),
  new Tuple2<>(1, 1),
  new Tuple2<>(3, 2),
  new Tuple2<>(3, 1)
 );
 JavaPairRDD<Integer, Integer> rdd = sc.parallelizePairs(pairs);
 JavaPairRDD<Integer, Integer> sums = rdd.foldByKey(0, (a, b) -> a + b);
 Assert.assertEquals(1, sums.lookup(1).get(0).intValue());
 Assert.assertEquals(2, sums.lookup(2).get(0).intValue());
 Assert.assertEquals(3, sums.lookup(3).get(0).intValue());
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

@Test
public void groupBy() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
 Function<Integer, Boolean> isOdd = x -> x % 2 == 0;
 JavaPairRDD<Boolean, Iterable<Integer>> oddsAndEvens = rdd.groupBy(isOdd);
 assertEquals(2, oddsAndEvens.count());
 assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
 oddsAndEvens = rdd.groupBy(isOdd, 1);
 assertEquals(2, oddsAndEvens.count());
 assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
public void groupBy() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
 Function<Integer, Boolean> isOdd = x -> x % 2 == 0;
 JavaPairRDD<Boolean, Iterable<Integer>> oddsAndEvens = rdd.groupBy(isOdd);
 assertEquals(2, oddsAndEvens.count());
 assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
 oddsAndEvens = rdd.groupBy(isOdd, 1);
 assertEquals(2, oddsAndEvens.count());
 assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

@Test
public void groupBy() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
 Function<Integer, Boolean> isOdd = x -> x % 2 == 0;
 JavaPairRDD<Boolean, Iterable<Integer>> oddsAndEvens = rdd.groupBy(isOdd);
 Assert.assertEquals(2, oddsAndEvens.count());
 Assert.assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 Assert.assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
 oddsAndEvens = rdd.groupBy(isOdd, 1);
 Assert.assertEquals(2, oddsAndEvens.count());
 Assert.assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 Assert.assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
}

代码示例来源：origin: org.apache.spark/spark-core_2.10

@Test
public void groupBy() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
 Function<Integer, Boolean> isOdd = x -> x % 2 == 0;
 JavaPairRDD<Boolean, Iterable<Integer>> oddsAndEvens = rdd.groupBy(isOdd);
 Assert.assertEquals(2, oddsAndEvens.count());
 Assert.assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 Assert.assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
 oddsAndEvens = rdd.groupBy(isOdd, 1);
 Assert.assertEquals(2, oddsAndEvens.count());
 Assert.assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 Assert.assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
public void groupBy() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
 Function<Integer, Boolean> isOdd = x -> x % 2 == 0;
 JavaPairRDD<Boolean, Iterable<Integer>> oddsAndEvens = rdd.groupBy(isOdd);
 Assert.assertEquals(2, oddsAndEvens.count());
 Assert.assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 Assert.assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
 oddsAndEvens = rdd.groupBy(isOdd, 1);
 Assert.assertEquals(2, oddsAndEvens.count());
 Assert.assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 Assert.assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
}

代码示例来源：origin: org.apache.spark/spark-core_2.10

@Test
public void groupBy() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
 Function<Integer, Boolean> isOdd = x -> x % 2 == 0;
 JavaPairRDD<Boolean, Iterable<Integer>> oddsAndEvens = rdd.groupBy(isOdd);
 assertEquals(2, oddsAndEvens.count());
 assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
 oddsAndEvens = rdd.groupBy(isOdd, 1);
 assertEquals(2, oddsAndEvens.count());
 assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
public void groupByOnPairRDD() {
 // Regression test for SPARK-4459
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
 Function<Tuple2<Integer, Integer>, Boolean> areOdd =
  x -> (x._1() % 2 == 0) && (x._2() % 2 == 0);
 JavaPairRDD<Integer, Integer> pairRDD = rdd.zip(rdd);
 JavaPairRDD<Boolean, Iterable<Tuple2<Integer, Integer>>> oddsAndEvens = pairRDD.groupBy(areOdd);
 assertEquals(2, oddsAndEvens.count());
 assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
 oddsAndEvens = pairRDD.groupBy(areOdd, 1);
 assertEquals(2, oddsAndEvens.count());
 assertEquals(2, Iterables.size(oddsAndEvens.lookup(true).get(0)));  // Evens
 assertEquals(5, Iterables.size(oddsAndEvens.lookup(false).get(0))); // Odds
}

代码示例来源：origin: org.apache.spark/spark-core

@SuppressWarnings("unchecked")
@Test
public void cogroup() {
 JavaPairRDD<String, String> categories = sc.parallelizePairs(Arrays.asList(
  new Tuple2<>("Apples", "Fruit"),
  new Tuple2<>("Oranges", "Fruit"),
  new Tuple2<>("Oranges", "Citrus")
  ));
 JavaPairRDD<String, Integer> prices = sc.parallelizePairs(Arrays.asList(
  new Tuple2<>("Oranges", 2),
  new Tuple2<>("Apples", 3)
 ));
 JavaPairRDD<String, Tuple2<Iterable<String>, Iterable<Integer>>> cogrouped =
   categories.cogroup(prices);
 assertEquals("[Fruit, Citrus]", Iterables.toString(cogrouped.lookup("Oranges").get(0)._1()));
 assertEquals("[2]", Iterables.toString(cogrouped.lookup("Oranges").get(0)._2()));
 cogrouped.collect();
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

@SuppressWarnings("unchecked")
@Test
public void keyByOnPairRDD() {
 // Regression test for SPARK-4459
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
 Function<Tuple2<Integer, Integer>, String> sumToString = x -> String.valueOf(x._1() + x._2());
 JavaPairRDD<Integer, Integer> pairRDD = rdd.zip(rdd);
 JavaPairRDD<String, Tuple2<Integer, Integer>> keyed = pairRDD.keyBy(sumToString);
 assertEquals(7, keyed.count());
 assertEquals(1, (long) keyed.lookup("2").get(0)._1());
}

代码示例来源：origin: org.apache.spark/spark-core_2.10

@SuppressWarnings("unchecked")
@Test
public void keyByOnPairRDD() {
 // Regression test for SPARK-4459
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
 Function<Tuple2<Integer, Integer>, String> sumToString = x -> String.valueOf(x._1() + x._2());
 JavaPairRDD<Integer, Integer> pairRDD = rdd.zip(rdd);
 JavaPairRDD<String, Tuple2<Integer, Integer>> keyed = pairRDD.keyBy(sumToString);
 assertEquals(7, keyed.count());
 assertEquals(1, (long) keyed.lookup("2").get(0)._1());
}

代码示例来源：origin: org.apache.spark/spark-core

@SuppressWarnings("unchecked")
@Test
public void keyByOnPairRDD() {
 // Regression test for SPARK-4459
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
 Function<Tuple2<Integer, Integer>, String> sumToString = x -> String.valueOf(x._1() + x._2());
 JavaPairRDD<Integer, Integer> pairRDD = rdd.zip(rdd);
 JavaPairRDD<String, Tuple2<Integer, Integer>> keyed = pairRDD.keyBy(sumToString);
 assertEquals(7, keyed.count());
 assertEquals(1, (long) keyed.lookup("2").get(0)._1());
}

内容来源于网络，如有侵权，请联系作者删除！

JavaPairRDD

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多

    JavaPairRDD类方法

    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com