org.apache.spark.api.java.JavaRDD.cache()方法的使用及代码示例

x33g5p2x  于2022-01-21 转载在 其他  
字(7.1k)|赞(0)|评价(0)|浏览(159)

本文整理了Java中org.apache.spark.api.java.JavaRDD.cache()方法的一些代码示例，展示了JavaRDD.cache()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。JavaRDD.cache()方法的具体详情如下：
包路径：org.apache.spark.api.java.JavaRDD
类名称：JavaRDD
方法名：cache

JavaRDD.cache介绍

暂无

代码示例

代码示例来源：origin: deeplearning4j/dl4j-examples

throw new IllegalStateException("No saved network parameters at " + f.getAbsolutePath());
Arrays.sort(content);
testDataPaths.cache();

代码示例来源：origin: OryxProject/oryx

newData.cache();
pastData.cache();
pastData.foreachPartition(p -> {});

代码示例来源：origin: databricks/learning-spark

trainingData.cache(); // Cache data since Logistic Regression is an iterative algorithm.

代码示例来源：origin: deeplearning4j/dl4j-examples

JavaRDD<String> trainDataPaths = SparkUtils.listPaths(sc, trainDataPathRootDir);
JavaRDD<String> testDataPaths = totalExamplesTest <= 0 ? null : listPathsSubset(sc, testDataPathRootDir, totalExamplesTest, 12345);
trainDataPaths.cache();
if(testDataPaths != null)
  testDataPaths.cache();

代码示例来源：origin: OryxProject/oryx

parsedTestRDD.cache();

代码示例来源：origin: org.apache.spark/spark-core_2.10

@Test
 public void testJavaJdbcRDD() throws Exception {
  JavaRDD<Integer> rdd = JdbcRDD.create(
   sc,
   () -> DriverManager.getConnection("jdbc:derby:target/JavaJdbcRDDSuiteDb"),
   "SELECT DATA FROM FOO WHERE ? <= ID AND ID <= ?",
   1, 100, 1,
   r -> r.getInt(1)
  ).cache();

  Assert.assertEquals(100, rdd.count());
  Assert.assertEquals(Integer.valueOf(10100), rdd.reduce((i1, i2) -> i1 + i2));
 }
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

@Test
 public void testJavaJdbcRDD() throws Exception {
  JavaRDD<Integer> rdd = JdbcRDD.create(
   sc,
   () -> DriverManager.getConnection("jdbc:derby:target/JavaJdbcRDDSuiteDb"),
   "SELECT DATA FROM FOO WHERE ? <= ID AND ID <= ?",
   1, 100, 1,
   r -> r.getInt(1)
  ).cache();

  Assert.assertEquals(100, rdd.count());
  Assert.assertEquals(Integer.valueOf(10100), rdd.reduce((i1, i2) -> i1 + i2));
 }
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
 public void testJavaJdbcRDD() throws Exception {
  JavaRDD<Integer> rdd = JdbcRDD.create(
   sc,
   () -> DriverManager.getConnection("jdbc:derby:target/JavaJdbcRDDSuiteDb"),
   "SELECT DATA FROM FOO WHERE ? <= ID AND ID <= ?",
   1, 100, 1,
   r -> r.getInt(1)
  ).cache();

  Assert.assertEquals(100, rdd.count());
  Assert.assertEquals(Integer.valueOf(10100), rdd.reduce((i1, i2) -> i1 + i2));
 }
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

@Test
public void testGetPersistentRDDs() {
 java.util.Map<Integer, JavaRDD<?>> cachedRddsMap = sc.getPersistentRDDs();
 assertTrue(cachedRddsMap.isEmpty());
 JavaRDD<String> rdd1 = sc.parallelize(Arrays.asList("a", "b")).setName("RDD1").cache();
 JavaRDD<String> rdd2 = sc.parallelize(Arrays.asList("c", "d")).setName("RDD2").cache();
 cachedRddsMap = sc.getPersistentRDDs();
 assertEquals(2, cachedRddsMap.size());
 assertEquals("RDD1", cachedRddsMap.get(0).name());
 assertEquals("RDD2", cachedRddsMap.get(1).name());
}

代码示例来源：origin: org.apache.spark/spark-core_2.10

@Test
public void testGetPersistentRDDs() {
 java.util.Map<Integer, JavaRDD<?>> cachedRddsMap = sc.getPersistentRDDs();
 assertTrue(cachedRddsMap.isEmpty());
 JavaRDD<String> rdd1 = sc.parallelize(Arrays.asList("a", "b")).setName("RDD1").cache();
 JavaRDD<String> rdd2 = sc.parallelize(Arrays.asList("c", "d")).setName("RDD2").cache();
 cachedRddsMap = sc.getPersistentRDDs();
 assertEquals(2, cachedRddsMap.size());
 assertEquals("RDD1", cachedRddsMap.get(0).name());
 assertEquals("RDD2", cachedRddsMap.get(1).name());
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
public void testGetPersistentRDDs() {
 java.util.Map<Integer, JavaRDD<?>> cachedRddsMap = sc.getPersistentRDDs();
 assertTrue(cachedRddsMap.isEmpty());
 JavaRDD<String> rdd1 = sc.parallelize(Arrays.asList("a", "b")).setName("RDD1").cache();
 JavaRDD<String> rdd2 = sc.parallelize(Arrays.asList("c", "d")).setName("RDD2").cache();
 cachedRddsMap = sc.getPersistentRDDs();
 assertEquals(2, cachedRddsMap.size());
 assertEquals("RDD1", cachedRddsMap.get(0).name());
 assertEquals("RDD2", cachedRddsMap.get(1).name());
}

代码示例来源：origin: OryxProject/oryx

parsedRDD.cache();

代码示例来源：origin: org.apache.spark/spark-core_2.10

@Test
public void map() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));
 JavaDoubleRDD doubles = rdd.mapToDouble(x -> 1.0 * x).cache();
 doubles.collect();
 JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x))
  .cache();
 pairs.collect();
 JavaRDD<String> strings = rdd.map(Object::toString).cache();
 strings.collect();
}

代码示例来源：origin: org.apache.spark/spark-core_2.10

@Test
public void map() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));
 JavaDoubleRDD doubles = rdd.mapToDouble(Integer::doubleValue).cache();
 doubles.collect();
 JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x)).cache();
 pairs.collect();
 JavaRDD<String> strings = rdd.map(Object::toString).cache();
 strings.collect();
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
public void map() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));
 JavaDoubleRDD doubles = rdd.mapToDouble(Integer::doubleValue).cache();
 doubles.collect();
 JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x)).cache();
 pairs.collect();
 JavaRDD<String> strings = rdd.map(Object::toString).cache();
 strings.collect();
}

代码示例来源：origin: org.apache.spark/spark-core

@Test
public void map() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));
 JavaDoubleRDD doubles = rdd.mapToDouble(x -> 1.0 * x).cache();
 doubles.collect();
 JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x))
  .cache();
 pairs.collect();
 JavaRDD<String> strings = rdd.map(Object::toString).cache();
 strings.collect();
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

@Test
public void map() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));
 JavaDoubleRDD doubles = rdd.mapToDouble(Integer::doubleValue).cache();
 doubles.collect();
 JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x)).cache();
 pairs.collect();
 JavaRDD<String> strings = rdd.map(Object::toString).cache();
 strings.collect();
}

代码示例来源：origin: org.apache.spark/spark-core_2.11

@Test
public void map() {
 JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));
 JavaDoubleRDD doubles = rdd.mapToDouble(x -> 1.0 * x).cache();
 doubles.collect();
 JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x))
  .cache();
 pairs.collect();
 JavaRDD<String> strings = rdd.map(Object::toString).cache();
 strings.collect();
}

代码示例来源：origin: deeplearning4j/dl4j-examples

processed.cache();

代码示例来源：origin: mahmoudparsian/data-algorithms-book

static JavaRDD<Vector> getFeatureizedData(String wikiData, JavaSparkContext context) {
  JavaRDD<Vector> data = context.textFile(wikiData).map(new Function<String, Vector>() {
    @Override
    public Vector call(String arg0) throws Exception {
      return Util.buildVector(arg0, "\t");
    }
  }).cache();
  return data;
}

内容来源于网络，如有侵权，请联系作者删除！

JavaRDD

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多

    JavaRDD类方法

    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com