org.kitesdk.data.Dataset.with()方法的使用及代码示例

x33g5p2x  于2022-01-18 转载在 其他  
字(6.8k)|赞(0)|评价(0)|浏览(164)

本文整理了Java中org.kitesdk.data.Dataset.with()方法的一些代码示例,展示了Dataset.with()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Dataset.with()方法的具体详情如下:
包路径:org.kitesdk.data.Dataset
类名称:Dataset
方法名:with

Dataset.with介绍

暂无

代码示例

代码示例来源:origin: kite-sdk/kite-examples

@Override
public int run(String[] args) throws Exception {
 // Load the users dataset
 Dataset<Record> users = Datasets.load(
   "dataset:hdfs:/tmp/data/users", Record.class);
 // Get a reader for the dataset and read all the users
 DatasetReader<Record> reader = null;
 try {
  reader = users.with("favoriteColor", "green").newReader();
  for (GenericRecord user : reader) {
   System.out.println(user);
  }
 } finally {
  if (reader != null) {
   reader.close();
  }
 }
 return 0;
}

代码示例来源:origin: kite-sdk/kite

@Test
public void testConstraintWithEncodedCharacters() {
 assertViewUriEquivalent("encoded constraints",
   "view:file:/tmp/test_name?color=a%2Fb",
   test.with("color", "a/b"));
}

代码示例来源:origin: kite-sdk/kite

@Test
public void testMultiValueConstraintWithEncodedCharacters() {
 assertViewUriEquivalent("encoded multi-value constraints",
   "view:file:/tmp/test_name?color=a%2Cb,c",
   test.with("color", "a,b", "c"));
}

代码示例来源:origin: kite-sdk/kite

@Test
public void testMixedConstraintViews() {
 assertViewUriEquivalent("id, color, and time constraints",
   "view:file:/tmp/test_name?color=,orange&id=exists()&timestamp=[0,9)",
   test.with("color", "", "orange").with("id")
     .from("timestamp", 0L).toBefore("timestamp", 9L));
}

代码示例来源:origin: kite-sdk/kite

testDataset.with("user_id", 0L));
assertContentEquals(Sets.newHashSet(sepEvent),
  testDataset.with("user_id", 0L).to("timestamp", octInstant));
assertContentEquals(Sets.newHashSet(sepEvent),
  testDataset.to("timestamp", octInstant).with("user_id", 0L));

代码示例来源:origin: kite-sdk/kite

@Test
public void testSimpleViews() {
 assertViewUriEquivalent("dataset",
   "dataset:file:/tmp/test_name", test);
 assertViewUriEquivalent("to constraint",
   "view:file:/tmp/test_name?timestamp=(,0]",
   test.to("timestamp", 0L));
 assertViewUriEquivalent("View with toBefore constraint",
   "view:file:/tmp/test_name?timestamp=(,0)",
   test.toBefore("timestamp", 0L));
 assertViewUriEquivalent("View with from constraint",
   "view:file:/tmp/test_name?timestamp=[0,)",
   test.from("timestamp", 0L));
 assertViewUriEquivalent("View with fromAfter constraint",
   "view:file:/tmp/test_name?timestamp=(0,)",
   test.fromAfter("timestamp", 0L));
 assertViewUriEquivalent("View with in(\"\") constraint",
   "view:file:/tmp/test_name?color=in()",
   test.with("color", ""));
 assertViewUriEquivalent("View with in constraint",
   "view:file:/tmp/test_name?color=orange,red",
   test.with("color", "orange", "red"));
 assertViewUriEquivalent("View with exists constraint",
   "view:file:/tmp/test_name?id=",
   test.with("id"));
}

代码示例来源:origin: kite-sdk/kite

@Test
@SuppressWarnings("deprecation")
public void testSignalReadyOutputView() throws Exception {
 Assume.assumeTrue(!Hadoop.isHadoop1());
 populateInputDataset();
 populateOutputDataset(); // existing output will be overwritten
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 View<Record> outputView = outputDataset.with("name", "apple", "banana", "carrot");
 DatasetKeyOutputFormat.configure(job).appendTo(outputView).withType(GenericData.Record.class);
 Assert.assertTrue(job.waitForCompletion(true));
 Assert.assertFalse("Output dataset should not be signaled ready",
   ((Signalable)outputDataset).isReady());
 Assert.assertTrue("Output view should be signaled ready",
   ((Signalable)outputView).isReady());
}

代码示例来源:origin: kite-sdk/kite

@Test
public void testPartitionedDatasetWithEscapedChars() throws Exception {
 File folder = temp.newFolder("a/b/c/d/e/dataset_name");
 Path root = new Path(temp.getRoot().toURI());
 FileSystem fs = LocalFileSystem.getInstance();
 URI datasetUri = URI.create("dataset:file:" + folder.getAbsolutePath());
 DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
   .schema(USER_SCHEMA)
   .partitionStrategy(new PartitionStrategy.Builder()
     .provided("s")
     .build())
   .build();
 Dataset<GenericRecord> dataset = Datasets.create(datasetUri, descriptor);
 // write two so that the descriptor uses the directory rather than a file
 writeUserToView(dataset.with("s", "test/-0"));
 writeUserToView(dataset.with("s", "test/-0"));
 Path datasetPath = new Path(folder.toURI());
 Path partitionPath = new Path(datasetPath, "s=test%2F-0");
 DatasetDescriptor actual = Iterables.getOnlyElement(
   FileSystemUtil.findPotentialDatasets(fs, root));
 Assert.assertFalse("Should not flag at mixed depth",
   descriptor.hasProperty("kite.filesystem.mixed-depth"));
 Assert.assertEquals("Location should be at the partition directory",
   URI.create(partitionPath.toString()), actual.getLocation());
 Assert.assertEquals("Should use user schema",
   USER_SCHEMA, actual.getSchema());
 Assert.assertEquals("Should have Avro format",
   Formats.AVRO, actual.getFormat());
 Assert.assertFalse("Should not be partitioned", actual.isPartitioned());
}

代码示例来源:origin: kite-sdk/kite

writeToView(users.with("version", 6), u1);
Assert.assertTrue(fileSystem.exists(new Path(datasetPath, "version=6")));
  DatasetTestUtilities.materialize(users.with("version", 6)));
  DatasetTestUtilities.materialize(users.with("version", 7)));

代码示例来源:origin: kite-sdk/kite

@Test
 public void testRefineIdentity() throws Exception {
   PartitionStrategy strategy = new PartitionStrategy.Builder()
       .identity("user_id")
       .build();

   DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
       .schemaUri("resource:standard_event.avsc")
       .partitionStrategy(strategy)
       .build();

   // Create a separate dataset to avoid conflicts with the above.
   Dataset<StandardEvent> identityDataset = repo.create(
     "ns", "test_identity", descriptor);

   DatasetWriter<StandardEvent> writer = null;

   try {
     writer = identityDataset.newWriter();
     writer.write(sepEvent);
     writer.write(octEvent);
     writer.write(novEvent);
   } finally {
     Closeables.close(writer, false);
   }

   assertContentEquals(Sets.newHashSet(sepEvent, novEvent),
       identityDataset.with("user_id", 0L));
 }
}

代码示例来源:origin: kite-sdk/kite

notPartitioned.toBefore("timestamp", now));
Assert.assertNotNull("with should succeed",
  notPartitioned.with("timestamp", now));

相关文章