org.apache.hadoop.fs.Path.<init>()方法的使用及代码示例

x33g5p2x  于2022-01-25 转载在 其他  
字(8.5k)|赞(0)|评价(0)|浏览(129)

本文整理了Java中org.apache.hadoop.fs.Path.<init>()方法的一些代码示例,展示了Path.<init>()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Path.<init>()方法的具体详情如下:
包路径:org.apache.hadoop.fs.Path
类名称:Path
方法名:<init>

Path.<init>介绍

[英]Construct a path from a String. Path strings are URIs, but with unescaped elements and some additional normalization.
[中]从字符串构造路径。路径字符串是URI,但带有未经扫描的元素和一些额外的规范化。

代码示例

代码示例来源:origin: apache/flink

/**
 * Returns the Path where the YARN application files should be uploaded to.
 *
 * @param appId YARN application id
 */
private Path getYarnFilesDir(final ApplicationId appId) throws IOException {
  final FileSystem fileSystem = FileSystem.get(yarnConfiguration);
  final Path homeDir = fileSystem.getHomeDirectory();
  return new Path(homeDir, ".flink/" + appId + '/');
}

代码示例来源:origin: apache/storm

private static Path getDirLockFile(Path dir) {
  return new Path(dir.toString() + Path.SEPARATOR_CHAR + DIR_LOCK_FILE);
}

代码示例来源:origin: alibaba/jstorm

@Override
  public Void run() throws IOException {
    FileSystem fs = renamedScriptPath.getFileSystem(conf);
    fs.rename(new Path(jstormMasterContext.scriptPath), renamedScriptPath);
    return null;
  }
});

代码示例来源:origin: Alluxio/alluxio

/**
 * Test for the {@link HadoopUtils#getPathWithoutScheme(Path)} method.
 */
@Test
public void testGetPathWithoutSchema() {
 final Path path = new Path("/foo/bar/baz");
 final String output = HadoopUtils.getPathWithoutScheme(path);
 assertEquals("/foo/bar/baz", output);
}

代码示例来源:origin: apache/hive

@Test
public void testAddPartitionNullLocationInTableToo() throws Exception {
 createTable(DB_NAME, TABLE_NAME, null);
 Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE, null);
 client.add_partition(partition);
 Partition part = client.getPartition(DB_NAME, TABLE_NAME, "year=2017");
 Assert.assertEquals(
   metaStore.getWarehouseRoot() + "/test_partition_db.db/test_partition_table/year=2017",
   part.getSd().getLocation());
 Assert.assertTrue(metaStore.isPathExists(new Path(part.getSd().getLocation())));
}

代码示例来源:origin: apache/hbase

@Test
public void testSetWALRootDir() throws Exception {
 Path p = new Path("file:///hbase/root");
 CommonFSUtils.setWALRootDir(conf, p);
 assertEquals(p.toString(), conf.get(CommonFSUtils.HBASE_WAL_DIR));
}

代码示例来源:origin: apache/hive

@Test
public void testMovePathsThatCanBeMerged() {
 final Path condInputPath = new Path("s3a://bucket/scratch/-ext-10000");
 final Path condOutputPath = new Path("s3a://bucket/scratch/-ext-10002");
 final Path targetMoveWorkPath = new Path("s3a://bucket/scratch/-ext-10003");
 final MoveWork mockWork = mock(MoveWork.class);
 when(mockWork.getLoadFileWork()).thenReturn(new LoadFileDesc(
   condOutputPath, targetMoveWorkPath, false, "", "", false));
 assertTrue("Merging BlobStore paths should be allowed.",
   GenMapRedUtils.shouldMergeMovePaths(hiveConf, condInputPath, condOutputPath, mockWork));
}

代码示例来源:origin: apache/hbase

@Test public void testMiniDFSCluster() throws Exception {
 HBaseTestingUtility hbt = new HBaseTestingUtility();
 MiniDFSCluster cluster = hbt.startMiniDFSCluster(null);
 FileSystem dfs = cluster.getFileSystem();
 Path dir = new Path("dir");
 Path qualifiedDir = dfs.makeQualified(dir);
 LOG.info("dir=" + dir + ", qualifiedDir=" + qualifiedDir);
 assertFalse(dfs.exists(qualifiedDir));
 assertTrue(dfs.mkdirs(qualifiedDir));
 assertTrue(dfs.delete(qualifiedDir, true));
 hbt.shutdownMiniCluster();
}

代码示例来源:origin: apache/hbase

@Test
public void testSkipEmptyColumns() throws Exception {
 Path bulkOutputPath = new Path(util.getDataTestDirOnTestFS(tn.getNameAsString()), "hfiles");
 args.put(ImportTsv.BULK_OUTPUT_CONF_KEY, bulkOutputPath.toString());
 args.put(ImportTsv.COLUMNS_CONF_KEY, "HBASE_ROW_KEY,HBASE_TS_KEY,FAM:A,FAM:B");
 args.put(ImportTsv.SEPARATOR_CONF_KEY, ",");
 args.put(ImportTsv.SKIP_EMPTY_COLUMNS, "true");
 // 2 Rows of data as input. Both rows are valid and only 3 columns are no-empty among 4
 String data = "KEY,1234,VALUE1,VALUE2\nKEY,1235,,VALUE2\n";
 doMROnTableTest(util, tn, FAMILY, data, args, 1, 3);
 util.deleteTable(tn);
}

代码示例来源:origin: apache/hbase

@Test
public void testNoSuchTable() throws IOException {
 final String name = "testNoSuchTable";
 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
 // Cleanup old tests if any detrius laying around.
 Path rootdir = new Path(UTIL.getDataTestDir(), name);
 TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
 assertNull("There shouldn't be any HTD for this table",
  htds.get(TableName.valueOf("NoSuchTable")));
}

代码示例来源:origin: apache/hive

@Test(expected = MetastoreException.class)
public void testInvalidPartitionKeyName()
 throws HiveException, AlreadyExistsException, IOException, MetastoreException {
 Table table = createTestTable();
 List<Partition> partitions = hive.getPartitions(table);
 assertEquals(2, partitions.size());
 // add a fake partition dir on fs
 fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf());
 Path fakePart = new Path(table.getDataLocation().toString(),
   "fakedate=2009-01-01/fakecity=sanjose");
 fs.mkdirs(fakePart);
 fs.deleteOnExit(fakePart);
 checker.checkMetastore(catName, dbName, tableName, null, new CheckResult());
}

代码示例来源:origin: apache/hbase

@Test
public void testShouldCreateNewTableDescriptorIfForcefulCreationIsFalse()
  throws IOException {
 final String name = this.name.getMethodName();
 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
 Path rootdir = new Path(UTIL.getDataTestDir(), name);
 FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
 assertTrue("Should create new table descriptor",
  fstd.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(), false));
}

代码示例来源:origin: apache/hbase

private Path generateUniqTempDir(boolean withDirCreated) throws IOException {
 FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
 Path dir = new Path(fs.getWorkingDirectory(), NAME);
 if (!fs.exists(dir)) {
  fs.mkdirs(dir);
 }
 Path newDir = new Path(dir, UUID.randomUUID().toString());
 if (withDirCreated) {
  fs.mkdirs(newDir);
 }
 return newDir;
}

代码示例来源:origin: apache/ignite

/** @throws Exception If failed. */
@Test
public void testSetWorkingDirectory() throws Exception {
  Path dir = new Path("/tmp/nested/dir");
  Path file = new Path("file");
  fs.mkdirs(dir);
  fs.setWorkingDirectory(dir);
  FSDataOutputStream os = fs.create(file);
  os.close();
  String filePath = fs.getFileStatus(new Path(dir, file)).getPath().toString();
  assertTrue(filePath.contains("/tmp/nested/dir/file"));
}

代码示例来源:origin: apache/ignite

/** @throws Exception If failed. */
@Test
public void testGetWorkingDirectory() throws Exception {
  Path dir = new Path("/tmp/some/dir");
  fs.mkdirs(dir);
  fs.setWorkingDirectory(dir);
  String path = fs.getWorkingDirectory().toString();
  assertTrue(path.endsWith("/tmp/some/dir"));
}

代码示例来源:origin: apache/hbase

@Test(expected = IOException.class) public void testBackupPathIsNotAccessible() throws Exception {
  Path path = new Path(PERMISSION_TEST_PATH);
  FileSystem rootFs = FileSystem.get(TEST_UTIL.getConnection().getConfiguration());
  rootFs.mkdirs(path.getParent());
  rootFs.setPermission(path.getParent(), FsPermission.createImmutable((short) 000));
  FileSystem fs =
    DFSTestUtil.getFileSystemAs(DIANA, TEST_UTIL.getConnection().getConfiguration());
  fs.mkdirs(path);
 }
}

代码示例来源:origin: apache/hbase

private List<Path> getFilesRecursively(String fileBackupDir)
  throws IllegalArgumentException, IOException {
 FileSystem fs = FileSystem.get((new Path(fileBackupDir)).toUri(), new Configuration());
 List<Path> list = new ArrayList<>();
 RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(fileBackupDir), true);
 while (it.hasNext()) {
  Path p = it.next().getPath();
  if (HFile.isHFileFormat(fs, p)) {
   list.add(p);
  }
 }
 return list;
}

代码示例来源:origin: apache/kylin

@Override
public void deleteSlice(String workingDir, String sliceFileName) throws IOException {
  Path path = new Path(workingDir, sliceFileName);
  logger.trace("delete slice at {}", path);
  if (fileSystem.exists(path)) {
    fileSystem.delete(path, false);
  }
}

代码示例来源:origin: apache/incubator-gobblin

private Path getHdfsLogDir(Path appWorkDir) throws IOException {
 Path logRootDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.APP_LOGS_DIR_NAME);
 if (!this.fs.exists(logRootDir)) {
  this.fs.mkdirs(logRootDir);
 }
 return logRootDir;
}

代码示例来源:origin: apache/hive

@Override
public boolean connect(StatsCollectionContext context) {
 conf = context.getHiveConf();
 List<String> statsDirs = context.getStatsTmpDirs();
 assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs;
 Path statsDir = new Path(statsDirs.get(0));
 LOG.debug("Connecting to : " + statsDir);
 statsMap = new HashMap<String, Map<String,String>>();
 try {
  return statsDir.getFileSystem(conf).exists(statsDir);
 } catch (IOException e) {
  LOG.error("Failed to check if dir exists", e);
  return false;
 }
}

相关文章